Add configurable HTTP logging to prevent verbose output from GitHub operations

docs/docs/mp-packages/github.md

@@ -85,4 +85,61 @@ where `appsettings.json` is constructed as follows:
 
 Once configured, Modular Pipelines will handle authentication and authorization automatically by utilizing the provided access token and will deliver a GitHub client that is ready for immediate use.
 
-**Important Note:** This is just an example; **do not store any confidential data in appsettings.json, .env files, and similar.** Use secret storage, key-vault services, etc., for storing sensitive data, and then use the described configuration practices as shown in the example above.
+**Important Note:** This is just an example; **do not store any confidential data in appsettings.json, .env files, and similar.** Use secret storage, key-vault services, etc., for storing sensitive data, and then use the described configuration practices as shown in the example above.
+
+## HTTP Logging
+
+By default, GitHub API requests log only the **HTTP status code** and **duration** to avoid verbose output. This is especially important when uploading large files, as logging the entire request/response body can generate thousands of lines of output.
+
+You can control the HTTP logging level using the `HttpLogging` property in `GitHubOptions`. The available options are:
+
+- `HttpLoggingType.None` - No HTTP logging
+- `HttpLoggingType.Request` - Log HTTP requests
+- `HttpLoggingType.Response` - Log HTTP responses
+- `HttpLoggingType.StatusCode` - Log HTTP status codes
+- `HttpLoggingType.Duration` - Log request duration
+
+You can combine these flags using the bitwise OR operator (`|`).
+
+### Setting HTTP Logging for GitHub
+
+Configure HTTP logging through `GitHubOptions`:
+
+```cs
+await PipelineHostBuilder.Create()
+    .ConfigureServices((context, collection) =>
+    {
+        collection.Configure<GitHubOptions>(options =>
+        {
+            // Disable all HTTP logging
+            options.HttpLogging = HttpLoggingType.None;
+
+            // Or enable only specific logging
+            options.HttpLogging = HttpLoggingType.StatusCode | HttpLoggingType.Duration;
+
+            // Or enable full logging (use with caution)
+            options.HttpLogging = HttpLoggingType.Request | HttpLoggingType.Response | 
+                                  HttpLoggingType.StatusCode | HttpLoggingType.Duration;
+        });
+    })
+    .ExecutePipelineAsync();
+```
+
+### Setting Default HTTP Logging
+
+You can also set a default HTTP logging level for all HTTP requests (not just GitHub) using `PipelineOptions`:
+
+```cs
+await PipelineHostBuilder.Create()
+    .ConfigureServices((context, collection) =>
+    {
+        collection.Configure<PipelineOptions>(options =>
+        {
+            // This applies to all HTTP requests unless overridden
+            options.DefaultHttpLogging = HttpLoggingType.StatusCode | HttpLoggingType.Duration;
+        });
+    })
+    .ExecutePipelineAsync();
+```
+
+If `GitHubOptions.HttpLogging` is not set, the value from `PipelineOptions.DefaultHttpLogging` will be used.

src/ModularPipelines.Azure.Pipelines/AzurePipeline.cs

@@ -34,6 +34,6 @@ public void EndConsoleLogGroup(string name)
 
     public void LogToConsole(string value)
     {
-        ((IConsoleWriter)_moduleLoggerProvider.GetLogger()).LogToConsole(value);
+        ((IConsoleWriter) _moduleLoggerProvider.GetLogger()).LogToConsole(value);
     }
 }

src/ModularPipelines.GitHub/GitHub.cs

@@ -3,6 +3,7 @@
 using ModularPipelines.GitHub.Options;
 using ModularPipelines.Http;
 using ModularPipelines.Logging;
+using ModularPipelines.Options;
 using Octokit;
 using Octokit.Internal;
 
@@ -11,6 +12,7 @@ namespace ModularPipelines.GitHub;
 internal class GitHub : IGitHub
 {
     private readonly GitHubOptions _options;
+    private readonly PipelineOptions _pipelineOptions;
     private readonly IModuleLoggerProvider _moduleLoggerProvider;
     private readonly IHttpLogger _httpLogger;
     private readonly Lazy<IGitHubClient> _client;
@@ -23,12 +25,14 @@ internal class GitHub : IGitHub
 
     public GitHub(
       IOptions<GitHubOptions> options,
+      IOptions<PipelineOptions> pipelineOptions,
       IGitHubEnvironmentVariables environmentVariables,
       IGitHubRepositoryInfo gitHubRepositoryInfo,
       IModuleLoggerProvider moduleLoggerProvider,
       IHttpLogger httpLogger)
     {
         _options = options.Value;
+        _pipelineOptions = pipelineOptions.Value;
         _moduleLoggerProvider = moduleLoggerProvider;
         _httpLogger = httpLogger;
         EnvironmentVariables = environmentVariables;
@@ -49,7 +53,7 @@ public void EndConsoleLogGroup(string name)
 
     public void LogToConsole(string value)
     {
-        ((IConsoleWriter)_moduleLoggerProvider.GetLogger()).LogToConsole(value);
+        ((IConsoleWriter) _moduleLoggerProvider.GetLogger()).LogToConsole(value);
     }
 
     // PRIVATE METHODS
@@ -65,21 +69,49 @@ private IGitHubClient InitializeClient()
                     ?? EnvironmentVariables.Token
                     ?? throw new ArgumentException("No GitHub access token or GITHUB_TOKEN found in environment variables.");
 
+        var loggingType = _options.HttpLogging ?? _pipelineOptions.DefaultHttpLogging;
+
         var connection = new Connection(new ProductHeaderValue("ModularPipelines"),
           new HttpClientAdapter(() =>
           {
               var moduleLogger = _moduleLoggerProvider.GetLogger();
 
-              return new RequestLoggingHttpHandler(moduleLogger, _httpLogger)
+              // Build handler chain from innermost to outermost
+              HttpMessageHandler handler = new HttpClientHandler();
+
+              if (loggingType.HasFlag(HttpLoggingType.StatusCode))
+              {
+                  handler = new StatusCodeLoggingHttpHandler(moduleLogger, _httpLogger)
+                  {
+                      InnerHandler = handler,
+                  };
+              }
+
+              if (loggingType.HasFlag(HttpLoggingType.Response))
+              {
+                  handler = new ResponseLoggingHttpHandler(moduleLogger, _httpLogger)
+                  {
+                      InnerHandler = handler,
+                  };
+              }
+
+              if (loggingType.HasFlag(HttpLoggingType.Request))
+              {
+                  handler = new RequestLoggingHttpHandler(moduleLogger, _httpLogger)
+                  {
+                      InnerHandler = handler,
+                  };
+              }
+
+              if (loggingType.HasFlag(HttpLoggingType.Duration))
               {
-                  InnerHandler = new ResponseLoggingHttpHandler(moduleLogger, _httpLogger)
+                  handler = new DurationLoggingHttpHandler(moduleLogger, _httpLogger)
                   {
-                      InnerHandler = new StatusCodeLoggingHttpHandler(moduleLogger, _httpLogger)
-                      {
-                          InnerHandler = new HttpClientHandler(),
-                      },
-                  },
-              };
+                      InnerHandler = handler,
+                  };
+              }
+
+              return handler;
           }));
 
         var client = new GitHubClient(connection)

src/ModularPipelines.GitHub/Options/GitHubOptions.cs

@@ -1,9 +1,16 @@
 using ModularPipelines.Attributes;
+using ModularPipelines.Http;
 
 namespace ModularPipelines.GitHub.Options;
 
 public record GitHubOptions
 {
     [SecretValue]
     public string? AccessToken { get; set; }
+
+    /// <summary>
+    /// Gets or sets the HTTP logging level for GitHub API requests.
+    /// If not set, defaults to <see cref="ModularPipelines.Options.PipelineOptions.DefaultHttpLogging"/>.
+    /// </summary>
+    public HttpLoggingType? HttpLogging { get; set; }
 }

src/ModularPipelines.TeamCity/TeamCity.cs

@@ -27,6 +27,6 @@ public void EndConsoleLogGroup(string name)
 
     public void LogToConsole(string value)
     {
-        ((IConsoleWriter)_moduleLoggerProvider.GetLogger()).LogToConsole(value);
+        ((IConsoleWriter) _moduleLoggerProvider.GetLogger()).LogToConsole(value);
     }
 }

src/ModularPipelines/Options/PipelineOptions.cs

@@ -60,4 +60,9 @@ public bool ShowProgressInConsole
     /// Gets or sets the default command logging level for all commands.
     /// </summary>
     public CommandLogging DefaultCommandLogging { get; set; } = CommandLogging.Default;
+
+    /// <summary>
+    /// Gets or sets the default HTTP logging level for all HTTP requests.
+    /// </summary>
+    public Http.HttpLoggingType DefaultHttpLogging { get; set; } = Http.HttpLoggingType.StatusCode | Http.HttpLoggingType.Duration;
 }

src/ModularPipelines/SmartCollapsableLogging.cs

@@ -40,7 +40,7 @@ private IModuleLogger ModuleLogger
         }
     }
 
-    private IConsoleWriter ModuleLoggerConsoleWriter => (IConsoleWriter)ModuleLogger;
+    private IConsoleWriter ModuleLoggerConsoleWriter => (IConsoleWriter) ModuleLogger;
 
     public SmartCollapsableLogging(IServiceProvider serviceProvider,
         ISmartCollapsableLoggingStringBlockProvider smartCollapsableLoggingStringBlockProvider,

test/ModularPipelines.UnitTests/Extensions/FileExtensionsTests.cs

@@ -17,8 +17,8 @@ public async Task EnumerablePaths()
         }.AsEnumerable();
 
         var paths = files.AsPaths();
-        await Assert.That((object)paths).IsAssignableTo<IEnumerable<string>>();
-        await Assert.That((object)paths).IsNotAssignableTo<List<string>>();
+        await Assert.That((object) paths).IsAssignableTo<IEnumerable<string>>();
+        await Assert.That((object) paths).IsNotAssignableTo<List<string>>();
         await Assert.That(paths).IsEquivalentTo(new List<string>
         {
             Path.Combine(TestContext.WorkingDirectory, "File1.txt"),
@@ -36,8 +36,8 @@ public async Task ListPaths()
         };
 
         var paths = files.AsPaths();
-        await Assert.That((object)paths).IsAssignableTo<IEnumerable<string>>();
-        await Assert.That((object)paths).IsAssignableTo<List<string>>();
+        await Assert.That((object) paths).IsAssignableTo<IEnumerable<string>>();
+        await Assert.That((object) paths).IsAssignableTo<List<string>>();
         await Assert.That(paths).IsEquivalentTo([
             Path.Combine(TestContext.WorkingDirectory, "File1.txt"),
             Path.Combine(TestContext.WorkingDirectory, "File2.txt")

test/ModularPipelines.UnitTests/Extensions/FolderExtensionsTests.cs

@@ -16,8 +16,8 @@ public async Task EnumerablePaths()
         }.AsEnumerable();
 
         var paths = folders.AsPaths();
-        await Assert.That((object)paths).IsAssignableTo<IEnumerable<string>>();
-        await Assert.That((object)paths).IsNotAssignableTo<List<string>>();
+        await Assert.That((object) paths).IsAssignableTo<IEnumerable<string>>();
+        await Assert.That((object) paths).IsNotAssignableTo<List<string>>();
         await Assert.That(paths).IsEquivalentTo(new List<string>
         {
             Path.Combine(TestContext.WorkingDirectory, "Folder1"),
@@ -35,9 +35,9 @@ public async Task ListPaths()
         };
 
         var paths = folders.AsPaths();
-        await Assert.That((object)paths).IsAssignableTo<IEnumerable>();
-        await Assert.That((object)paths).IsAssignableTo<IEnumerable<string>>();
-        await Assert.That((object)paths).IsAssignableTo<List<string>>();
+        await Assert.That((object) paths).IsAssignableTo<IEnumerable>();
+        await Assert.That((object) paths).IsAssignableTo<IEnumerable<string>>();
+        await Assert.That((object) paths).IsAssignableTo<List<string>>();
         await Assert.That(paths).IsEquivalentTo([
             Path.Combine(TestContext.WorkingDirectory, "Folder1"),
             Path.Combine(TestContext.WorkingDirectory, "Folder2")

test/ModularPipelines.UnitTests/Helpers/EnvironmentContextTests.cs

@@ -29,7 +29,7 @@ public async Task Can_List_Environment_Variables()
 
         var result = context.EnvironmentVariables.GetEnvironmentVariables();
         await Assert.That(result).IsNotNull();
-        await Assert.That((object)result).IsAssignableTo<IDictionary<string, string>>();
+        await Assert.That((object) result).IsAssignableTo<IDictionary<string, string>>();
         await Assert.That(result[guid]).IsEqualTo("Foo bar!");
     }
 

test/ModularPipelines.UnitTests/Helpers/GitHubHttpLoggingTests.cs

@@ -0,0 +1,107 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+using ModularPipelines.Context;
+using ModularPipelines.GitHub.Options;
+using ModularPipelines.Http;
+using ModularPipelines.Modules;
+using ModularPipelines.Options;
+using ModularPipelines.TestHelpers;
+
+namespace ModularPipelines.UnitTests.Helpers;
+
+public class GitHubHttpLoggingTests : TestBase
+{
+    public class GitHubOptionsModule : Module<GitHubOptions>
+    {
+        protected override async Task<GitHubOptions?> ExecuteAsync(IPipelineContext context, CancellationToken cancellationToken)
+        {
+            await Task.Yield();
+            var options = context.ServiceProvider.GetRequiredService<IOptions<GitHubOptions>>();
+            return options.Value;
+        }
+    }
+
+    public class PipelineOptionsModule : Module<PipelineOptions>
+    {
+        protected override async Task<PipelineOptions?> ExecuteAsync(IPipelineContext context, CancellationToken cancellationToken)
+        {
+            await Task.Yield();
+            var options = context.ServiceProvider.GetRequiredService<IOptions<PipelineOptions>>();
+            return options.Value;
+        }
+    }
+
+    [Test]
+    public async Task GitHub_HttpLogging_Defaults_To_PipelineOptions()
+    {
+        var module = await RunModule<PipelineOptionsModule>();
+
+        var pipelineOptions = module.Result.Value!;
+
+        using (Assert.Multiple())
+        {
+            await Assert.That(pipelineOptions).IsNotNull();
+            await Assert.That(pipelineOptions.DefaultHttpLogging).IsEqualTo(HttpLoggingType.StatusCode | HttpLoggingType.Duration);
+        }
+    }
+
+    [Test]
+    public async Task GitHub_HttpLogging_Can_Be_Overridden()
+    {
+        var (service, _) = await GetService<IOptions<GitHubOptions>>((_, collection) =>
+        {
+            collection.Configure<GitHubOptions>(opt =>
+            {
+                opt.HttpLogging = HttpLoggingType.Request | HttpLoggingType.Response;
+            });
+        });
+
+        var options = service.Value;
+
+        using (Assert.Multiple())
+        {
+            await Assert.That(options).IsNotNull();
+            await Assert.That(options.HttpLogging).IsEqualTo(HttpLoggingType.Request | HttpLoggingType.Response);
+        }
+    }
+
+    [Test]
+    public async Task GitHub_HttpLogging_Can_Be_Set_To_None()
+    {
+        var (service, _) = await GetService<IOptions<GitHubOptions>>((_, collection) =>
+        {
+            collection.Configure<GitHubOptions>(opt =>
+            {
+                opt.HttpLogging = HttpLoggingType.None;
+            });
+        });
+
+        var options = service.Value;
+
+        using (Assert.Multiple())
+        {
+            await Assert.That(options).IsNotNull();
+            await Assert.That(options.HttpLogging).IsEqualTo(HttpLoggingType.None);
+        }
+    }
+
+    [Test]
+    public async Task PipelineOptions_DefaultHttpLogging_Can_Be_Configured()
+    {
+        var (service, _) = await GetService<IOptions<PipelineOptions>>((_, collection) =>
+        {
+            collection.Configure<PipelineOptions>(opt =>
+            {
+                opt.DefaultHttpLogging = HttpLoggingType.Request | HttpLoggingType.StatusCode;
+            });
+        });
+
+        var options = service.Value;
+
+        using (Assert.Multiple())
+        {
+            await Assert.That(options).IsNotNull();
+            await Assert.That(options.DefaultHttpLogging).IsEqualTo(HttpLoggingType.Request | HttpLoggingType.StatusCode);
+        }
+    }
+}

test/ModularPipelines.UnitTests/ModuleLoggerTests.cs

@@ -19,9 +19,9 @@ private class Module1 : Module
     {
         protected override async Task<IDictionary<string, object>?> ExecuteAsync(IPipelineContext context, CancellationToken cancellationToken)
         {
-            ((IConsoleWriter)context.Logger).LogToConsole(RandomString);
+            ((IConsoleWriter) context.Logger).LogToConsole(RandomString);
 
-            ((IConsoleWriter)context.Logger).LogToConsole(new MySecrets().Value1!);
+            ((IConsoleWriter) context.Logger).LogToConsole(new MySecrets().Value1!);
 
             return await NothingAsync();
         }