Implement --resume support (#2164), fixes #1799

Co-authored-by: Maykon Elias <melias@devexperts.com>
Co-authored-by: Adam Sitnik <adam.sitnik@gmail.com>

Author: 3 people

src/BenchmarkDotNet/Configs/ConfigOptions.cs

@@ -44,7 +44,11 @@ public enum ConfigOptions
         /// <summary>
         /// Performs apples-to-apples comparison for provided benchmarks and jobs. Experimental, will change in the near future!
         /// </summary>
-        ApplesToApples = 1 << 8
+        ApplesToApples = 1 << 8,
+        /// <summary>
+        /// Continue the execution if the last run was stopped.
+        /// </summary>
+        Resume = 1 << 9
     }
 
     internal static class ConfigOptionsExtensions

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -215,6 +215,9 @@ public bool UseDisassemblyDiagnoser
         [Option("noForcedGCs", Required = false, HelpText = "Specifying would not forcefully induce any GCs.")]
         public bool NoForcedGCs { get; set; }
 
+        [Option("resume", Required = false, Default = false, HelpText = "Continue the execution if the last run was stopped.")]
+        public bool Resume { get; set; }
+
         internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any();
 
         [Usage(ApplicationAlias = "")]

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -245,6 +245,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo
             config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput);
             config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog);
             config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples);
+            config.WithOption(ConfigOptions.Resume, options.Resume);
 
             if (options.MaxParameterColumnWidth.HasValue)
                 config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value));

src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs

@@ -3,6 +3,7 @@
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Columns;
@@ -42,6 +43,7 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
             var rootArtifactsFolderPath = GetRootArtifactsFolderPath(benchmarkRunInfos);
             var resultsFolderPath = GetResultsFolderPath(rootArtifactsFolderPath, benchmarkRunInfos);
             var logFilePath = Path.Combine(rootArtifactsFolderPath, title + ".log");
+            var idToResume = GetIdToResume(rootArtifactsFolderPath, title, benchmarkRunInfos);
 
             using (var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath)))
             {
@@ -62,7 +64,7 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
                     return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) };
 
                 int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length);
-                int benchmarksToRunCount = totalBenchmarkCount;
+                int benchmarksToRunCount = totalBenchmarkCount - (idToResume + 1); // ids are indexed from 0
                 compositeLogger.WriteLineHeader("// ***** BenchmarkRunner: Start   *****");
                 compositeLogger.WriteLineHeader($"// ***** Found {totalBenchmarkCount} benchmark(s) in total *****");
                 var globalChronometer = Chronometer.Start();
@@ -84,6 +86,16 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
 
                     foreach (var benchmarkRunInfo in supportedBenchmarks) // we run them in the old order now using the new build artifacts
                     {
+                        if (idToResume >= 0)
+                        {
+                            var benchmarkWithHighestIdForGivenType = benchmarkRunInfo.BenchmarksCases.Last();
+                            if (benchmarkToBuildResult[benchmarkWithHighestIdForGivenType].Id.Value <= idToResume)
+                            {
+                                compositeLogger.WriteLineInfo($"Skipping {benchmarkRunInfo.BenchmarksCases.Length} benchmark(s) defined by {benchmarkRunInfo.Type.GetCorrectCSharpTypeName()}.");
+                                continue;
+                            }
+                        }
+
                         var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, artifactsToCleanup,
                             resultsFolderPath, logFilePath, totalBenchmarkCount, in runsChronometer, ref benchmarksToRunCount);
 
@@ -678,5 +690,36 @@ private static void PrintValidationErrors(ILogger logger, IEnumerable<Validation
                 logger.WriteLine();
             }
         }
+
+        private static int GetIdToResume(string rootArtifactsFolderPath, string currentLogFileName, BenchmarkRunInfo[] benchmarkRunInfos)
+        {
+            if (benchmarkRunInfos.Any(benchmark => benchmark.Config.Options.IsSet(ConfigOptions.Resume)))
+            {
+                var directoryInfo = new DirectoryInfo(rootArtifactsFolderPath);
+                var logFilesExceptCurrent = directoryInfo
+                    .GetFiles($"{currentLogFileName.Split('-')[0]}*")
+                    .Where(file => Path.GetFileNameWithoutExtension(file.Name) != currentLogFileName)
+                    .ToArray();
+
+                if (logFilesExceptCurrent.Length > 0)
+                {
+                    var previousRunLogFile = logFilesExceptCurrent
+                        .OrderByDescending(o => o.LastWriteTime)
+                        .First();
+
+                    var regex = new Regex("--benchmarkId (.*?) in", RegexOptions.Compiled);
+                    foreach (var line in File.ReadLines(previousRunLogFile.FullName).Reverse())
+                    {
+                        var match = regex.Match(line);
+                        if (match.Success)
+                        {
+                            return int.Parse(match.Groups[1].Value);
+                        }
+                    }
+                }
+            }
+
+            return -1;
+        }
     }
 }

tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs

@@ -332,6 +332,21 @@ public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromAssembly()
             Assert.Contains("static", logger.GetLog());
         }
 
+        [Fact]
+        public void WhenUserAddTheResumeAttributeAndRunTheBenchmarks()
+        {
+            var logger = new OutputLogger(Output);
+            var config = ManualConfig.CreateEmpty().AddLogger(logger);
+
+            var types = new[] { typeof(WithDryAttributeAndCategory) };
+            var switcher = new BenchmarkSwitcher(types);
+
+            // the first run should execute all benchmarks
+            Assert.Single(switcher.Run(new[] { "--filter", "*WithDryAttributeAndCategory*" }, config));
+            // resuming after succesfull run should run nothing
+            Assert.Empty(switcher.Run(new[] { "--resume", "--filter", "*WithDryAttributeAndCategory*" }, config));
+        }
+
         private class UserInteractionMock : IUserInteraction
         {
             private readonly IReadOnlyList<Type> returnValue;