Repository: dotnet/BenchmarkDotNet Branch: master Commit: df0dfb7f0d12 Files: 1333 Total size: 4.6 MB Directory structure: gitextract_8mg2sr0i/ ├── .github/ │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── FUNDING.yml │ └── workflows/ │ ├── generate-changelog.yaml │ ├── generate-coverage-report.yaml │ ├── generate-gh-pages.yaml │ ├── publish-nightly.yaml │ ├── release.yaml │ ├── run-tests-selected.yaml │ └── run-tests.yaml ├── .gitignore ├── BenchmarkDotNet.slnx ├── BenchmarkDotNet.slnx.DotSettings ├── LICENSE.md ├── NuGet.Config ├── README.md ├── build/ │ ├── BenchmarkDotNet.Build/ │ │ ├── BenchmarkDotNet.Build.csproj │ │ ├── BuildContext.cs │ │ ├── CommandLineParser.cs │ │ ├── EnvVar.cs │ │ ├── Example.cs │ │ ├── Folder.DotSettings │ │ ├── HelpInfo.cs │ │ ├── Helpers/ │ │ │ ├── OctokitExtensions.cs │ │ │ └── Utils.cs │ │ ├── IHelpProvider.cs │ │ ├── Meta/ │ │ │ ├── Repo.cs │ │ │ └── VersionHistory.cs │ │ ├── Options/ │ │ │ ├── BoolOption.cs │ │ │ ├── IOption.cs │ │ │ ├── KnownOptions.cs │ │ │ ├── Option.cs │ │ │ └── StringOption.cs │ │ ├── Program.cs │ │ └── Runners/ │ │ ├── BuildRunner.cs │ │ ├── Changelog/ │ │ │ ├── ChangelogBuilder.cs │ │ │ └── ChangelogDetailsBuilder.cs │ │ ├── DocumentationRunner.cs │ │ ├── GitRunner.cs │ │ ├── ReleaseRunner.cs │ │ └── UnitTestRunner.cs │ ├── CodingStyle.ruleset │ ├── build.bat │ ├── build.ps1 │ ├── build.sh │ ├── cSpell.json │ ├── common.props │ ├── common.targets │ ├── sdk/ │ │ └── global.json │ ├── strongNameKey.snk │ └── versions.txt ├── build.cmd ├── docs/ │ ├── .gitignore │ ├── _redirects/ │ │ └── _redirects │ ├── api/ │ │ └── index.md │ ├── articles/ │ │ ├── configs/ │ │ │ ├── analysers.md │ │ │ ├── columns.md │ │ │ ├── configoptions.md │ │ │ ├── configs.md │ │ │ ├── diagnosers.md │ │ │ ├── exporters.md │ │ │ ├── filters.md │ │ │ ├── jobs.md │ │ │ ├── loggers.md │ │ │ ├── orderers.md │ │ │ ├── powerplans.md │ │ │ ├── toc.yml │ │ │ ├── toolchains.md │ │ │ └── validators.md │ │ ├── contributing/ │ │ │ ├── building.md │ │ │ ├── debugging.md │ │ │ ├── disassembler.md │ │ │ ├── documentation.md │ │ │ ├── miscellaneous.md │ │ │ ├── running-tests.md │ │ │ └── toc.yml │ │ ├── faq.md │ │ ├── features/ │ │ │ ├── baselines.md │ │ │ ├── disassembler.md │ │ │ ├── etwprofiler.md │ │ │ ├── event-pipe-profiler.md │ │ │ ├── parameterization.md │ │ │ ├── setup-and-cleanup.md │ │ │ ├── statistics.md │ │ │ ├── toc.yml │ │ │ ├── vsprofiler.md │ │ │ └── vstest.md │ │ ├── guides/ │ │ │ ├── choosing-run-strategy.md │ │ │ ├── console-args.md │ │ │ ├── customizing-runtime.md │ │ │ ├── dotnet-new-templates.md │ │ │ ├── getting-started.md │ │ │ ├── good-practices.md │ │ │ ├── how-it-works.md │ │ │ ├── how-to-run.md │ │ │ ├── nuget.md │ │ │ ├── toc.yml │ │ │ └── troubleshooting.md │ │ ├── license.md │ │ ├── overview.md │ │ ├── samples/ │ │ │ ├── IntroArguments.md │ │ │ ├── IntroArgumentsPriority.md │ │ │ ├── IntroArgumentsSource.md │ │ │ ├── IntroArrayParam.md │ │ │ ├── IntroBasic.md │ │ │ ├── IntroBenchmarkBaseline.md │ │ │ ├── IntroCategories.md │ │ │ ├── IntroCategoryBaseline.md │ │ │ ├── IntroCategoryDiscoverer.md │ │ │ ├── IntroColdStart.md │ │ │ ├── IntroComparableComplexParam.md │ │ │ ├── IntroConfigSource.md │ │ │ ├── IntroConfigUnion.md │ │ │ ├── IntroCustomMono.md │ │ │ ├── IntroCustomMonoArguments.md │ │ │ ├── IntroDeferredExecution.md │ │ │ ├── IntroDisassembly.md │ │ │ ├── IntroDisassemblyAllJits.md │ │ │ ├── IntroDisassemblyDry.md │ │ │ ├── IntroDisassemblyRyuJit.md │ │ │ ├── IntroDotMemoryDiagnoser.md │ │ │ ├── IntroDotTraceDiagnoser.md │ │ │ ├── IntroEnvVars.md │ │ │ ├── IntroEventPipeProfiler.md │ │ │ ├── IntroEventPipeProfilerAdvanced.md │ │ │ ├── IntroExceptionDiagnoser.md │ │ │ ├── IntroExport.md │ │ │ ├── IntroExportJson.md │ │ │ ├── IntroExportXml.md │ │ │ ├── IntroFilters.md │ │ │ ├── IntroFluentConfigBuilder.md │ │ │ ├── IntroGcMode.md │ │ │ ├── IntroGenericTypeArguments.md │ │ │ ├── IntroHardwareCounters.md │ │ │ ├── IntroInProcess.md │ │ │ ├── IntroInProcessWrongEnv.md │ │ │ ├── IntroInliningDiagnoser.md │ │ │ ├── IntroJitStatsDiagnoser.md │ │ │ ├── IntroJobBaseline.md │ │ │ ├── IntroJoin.md │ │ │ ├── IntroLargeAddressAware.md │ │ │ ├── IntroMaui.md │ │ │ ├── IntroMonitoring.md │ │ │ ├── IntroMultimodal.md │ │ │ ├── IntroNativeMemory.md │ │ │ ├── IntroNuGet.md │ │ │ ├── IntroOrderAttr.md │ │ │ ├── IntroOrderManual.md │ │ │ ├── IntroOutliers.md │ │ │ ├── IntroParams.md │ │ │ ├── IntroParamsAllValues.md │ │ │ ├── IntroParamsPriority.md │ │ │ ├── IntroParamsSource.md │ │ │ ├── IntroPercentiles.md │ │ │ ├── IntroPowerPlan.md │ │ │ ├── IntroRankColumn.md │ │ │ ├── IntroRatioSD.md │ │ │ ├── IntroRatioStyle.md │ │ │ ├── IntroSetupCleanupGlobal.md │ │ │ ├── IntroSetupCleanupIteration.md │ │ │ ├── IntroSetupCleanupTarget.md │ │ │ ├── IntroStaThread.md │ │ │ ├── IntroStatisticalTesting.md │ │ │ ├── IntroStatisticsColumns.md │ │ │ ├── IntroStopOnFirstError.md │ │ │ ├── IntroSummaryStyle.md │ │ │ ├── IntroTagColumn.md │ │ │ ├── IntroTailcall.md │ │ │ ├── IntroTemplate.txt │ │ │ ├── IntroThreadingDiagnoser.md │ │ │ ├── IntroUnicode.md │ │ │ ├── IntroVisualStudioProfiler.md │ │ │ ├── IntroWakeLock.md │ │ │ ├── IntroWasm.md │ │ │ └── toc.yml │ │ ├── team.md │ │ └── toc.yml │ ├── docfx.json │ ├── filter.yml │ ├── guide/ │ │ └── README.md │ ├── logo/ │ │ ├── logo-cmyk.eps │ │ └── logo-rgb.eps │ ├── template/ │ │ └── public/ │ │ ├── main.css │ │ └── main.js │ └── toc.yml ├── samples/ │ ├── BenchmarkDotNet.Maui.slnx │ ├── BenchmarkDotNet.Samples/ │ │ ├── BenchmarkDotNet.Samples.csproj │ │ ├── BenchmarkDotNet.Samples.csproj.DotSettings │ │ ├── IntroArguments.cs │ │ ├── IntroArgumentsPriority.cs │ │ ├── IntroArgumentsSource.cs │ │ ├── IntroArrayParam.cs │ │ ├── IntroBasic.cs │ │ ├── IntroBenchmarkBaseline.cs │ │ ├── IntroCategories.cs │ │ ├── IntroCategoryBaseline.cs │ │ ├── IntroCategoryDiscoverer.cs │ │ ├── IntroColdStart.cs │ │ ├── IntroComparableComplexParam.cs │ │ ├── IntroConfigSource.cs │ │ ├── IntroConfigUnion.cs │ │ ├── IntroCultureInfo.cs │ │ ├── IntroCustomMono.cs │ │ ├── IntroCustomMonoArguments.cs │ │ ├── IntroDeferredExecution.cs │ │ ├── IntroDisassembly.cs │ │ ├── IntroDisassemblyAllJits.cs │ │ ├── IntroDisassemblyDry.cs │ │ ├── IntroDisassemblyRyuJit.cs │ │ ├── IntroDotMemoryDiagnoser.cs │ │ ├── IntroDotTraceDiagnoser.cs │ │ ├── IntroEnvVars.cs │ │ ├── IntroEventPipeProfiler.cs │ │ ├── IntroEventPipeProfilerAdvanced.cs │ │ ├── IntroExceptionDiagnoser.cs │ │ ├── IntroExport.cs │ │ ├── IntroExportJson.cs │ │ ├── IntroExportXml.cs │ │ ├── IntroFilters.cs │ │ ├── IntroFluentConfigBuilder.cs │ │ ├── IntroGcMode.cs │ │ ├── IntroGenericTypeArguments.cs │ │ ├── IntroHardwareCounters.cs │ │ ├── IntroHidingColumns.cs │ │ ├── IntroInProcess.cs │ │ ├── IntroInProcessWrongEnv.cs │ │ ├── IntroInliningDiagnoser.cs │ │ ├── IntroJitStatsDiagnoser.cs │ │ ├── IntroJobBaseline.cs │ │ ├── IntroJoin.cs │ │ ├── IntroLargeAddressAware.cs │ │ ├── IntroMemoryRandomization.cs │ │ ├── IntroMonitoring.cs │ │ ├── IntroMultimodal.cs │ │ ├── IntroNativeMemory.cs │ │ ├── IntroNuGet.cs │ │ ├── IntroOrderAttr.cs │ │ ├── IntroOrderManual.cs │ │ ├── IntroOutliers.cs │ │ ├── IntroParams.cs │ │ ├── IntroParamsAllValues.cs │ │ ├── IntroParamsPriority.cs │ │ ├── IntroParamsSource.cs │ │ ├── IntroPercentiles.cs │ │ ├── IntroPerfCollectProfiler.cs │ │ ├── IntroPowerPlan.cs │ │ ├── IntroRankColumn.cs │ │ ├── IntroRatioSD.cs │ │ ├── IntroRatioStyle.cs │ │ ├── IntroSetupCleanupGlobal.cs │ │ ├── IntroSetupCleanupIteration.cs │ │ ├── IntroSetupCleanupTarget.cs │ │ ├── IntroSmokeEmptyBasic.cs │ │ ├── IntroSmokeIncrements.cs │ │ ├── IntroSmokeStringBuilder.cs │ │ ├── IntroSmokeValueTypes.cs │ │ ├── IntroStaThread.cs │ │ ├── IntroStatisticalTesting.cs │ │ ├── IntroStatisticsColumns.cs │ │ ├── IntroStopOnFirstError.cs │ │ ├── IntroSummaryStyle.cs │ │ ├── IntroTagColumn.cs │ │ ├── IntroTailcall.cs │ │ ├── IntroThreadingDiagnoser.cs │ │ ├── IntroUnicode.cs │ │ ├── IntroVisualStudioDiagnoser.cs │ │ ├── IntroWakeLock.cs │ │ ├── IntroWasm.cs │ │ ├── Program.cs │ │ └── Properties/ │ │ ├── AssemblyInfo.cs │ │ └── launchSettings.json │ ├── BenchmarkDotNet.Samples.FSharp/ │ │ ├── BenchmarkDotNet.Samples.FSharp.fsproj │ │ └── Program.fs │ ├── BenchmarkDotNet.Samples.Maui/ │ │ ├── App.xaml │ │ ├── App.xaml.cs │ │ ├── AppShell.xaml │ │ ├── AppShell.xaml.cs │ │ ├── BenchmarkDotNet.Samples.Maui.csproj │ │ ├── MainPage.xaml │ │ ├── MainPage.xaml.cs │ │ ├── MauiProgram.cs │ │ ├── Platforms/ │ │ │ ├── Android/ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ ├── MainActivity.cs │ │ │ │ ├── MainApplication.cs │ │ │ │ └── Resources/ │ │ │ │ └── values/ │ │ │ │ └── colors.xml │ │ │ ├── MacCatalyst/ │ │ │ │ ├── AppDelegate.cs │ │ │ │ ├── Entitlements.plist │ │ │ │ ├── Info.plist │ │ │ │ └── Program.cs │ │ │ ├── Windows/ │ │ │ │ ├── App.xaml │ │ │ │ ├── App.xaml.cs │ │ │ │ ├── Package.appxmanifest │ │ │ │ └── app.manifest │ │ │ └── iOS/ │ │ │ ├── AppDelegate.cs │ │ │ ├── Info.plist │ │ │ ├── Program.cs │ │ │ └── Resources/ │ │ │ └── PrivacyInfo.xcprivacy │ │ ├── Properties/ │ │ │ └── launchSettings.json │ │ └── Resources/ │ │ └── Styles/ │ │ ├── Colors.xaml │ │ └── Styles.xaml │ └── Directory.Build.props ├── src/ │ ├── BenchmarkDotNet/ │ │ ├── Analysers/ │ │ │ ├── AnalyserBase.cs │ │ │ ├── BaselineCustomAnalyzer.cs │ │ │ ├── CompositeAnalyser.cs │ │ │ ├── Conclusion.cs │ │ │ ├── ConclusionHelper.cs │ │ │ ├── ConclusionKind.cs │ │ │ ├── EnvironmentAnalyser.cs │ │ │ ├── HideColumnsAnalyser.cs │ │ │ ├── IAnalyser.cs │ │ │ ├── MinIterationTimeAnalyser.cs │ │ │ ├── MultimodalDistributionAnalyzer.cs │ │ │ ├── OutliersAnalyser.cs │ │ │ ├── RuntimeErrorAnalyser.cs │ │ │ ├── ZeroMeasurementAnalyser.cs │ │ │ └── ZeroMeasurementHelper.cs │ │ ├── Attributes/ │ │ │ ├── ArtifactsPathAttribute.cs │ │ │ ├── CategoryDiscovererAttribute.cs │ │ │ ├── Columns/ │ │ │ │ ├── AllStatisticsColumnAttribute.cs │ │ │ │ ├── BaselineColumnAttribute.cs │ │ │ │ ├── CategoriesColumnAttribute.cs │ │ │ │ ├── ColumnConfigBaseAttribute.cs │ │ │ │ ├── ConfidenceIntervalErrorColumnAttribute.cs │ │ │ │ ├── IterationsColumnAttribute.cs │ │ │ │ ├── KurtosisColumnAttribute.cs │ │ │ │ ├── LogicalGroupColumnAttribute.cs │ │ │ │ ├── MValueColumnAttribute.cs │ │ │ │ ├── MaxColumnAttribute.cs │ │ │ │ ├── MeanColumnAttribute.cs │ │ │ │ ├── MedianColumnAttribute.cs │ │ │ │ ├── MinColumnAttribute.cs │ │ │ │ ├── NamespaceColumnAttribute.cs │ │ │ │ ├── OperationsPerSecondAttribute.cs │ │ │ │ ├── Q1ColumnAttribute.cs │ │ │ │ ├── Q3ColumnAttribute.cs │ │ │ │ ├── RankColumnAttribute.cs │ │ │ │ ├── SkewnessColumnAttribute.cs │ │ │ │ ├── StdDevColumnAttribute.cs │ │ │ │ ├── StdErrorColumnAttribute.cs │ │ │ │ └── WelchTTestPValueColumnAttribute.cs │ │ │ ├── ConfigAttribute.cs │ │ │ ├── DisassemblyDiagnoserAttribute.cs │ │ │ ├── EventPipeProfilerAttribute.cs │ │ │ ├── ExceptionDiagnoserAttribute.cs │ │ │ ├── ExceptionDiagnoserConfig.cs │ │ │ ├── Exporters/ │ │ │ │ ├── AsciiDocExporterAttribute.cs │ │ │ │ ├── CsvExporterAttribute.cs │ │ │ │ ├── CsvMeasurementsExporterAttribute.cs │ │ │ │ ├── ExporterConfigBaseAttribute.cs │ │ │ │ ├── HtmlExporterAttribute.cs │ │ │ │ ├── JsonExporterAttribute.cs │ │ │ │ ├── MarkdownExporterAttribute.cs │ │ │ │ ├── OpenMetricsExporterAttribute.cs │ │ │ │ ├── PerfonarExporterAttribute.cs │ │ │ │ ├── PlainExporterAttribute.cs │ │ │ │ ├── RPlotExporterAttribute.cs │ │ │ │ └── XmlExporterAttribute.cs │ │ │ ├── Filters/ │ │ │ │ ├── AllCategoriesFilterAttribute.cs │ │ │ │ ├── AnyCategoriesFilterAttribute.cs │ │ │ │ ├── AotFilterAttribute.cs │ │ │ │ ├── FilterConfigBaseAttribute.cs │ │ │ │ ├── OperatingSystemsArchitectureFilterAttribute.cs │ │ │ │ └── OperatingSystemsFilterAttribute.cs │ │ │ ├── GroupBenchmarksByAttribute.cs │ │ │ ├── HardwareCountersAttribute.cs │ │ │ ├── HideColumnsAttribute.cs │ │ │ ├── Jobs/ │ │ │ │ ├── DryJobAttribute.cs │ │ │ │ ├── InProcessAttribute.cs │ │ │ │ ├── JobConfigbaseAttribute.cs │ │ │ │ ├── LegacyJitX64JobAttribute.cs │ │ │ │ ├── LegacyJitX86JobAttribute.cs │ │ │ │ ├── LongRunJobAttribute.cs │ │ │ │ ├── MediumRunJobAttribute.cs │ │ │ │ ├── MonoJobAttribute.cs │ │ │ │ ├── RyuJitX64JobAttribute.cs │ │ │ │ ├── RyuJitX86JobAttribute.cs │ │ │ │ ├── ShortRunJobAttribute.cs │ │ │ │ ├── SimpleJobAttribute.cs │ │ │ │ └── VeryLongRunJobAttribute.cs │ │ │ ├── KeepBenchmarkFilesAttribute.cs │ │ │ ├── MemoryDiagnoserAttribute.cs │ │ │ ├── Mutators/ │ │ │ │ ├── EvaluateOverheadAttribute.cs │ │ │ │ ├── GcConcurrentAttribute.cs │ │ │ │ ├── GcForceAttribute.cs │ │ │ │ ├── GcServerAttribute.cs │ │ │ │ ├── InnerIterationCountAttribute.cs │ │ │ │ ├── InvocationCountAttribute.cs │ │ │ │ ├── IterationCountAttribute.cs │ │ │ │ ├── IterationTimeAttribute.cs │ │ │ │ ├── JobMutatorConfigBaseAttribute.cs │ │ │ │ ├── MaxAbsoluteErrorAttribute.cs │ │ │ │ ├── MaxIterationCountAttribute.cs │ │ │ │ ├── MaxRelativeErrorAttribute.cs │ │ │ │ ├── MaxWarmupCountAttribute.cs │ │ │ │ ├── MemoryRandomizationAttribute.cs │ │ │ │ ├── MinInvokeCountAttribute.cs │ │ │ │ ├── MinIterationCountAttribute.cs │ │ │ │ ├── MinIterationTimeAttribute.cs │ │ │ │ ├── MinWarmupCountAttribute.cs │ │ │ │ ├── OutliersAttribute.cs │ │ │ │ ├── ProcessCountAttribute.cs │ │ │ │ ├── RunOncePerIterationAttribute.cs │ │ │ │ └── WarmupCountAttribute.cs │ │ │ ├── OrdererAttribute.cs │ │ │ ├── PerfCollectProfilerAttribute.cs │ │ │ ├── StopOnFirstErrorAttribute.cs │ │ │ ├── ThreadingDiagnoserAttribute.cs │ │ │ ├── UnicodeConsoleLoggerAttribute.cs │ │ │ ├── Validators/ │ │ │ │ ├── ExecutionValidatorAttribute.cs │ │ │ │ ├── ReturnValueValidatorAttribute.cs │ │ │ │ └── ValidatorConfigBaseAttribute.cs │ │ │ └── WakeLockAttribute.cs │ │ ├── BenchmarkDotNet.csproj │ │ ├── BenchmarkDotNet.csproj.DotSettings │ │ ├── BenchmarkDotNet.targets │ │ ├── Characteristics/ │ │ │ ├── Characteristic.cs │ │ │ ├── CharacteristicHelper.cs │ │ │ ├── CharacteristicObject.cs │ │ │ ├── CharacteristicObject`1.cs │ │ │ ├── CharacteristicPresenter.cs │ │ │ ├── CharacteristicSet.cs │ │ │ ├── CharacteristicSetPresenter.cs │ │ │ ├── Characteristic`1.cs │ │ │ ├── CompositeResolver.cs │ │ │ ├── IResolver.cs │ │ │ └── Resolver.cs │ │ ├── Code/ │ │ │ ├── ArrayParam.cs │ │ │ ├── CodeGenBenchmarkRunCallType.cs │ │ │ ├── CodeGenerator.cs │ │ │ ├── DeclarationsProvider.cs │ │ │ ├── EnumParam.cs │ │ │ └── IParam.cs │ │ ├── Columns/ │ │ │ ├── BaselineAllocationRatioColumn.cs │ │ │ ├── BaselineColumn.cs │ │ │ ├── BaselineCustomColumn.cs │ │ │ ├── BaselineRatioColumn.cs │ │ │ ├── CategoriesColumn.cs │ │ │ ├── Column.cs │ │ │ ├── ColumnCategory.cs │ │ │ ├── ColumnExtensions.cs │ │ │ ├── ColumnHidingByIdRule.cs │ │ │ ├── ColumnHidingByNameRule.cs │ │ │ ├── CompositeColumnProvider.cs │ │ │ ├── DefaultColumnProvider.cs │ │ │ ├── EmptyColumnProvider.cs │ │ │ ├── IColumn.cs │ │ │ ├── IColumnHidingRule.cs │ │ │ ├── IColumnProvider.cs │ │ │ ├── JobCharacteristicColumn.cs │ │ │ ├── LogicalGroupColumn.cs │ │ │ ├── MetricColumn.cs │ │ │ ├── ParamColumn.cs │ │ │ ├── RankColumn.cs │ │ │ ├── RatioColumnStyle.cs │ │ │ ├── SimpleColumnProvider.cs │ │ │ ├── StatisticColumn.cs │ │ │ ├── StatisticalTestColumn.cs │ │ │ ├── TagColumn.cs │ │ │ ├── TargetMethodColumn.cs │ │ │ └── UnitType.cs │ │ ├── Configs/ │ │ │ ├── BenchmarkLogicalGroupRule.cs │ │ │ ├── ConfigExtensions.cs │ │ │ ├── ConfigOptions.cs │ │ │ ├── ConfigUnionRule.cs │ │ │ ├── DebugConfig.cs │ │ │ ├── DefaultConfig.cs │ │ │ ├── IConfig.cs │ │ │ ├── IConfigSource.cs │ │ │ ├── ImmutableConfig.cs │ │ │ ├── ImmutableConfigBuilder.cs │ │ │ ├── ManualConfig.cs │ │ │ └── WakeLockType.cs │ │ ├── ConsoleArguments/ │ │ │ ├── CommandLineOptions.cs │ │ │ ├── ConfigParser.cs │ │ │ ├── CorrectionsSuggester.cs │ │ │ ├── LevenshteinDistanceCalculator.cs │ │ │ ├── ListBenchmarks/ │ │ │ │ ├── BenchmarkCasesPrinter.cs │ │ │ │ ├── FlatBenchmarkCasesPrinter.cs │ │ │ │ ├── IBenchmarkCasesPrinter.cs │ │ │ │ ├── ListBechnmarkCaseMode.cs │ │ │ │ ├── Node.cs │ │ │ │ └── TreeBenchmarkCasesPrinter.cs │ │ │ ├── LoggerWrapper.cs │ │ │ └── RuntimeFlavor.cs │ │ ├── Detectors/ │ │ │ ├── Cpu/ │ │ │ │ ├── HardwareIntrinsics.cs │ │ │ │ ├── ICpuDetector.cs │ │ │ │ ├── Linux/ │ │ │ │ │ ├── LinuxCpuDetector.cs │ │ │ │ │ └── LinuxCpuInfoParser.cs │ │ │ │ ├── Windows/ │ │ │ │ │ ├── MosCpuDetector.cs │ │ │ │ │ ├── PowershellWmiCpuDetector.cs │ │ │ │ │ ├── PowershellWmiCpuInfoParser.cs │ │ │ │ │ ├── WindowsCpuDetector.cs │ │ │ │ │ ├── WmicCpuDetector.cs │ │ │ │ │ ├── WmicCpuInfoKeyNames.cs │ │ │ │ │ └── WmicCpuInfoParser.cs │ │ │ │ └── macOS/ │ │ │ │ ├── MacOsCpuDetector.cs │ │ │ │ └── SysctlCpuInfoParser.cs │ │ │ ├── CpuDetector.cs │ │ │ └── OsDetector.cs │ │ ├── Diagnosers/ │ │ │ ├── AllocatedMemoryMetricDescriptor.cs │ │ │ ├── AllocatedNativeMemoryDescriptor.cs │ │ │ ├── CompositeDiagnoser.cs │ │ │ ├── DiagnoserActionParameters.cs │ │ │ ├── DiagnoserResults.cs │ │ │ ├── DiagnosersLoader.cs │ │ │ ├── EventPipeProfile.cs │ │ │ ├── EventPipeProfileMapper.cs │ │ │ ├── EventPipeProfiler.cs │ │ │ ├── ExceptionDiagnoser.cs │ │ │ ├── HardwareCounter.cs │ │ │ ├── IDiagnoser.cs │ │ │ ├── IHardwareCounterDiagnoser.cs │ │ │ ├── IProfiler.cs │ │ │ ├── InProcessDiagnoserActionParameters.cs │ │ │ ├── InProcessDiagnoserHandlerData.cs │ │ │ ├── InProcessDiagnoserRouter.cs │ │ │ ├── MemoryDiagnoser.cs │ │ │ ├── MemoryDiagnoserConfig.cs │ │ │ ├── PerfCollectProfiler.cs │ │ │ ├── PerfCollectProfilerConfig.cs │ │ │ ├── PmcMetricDescriptor.cs │ │ │ ├── PmcStats.cs │ │ │ ├── PreciseMachineCounter.cs │ │ │ ├── RunMode.cs │ │ │ ├── SnapshotProfilerBase.cs │ │ │ ├── SpeedScopeExporter.cs │ │ │ ├── ThreadingDiagnoser.cs │ │ │ ├── ThreadingDiagnoserConfig.cs │ │ │ └── UnresolvedDiagnoser.cs │ │ ├── Disassemblers/ │ │ │ ├── Arm64Disassembler.cs │ │ │ ├── Arm64InstructionFormatter.cs │ │ │ ├── ClrMdArgs.cs │ │ │ ├── ClrMdDisassembler.cs │ │ │ ├── DataContracts.cs │ │ │ ├── DisassemblyAnalyzer.cs │ │ │ ├── DisassemblyDiagnoser.cs │ │ │ ├── DisassemblyDiagnoserConfig.cs │ │ │ ├── DisassemblySyntax.cs │ │ │ ├── Exporters/ │ │ │ │ ├── CombinedDisassemblyExporter.cs │ │ │ │ ├── DisassemblyPrettifier.cs │ │ │ │ ├── GithubMarkdownDiffDisassemblyExporter.cs │ │ │ │ ├── GithubMarkdownDisassemblyExporter.cs │ │ │ │ ├── HtmlDisassemblyExporter.cs │ │ │ │ └── SymbolResolver.cs │ │ │ ├── InstructionFormatter.cs │ │ │ ├── IntelDisassembler.cs │ │ │ ├── IntelInstructionFormatter.cs │ │ │ ├── MonoDisassembler.cs │ │ │ └── SourceCodeProvider.cs │ │ ├── Engines/ │ │ │ ├── AnonymousPipesHost.cs │ │ │ ├── BenchmarkSignal.cs │ │ │ ├── Consumer.cs │ │ │ ├── ConsumerExtensions.cs │ │ │ ├── DeadCodeEliminationHelper.cs │ │ │ ├── Engine.cs │ │ │ ├── EngineActualStage.cs │ │ │ ├── EngineEventSource.cs │ │ │ ├── EngineFactory.cs │ │ │ ├── EngineJitStage.cs │ │ │ ├── EngineParameters.cs │ │ │ ├── EnginePilotStage.cs │ │ │ ├── EngineResolver.cs │ │ │ ├── EngineStage.cs │ │ │ ├── EngineWarmupStage.cs │ │ │ ├── GcStats.cs │ │ │ ├── HostExtensions.cs │ │ │ ├── HostSignal.cs │ │ │ ├── IEngine.cs │ │ │ ├── IEngineFactory.cs │ │ │ ├── IHost.cs │ │ │ ├── IterationData.cs │ │ │ ├── IterationMode.cs │ │ │ ├── IterationStage.cs │ │ │ ├── NoAcknowledgementConsoleHost.cs │ │ │ ├── RunResults.cs │ │ │ └── RunStrategy.cs │ │ ├── Environments/ │ │ │ ├── BenchmarkEnvironmentInfo.cs │ │ │ ├── EnvironmentResolver.cs │ │ │ ├── GcResolver.cs │ │ │ ├── HostEnvironmentInfo.cs │ │ │ ├── InfrastructureResolver.cs │ │ │ ├── Jit.cs │ │ │ ├── PhysicalMemoryInfo.cs │ │ │ ├── Platform.cs │ │ │ ├── PowerPlan.cs │ │ │ └── Runtimes/ │ │ │ ├── ClrRuntime.cs │ │ │ ├── CoreRuntime.cs │ │ │ ├── CustomRuntime.cs │ │ │ ├── MonoAotLLVMRuntime.cs │ │ │ ├── MonoRuntime.cs │ │ │ ├── NativeAotRuntime.cs │ │ │ ├── R2RRuntime.cs │ │ │ ├── Runtime.cs │ │ │ └── WasmRuntime.cs │ │ ├── EventProcessors/ │ │ │ ├── CompositeEventProcessor.cs │ │ │ └── EventProcessor.cs │ │ ├── Exporters/ │ │ │ ├── AsciiDocExporter.cs │ │ │ ├── BenchmarkReportExporter.cs │ │ │ ├── CompositeExporter.cs │ │ │ ├── Csv/ │ │ │ │ ├── CsvExporter.cs │ │ │ │ ├── CsvHelper.cs │ │ │ │ ├── CsvMeasurementsExporter.cs │ │ │ │ ├── CsvSeparator.cs │ │ │ │ └── CsvSeparatorExtensions.cs │ │ │ ├── DefaultExporters.cs │ │ │ ├── ExporterBase.cs │ │ │ ├── FullNameProvider.cs │ │ │ ├── HtmlExporter.cs │ │ │ ├── IExporter.cs │ │ │ ├── IExporterDependencies.cs │ │ │ ├── InstructionPointerExporter.cs │ │ │ ├── Json/ │ │ │ │ ├── JsonExporter.cs │ │ │ │ └── JsonExporterBase.cs │ │ │ ├── MarkdownExporter.cs │ │ │ ├── OpenMetrics/ │ │ │ │ └── OpenMetricsExporter.cs │ │ │ ├── PerfonarJsonExporter.cs │ │ │ ├── PerfonarMdExporter.cs │ │ │ ├── PlainExporter.cs │ │ │ ├── RPlotExporter.cs │ │ │ └── Xml/ │ │ │ ├── IXmlSerializer.cs │ │ │ ├── SimpleXmlWriter.cs │ │ │ ├── SummaryDto.cs │ │ │ ├── XmlExporter.cs │ │ │ ├── XmlExporterBase.cs │ │ │ └── XmlSerializer.cs │ │ ├── Extensions/ │ │ │ ├── AssemblyExtensions.cs │ │ │ ├── CommonExtensions.cs │ │ │ ├── ConfigurationExtensions.cs │ │ │ ├── CultureInfoExtensions.cs │ │ │ ├── DoubleExtensions.cs │ │ │ ├── EncodingExtensions.cs │ │ │ ├── Hashing.cs │ │ │ ├── MathExtensions.cs │ │ │ ├── MethodInfoExtensions.cs │ │ │ ├── PathFeatures.cs │ │ │ ├── Polyfills/ │ │ │ │ ├── ArgumentExceptionExtensions.cs │ │ │ │ ├── ArgumentNullExceptionExtensions.cs │ │ │ │ └── EnumExtensions.cs │ │ │ ├── ProcessExtensions.cs │ │ │ ├── ReflectionExtensions.cs │ │ │ ├── ReportExtensions.cs │ │ │ ├── RuntimeMonikerExtensions.cs │ │ │ ├── StatisticsExtensions.cs │ │ │ ├── StringAndTextExtensions.cs │ │ │ └── ThreadExtensions.cs │ │ ├── Filters/ │ │ │ ├── AllCategoriesFilter.cs │ │ │ ├── AnyCategoriesFilter.cs │ │ │ ├── AttributesFilter.cs │ │ │ ├── DisjunctionFilter.cs │ │ │ ├── GlobFilter.cs │ │ │ ├── IFilter.cs │ │ │ ├── NameFilter.cs │ │ │ ├── SimpleFilter.cs │ │ │ └── UnionFilter.cs │ │ ├── Helpers/ │ │ │ ├── ArtifactFileNameHelper.cs │ │ │ ├── AsciiHelper.cs │ │ │ ├── Assertion.cs │ │ │ ├── AwaitHelper.cs │ │ │ ├── CodeAnnotations.cs │ │ │ ├── ConsoleExitHandler.cs │ │ │ ├── DefaultCultureInfo.cs │ │ │ ├── DirtyAssemblyResolveHelper.cs │ │ │ ├── DisposeAtProcessTermination.cs │ │ │ ├── ExternalToolsHelper.cs │ │ │ ├── FolderNameHelper.cs │ │ │ ├── FrameworkVersionHelper.cs │ │ │ ├── GenericBenchmarksBuilder.cs │ │ │ ├── HashCode.cs │ │ │ ├── LinuxOsReleaseHelper.cs │ │ │ ├── PowerManagementHelper.cs │ │ │ ├── PowerShellLocator.cs │ │ │ ├── ProcessHelper.cs │ │ │ ├── Reflection.Emit/ │ │ │ │ ├── EmitParameterInfo.cs │ │ │ │ ├── IlGeneratorCallExtensions.cs │ │ │ │ ├── IlGeneratorEmitOpExtensions.cs │ │ │ │ ├── IlGeneratorStatementExtensions.cs │ │ │ │ ├── MethodBuilderExtensions.cs │ │ │ │ └── TypeBuilderExtensions.cs │ │ │ ├── ResourceHelper.cs │ │ │ ├── SectionsHelper.cs │ │ │ ├── SourceCodeHelper.cs │ │ │ ├── TaskbarProgress.cs │ │ │ ├── UnitHelper.cs │ │ │ ├── UserInteractionHelper.cs │ │ │ └── XUnitHelper.cs │ │ ├── Jobs/ │ │ │ ├── AccuracyMode.cs │ │ │ ├── Argument.cs │ │ │ ├── EnvironmentMode.cs │ │ │ ├── EnvironmentVariable.cs │ │ │ ├── GcMode.cs │ │ │ ├── GcModeExtensions.cs │ │ │ ├── InfrastructureMode.cs │ │ │ ├── Job.cs │ │ │ ├── JobComparer.cs │ │ │ ├── JobExtensions.cs │ │ │ ├── JobIdGenerator.cs │ │ │ ├── JobMode`1.cs │ │ │ ├── MetaMode.cs │ │ │ └── RunMode.cs │ │ ├── Loggers/ │ │ │ ├── AccumulationLogger.cs │ │ │ ├── AsyncProcessOutputReader.cs │ │ │ ├── Broker.cs │ │ │ ├── CompositeLogger.cs │ │ │ ├── ConsoleLogger.cs │ │ │ ├── ILogger.cs │ │ │ ├── LinqPadLogger.cs │ │ │ ├── LogCapture.cs │ │ │ ├── LogKind.cs │ │ │ ├── LoggerExtensions.cs │ │ │ ├── LoggerWithPrefix.cs │ │ │ ├── NullLogger.cs │ │ │ ├── StreamLogger.cs │ │ │ └── TextLogger.cs │ │ ├── Mathematics/ │ │ │ ├── LegacyConfidenceInterval.cs │ │ │ ├── MathHelper.cs │ │ │ ├── MeasurementsStatistics.cs │ │ │ ├── NumeralSystem.cs │ │ │ ├── NumeralSystemExtensions.cs │ │ │ ├── PercentileValues.cs │ │ │ ├── RankHelper.cs │ │ │ ├── RatioStatistics.cs │ │ │ └── Statistics.cs │ │ ├── Models/ │ │ │ ├── BdnBenchmark.cs │ │ │ ├── BdnEnvironment.cs │ │ │ ├── BdnExecution.cs │ │ │ ├── BdnHostInfo.cs │ │ │ ├── BdnLifecycle.cs │ │ │ └── BdnSchema.cs │ │ ├── Order/ │ │ │ ├── CategoryComparer.cs │ │ │ ├── DefaultOrderer.cs │ │ │ ├── IOrderer.cs │ │ │ ├── JobOrderPolicy.cs │ │ │ ├── MethodOrderPolicy.cs │ │ │ └── SummaryOrderPolicy.cs │ │ ├── Parameters/ │ │ │ ├── ParameterComparer.cs │ │ │ ├── ParameterDefinition.cs │ │ │ ├── ParameterDefinitions.cs │ │ │ ├── ParameterEqualityComparer.cs │ │ │ ├── ParameterInstance.cs │ │ │ ├── ParameterInstances.cs │ │ │ └── SmartParamBuilder.cs │ │ ├── Portability/ │ │ │ ├── Antivirus.cs │ │ │ ├── CodeGen.cs │ │ │ ├── HyperV.cs │ │ │ ├── JitInfo.cs │ │ │ ├── Libc.cs │ │ │ ├── RuntimeInformation.cs │ │ │ ├── StringExtensions.cs │ │ │ ├── VMware.cs │ │ │ ├── VirtualBox.cs │ │ │ └── VirtualMachineHypervisor.cs │ │ ├── Properties/ │ │ │ ├── AssemblyInfo.cs │ │ │ └── BenchmarkDotNetInfo.cs │ │ ├── Reports/ │ │ │ ├── BaseliningStrategy.cs │ │ │ ├── BenchmarkReport.cs │ │ │ ├── BenchmarkReportExtensions.cs │ │ │ ├── DisplayPrecisionManager.cs │ │ │ ├── Measurement.cs │ │ │ ├── MeasurementExtensions.cs │ │ │ ├── Metric.cs │ │ │ ├── Summary.cs │ │ │ ├── SummaryExtensions.cs │ │ │ ├── SummaryStyle.cs │ │ │ ├── SummaryTable.cs │ │ │ └── SummaryTableExtensions.cs │ │ ├── Running/ │ │ │ ├── BenchmarkBuildInfo.cs │ │ │ ├── BenchmarkCase.cs │ │ │ ├── BenchmarkConverter.cs │ │ │ ├── BenchmarkId.cs │ │ │ ├── BenchmarkPartitioner.cs │ │ │ ├── BenchmarkRunInfo.cs │ │ │ ├── BenchmarkRunnerClean.cs │ │ │ ├── BenchmarkRunnerDirty.cs │ │ │ ├── BenchmarkSwitcher.cs │ │ │ ├── BuildPartition.cs │ │ │ ├── ConsoleTitler.cs │ │ │ ├── DefaultCategoryDiscoverer.cs │ │ │ ├── Descriptor.cs │ │ │ ├── DescriptorComparer.cs │ │ │ ├── ICategoryDiscoverer.cs │ │ │ ├── IUserInteraction.cs │ │ │ ├── InvalidBenchmarkDeclarationException.cs │ │ │ ├── PowerManagementApplier.cs │ │ │ ├── TypeFilter.cs │ │ │ ├── UserInteraction.cs │ │ │ ├── WakeLock.PInvoke.cs │ │ │ ├── WakeLock.SafePowerHandle.cs │ │ │ └── WakeLock.cs │ │ ├── Serialization/ │ │ │ ├── BdnJsonSerializer.cs │ │ │ ├── BdnJsonSerializerContext.cs │ │ │ └── BdnSimpleJsonSerializer.cs │ │ ├── Templates/ │ │ │ ├── BenchmarkProgram.txt │ │ │ ├── BenchmarkType.txt │ │ │ ├── BuildPlots.R │ │ │ ├── CsProj.txt │ │ │ ├── MonoAOTLLVMCsProj.txt │ │ │ ├── R2RCsProj.txt │ │ │ ├── WasmCsProj.txt │ │ │ ├── WasmLinkerDescription.xml │ │ │ ├── benchmark-main.mjs │ │ │ ├── highlightingLabelsScript.js │ │ │ └── perfcollect │ │ ├── Toolchains/ │ │ │ ├── AppConfigGenerator.cs │ │ │ ├── ArtifactsPaths.cs │ │ │ ├── CoreRun/ │ │ │ │ ├── CoreRunGenerator.cs │ │ │ │ ├── CoreRunPublisher.cs │ │ │ │ └── CoreRunToolchain.cs │ │ │ ├── CsProj/ │ │ │ │ ├── CsProjClassicNetToolchain.cs │ │ │ │ ├── CsProjCoreToolchain.cs │ │ │ │ └── CsProjGenerator.cs │ │ │ ├── DotNetCli/ │ │ │ │ ├── CustomDotNetCliToolchainBuilder.cs │ │ │ │ ├── DotNetCliBuilder.cs │ │ │ │ ├── DotNetCliCommand.cs │ │ │ │ ├── DotNetCliCommandExecutor.cs │ │ │ │ ├── DotNetCliCommandResult.cs │ │ │ │ ├── DotNetCliExecutor.cs │ │ │ │ ├── DotNetCliGenerator.cs │ │ │ │ ├── DotNetCliPublisher.cs │ │ │ │ ├── MsBuildErrorMapper.cs │ │ │ │ └── NetCoreAppSettings.cs │ │ │ ├── Executor.cs │ │ │ ├── GeneratorBase.cs │ │ │ ├── IBuilder.cs │ │ │ ├── IExecutor.cs │ │ │ ├── IGenerator.cs │ │ │ ├── IToolchain.cs │ │ │ ├── InProcess/ │ │ │ │ ├── Emit/ │ │ │ │ │ ├── Implementation/ │ │ │ │ │ │ ├── ConsumableTypeInfo.cs │ │ │ │ │ │ ├── Emitters/ │ │ │ │ │ │ │ ├── EmitExtensions.cs │ │ │ │ │ │ │ └── RunnableEmitter.cs │ │ │ │ │ │ └── Runnable/ │ │ │ │ │ │ ├── RunnableConstants.cs │ │ │ │ │ │ ├── RunnableProgram.cs │ │ │ │ │ │ ├── RunnableReflectionHelpers.cs │ │ │ │ │ │ └── RunnableReuse.cs │ │ │ │ │ ├── InProcessEmitArtifactsPath.cs │ │ │ │ │ ├── InProcessEmitBuilder.cs │ │ │ │ │ ├── InProcessEmitExecutor.cs │ │ │ │ │ ├── InProcessEmitGenerator.cs │ │ │ │ │ ├── InProcessEmitSettings.cs │ │ │ │ │ └── InProcessEmitToolchain.cs │ │ │ │ ├── InProcessHost.cs │ │ │ │ ├── InProcessSettings.cs │ │ │ │ ├── InProcessValidator.cs │ │ │ │ └── NoEmit/ │ │ │ │ ├── BenchmarkAction.cs │ │ │ │ ├── BenchmarkActionFactory.cs │ │ │ │ ├── BenchmarkActionFactory_Base.cs │ │ │ │ ├── BenchmarkActionFactory_Implementations.cs │ │ │ │ ├── InProcessNoEmitBuilder.cs │ │ │ │ ├── InProcessNoEmitExecutor.cs │ │ │ │ ├── InProcessNoEmitGenerator.cs │ │ │ │ ├── InProcessNoEmitRunner.cs │ │ │ │ ├── InProcessNoEmitSettings.cs │ │ │ │ └── InProcessNoEmitToolchain.cs │ │ │ ├── LargeAddressAware.cs │ │ │ ├── Mono/ │ │ │ │ ├── MonoAotBuilder.cs │ │ │ │ ├── MonoAotToolchain.cs │ │ │ │ ├── MonoGenerator.cs │ │ │ │ ├── MonoPublisher.cs │ │ │ │ └── MonoToolchain.cs │ │ │ ├── MonoAotLLVM/ │ │ │ │ ├── MonoAotCompilerMode.cs │ │ │ │ ├── MonoAotLLVMGenerator.cs │ │ │ │ └── MonoAotLLVMToolChain.cs │ │ │ ├── MonoWasm/ │ │ │ │ ├── WasmExecutor.cs │ │ │ │ ├── WasmGenerator.cs │ │ │ │ └── WasmToolchain.cs │ │ │ ├── NativeAot/ │ │ │ │ ├── Generator.cs │ │ │ │ ├── NativeAotToolchain.cs │ │ │ │ └── NativeAotToolchainBuilder.cs │ │ │ ├── Parameters/ │ │ │ │ └── ExecuteParameters.cs │ │ │ ├── R2R/ │ │ │ │ ├── R2RGenerator.cs │ │ │ │ └── R2RToolchain.cs │ │ │ ├── Results/ │ │ │ │ ├── BuildResult.cs │ │ │ │ ├── ExecuteResult.cs │ │ │ │ └── GenerateResult.cs │ │ │ ├── Roslyn/ │ │ │ │ ├── Builder.cs │ │ │ │ ├── Generator.cs │ │ │ │ ├── RoslynToolchain.cs │ │ │ │ └── RoslynWorkarounds.cs │ │ │ ├── Toolchain.cs │ │ │ └── ToolchainExtensions.cs │ │ └── Validators/ │ │ ├── BaselineValidator.cs │ │ ├── BenchmarkProcessValidator.cs │ │ ├── CompilationValidator.cs │ │ ├── CompositeValidator.cs │ │ ├── ConfigValidator.cs │ │ ├── DeferredExecutionValidator.cs │ │ ├── DiagnosersValidator.cs │ │ ├── DotNetSdkValidator.cs │ │ ├── ExecutionValidator.cs │ │ ├── ExecutionValidatorBase.cs │ │ ├── GenericBenchmarksValidator.cs │ │ ├── IValidator.cs │ │ ├── JitOptimizationsValidator.cs │ │ ├── ParamsAllValuesValidator.cs │ │ ├── ParamsValidator.cs │ │ ├── ReturnValueValidator.cs │ │ ├── RunModeValidator.cs │ │ ├── RuntimeValidator.cs │ │ ├── SetupCleanupValidator.cs │ │ ├── ShadowCopyValidator.cs │ │ ├── ValidationError.cs │ │ ├── ValidationErrorReporter.cs │ │ └── ValidationParameters.cs │ ├── BenchmarkDotNet.Analyzers/ │ │ ├── AnalyzerHelper.cs │ │ ├── AnalyzerReleases.Shipped.md │ │ ├── AnalyzerReleases.Unshipped.md │ │ ├── Attributes/ │ │ │ ├── ArgumentsAttributeAnalyzer.cs │ │ │ ├── GeneralArgumentAttributesAnalyzer.cs │ │ │ ├── GeneralParameterAttributesAnalyzer.cs │ │ │ ├── ParamsAllValuesAttributeAnalyzer.cs │ │ │ └── ParamsAttributeAnalyzer.cs │ │ ├── BenchmarkDotNet.Analyzers.csproj │ │ ├── BenchmarkDotNetAnalyzerResources.Designer.cs │ │ ├── BenchmarkDotNetAnalyzerResources.resx │ │ ├── BenchmarkRunner/ │ │ │ └── RunAnalyzer.cs │ │ ├── DiagnosticIds.cs │ │ └── General/ │ │ └── BenchmarkClassAnalyzer.cs │ ├── BenchmarkDotNet.Annotations/ │ │ ├── Attributes/ │ │ │ ├── ArgumentsAttribute.cs │ │ │ ├── ArgumentsSourceAttribute.cs │ │ │ ├── BenchmarkAttribute.cs │ │ │ ├── BenchmarkCategoryAttribute.cs │ │ │ ├── GenericTypeArgumentsAttribute.cs │ │ │ ├── GlobalCleanupAttribute.cs │ │ │ ├── GlobalSetupAttribute.cs │ │ │ ├── IterationCleanupAttribute.cs │ │ │ ├── IterationSetupAttribute.cs │ │ │ ├── ParamsAllValuesAttribute.cs │ │ │ ├── ParamsAttribute.cs │ │ │ ├── ParamsSourceAttribute.cs │ │ │ ├── PriorityAttribute.cs │ │ │ └── TargetedAttribute.cs │ │ ├── BenchmarkDotNet.Annotations.csproj │ │ ├── Jobs/ │ │ │ └── RuntimeMoniker.cs │ │ └── buildTransitive/ │ │ └── netstandard2.0/ │ │ └── BenchmarkDotNet.Annotations.targets │ ├── BenchmarkDotNet.Diagnostics.Windows/ │ │ ├── BenchmarkDotNet.Diagnostics.Windows.csproj │ │ ├── ConcurrencyVisualizerProfiler.cs │ │ ├── Configs/ │ │ │ ├── ConcurrencyVisualizerProfilerAttribute.cs │ │ │ ├── EtwProfilerAttribute.cs │ │ │ ├── InliningDiagnoserAttribute.cs │ │ │ ├── JitStatsDiagnoserAttribute.cs │ │ │ ├── NativeMemoryProfilerAttribute.cs │ │ │ └── TailCallDiagnoserAttribute.cs │ │ ├── EtwDiagnoser.cs │ │ ├── EtwProfiler.cs │ │ ├── EtwProfilerConfig.cs │ │ ├── HardwareCounters.cs │ │ ├── InliningDiagnoser.cs │ │ ├── JitDiagnoser.cs │ │ ├── JitStatsDiagnoser.cs │ │ ├── NativeMemoryProfiler.cs │ │ ├── Properties/ │ │ │ └── AssemblyInfo.cs │ │ ├── Sessions.cs │ │ ├── TailCallDiagnoser.cs │ │ └── Tracing/ │ │ ├── BenchmarkEvent.cs │ │ ├── EngineEventLogParser.cs │ │ ├── IterationEvent.cs │ │ ├── NativeMemoryLogParser.cs │ │ └── TraceLogParser.cs │ ├── BenchmarkDotNet.Diagnostics.dotMemory/ │ │ ├── BenchmarkDotNet.Diagnostics.dotMemory.csproj │ │ ├── DotMemoryDiagnoser.cs │ │ ├── DotMemoryDiagnoserAttribute.cs │ │ └── Properties/ │ │ └── AssemblyInfo.cs │ ├── BenchmarkDotNet.Diagnostics.dotTrace/ │ │ ├── BenchmarkDotNet.Diagnostics.dotTrace.csproj │ │ ├── DotTraceDiagnoser.cs │ │ ├── DotTraceDiagnoserAttribute.cs │ │ └── Properties/ │ │ └── AssemblyInfo.cs │ ├── BenchmarkDotNet.Disassembler/ │ │ ├── BenchmarkDotNet.Disassembler.csproj │ │ └── Program.cs │ ├── BenchmarkDotNet.Exporters.Plotting/ │ │ ├── BenchmarkDotNet.Exporters.Plotting.csproj │ │ └── ScottPlotExporter.cs │ ├── BenchmarkDotNet.TestAdapter/ │ │ ├── BenchmarkCaseExtensions.cs │ │ ├── BenchmarkDotNet.TestAdapter.csproj │ │ ├── BenchmarkEnumerator.cs │ │ ├── BenchmarkExecutor.cs │ │ ├── Remoting/ │ │ │ ├── BenchmarkEnumeratorWrapper.cs │ │ │ ├── BenchmarkExecutorWrapper.cs │ │ │ ├── MessageLoggerWrapper.cs │ │ │ ├── SerializationHelpers.cs │ │ │ └── TestExecutionRecorderWrapper.cs │ │ ├── Utility/ │ │ │ ├── LoggerHelper.cs │ │ │ └── TestCaseFilter.cs │ │ ├── VSTestAdapter.cs │ │ ├── VSTestEventProcessor.cs │ │ ├── VSTestLogger.cs │ │ ├── VSTestProperties.cs │ │ ├── build/ │ │ │ └── BenchmarkDotNet.TestAdapter.props │ │ └── entrypoints/ │ │ ├── EntryPoint.cs │ │ ├── EntryPoint.fs │ │ └── EntryPoint.vb │ └── BenchmarkDotNet.Weaver/ │ ├── .gitignore │ ├── BenchmarkDotNet.Weaver.csproj │ ├── buildTransitive/ │ │ └── netstandard2.0/ │ │ └── BenchmarkDotNet.Weaver.targets │ ├── packages/ │ │ └── BenchmarkDotNet.Weaver.0.16.0-develop-3.nupkg │ └── src/ │ ├── ReferencePathAssemblyResolver.cs │ └── WeaveAssemblyTask.cs ├── templates/ │ ├── BenchmarkDotNet.Templates.csproj │ ├── install-from-source.bat │ ├── install-from-source.sh │ └── templates/ │ ├── BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/ │ │ ├── .template.config/ │ │ │ ├── dotnetcli.host.json │ │ │ └── template.json │ │ ├── BenchmarkConfig.cs │ │ ├── Program.cs │ │ ├── _BenchmarkName_.cs │ │ └── _BenchmarkProjectName_.csproj │ ├── BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/ │ │ ├── .template.config/ │ │ │ ├── dotnetcli.host.json │ │ │ └── template.json │ │ ├── BenchmarkConfig.fs │ │ ├── Program.fs │ │ ├── _BenchmarkName_.fs │ │ └── _BenchmarkProjectName_.fsproj │ └── BenchmarkDotNet.BenchmarkProjectTemplate.VB/ │ ├── .template.config/ │ │ ├── dotnetcli.host.json │ │ └── template.json │ ├── BenchmarkConfig.vb │ ├── Program.vb │ ├── _BenchmarkName_.vb │ └── _BenchmarkProjectName_.vbproj └── tests/ ├── BenchmarkDotNet.Analyzers.Tests/ │ ├── AnalyzerTests/ │ │ ├── Attributes/ │ │ │ ├── ArgumentsAttributeAnalyzerTests.cs │ │ │ ├── GeneralArgumentAttributesAnalyzerTests.cs │ │ │ ├── GeneralParameterAttributesAnalyzerTests.cs │ │ │ ├── ParamsAllValuesAttributeAnalyzerTests.cs │ │ │ └── ParamsAttributeAnalyzerTests.cs │ │ ├── BenchmarkRunner/ │ │ │ └── RunAnalyzerTests.cs │ │ └── General/ │ │ └── BenchmarkClassAnalyzerTests.cs │ ├── BenchmarkDotNet.Analyzers.Tests.csproj │ ├── BenchmarkDotNet.Analyzers.Tests.csproj.DotSettings │ └── Fixtures/ │ ├── AnalyzerTestFixture.cs │ ├── Extensions/ │ │ └── TheoryDataExtensions.cs │ ├── Generators/ │ │ └── CombinationsGenerator.cs │ ├── Serializable/ │ │ ├── ValueTupleDouble.cs │ │ └── ValueTupleTriple.cs │ └── TheoryData/ │ ├── FieldOrPropertyDeclarationsTheoryData.cs │ ├── NonPublicClassAccessModifiersTheoryData.cs │ ├── NonPublicClassMemberAccessModifiersTheoryData.cs │ └── NonPublicPropertySetterAccessModifiersTheoryData.cs ├── BenchmarkDotNet.Exporters.Plotting.Tests/ │ ├── BenchmarkDotNet.Exporters.Plotting.Tests.csproj │ └── ScottPlotExporterTests.cs ├── BenchmarkDotNet.IntegrationTests/ │ ├── AllSetupAndCleanupTest.cs │ ├── App.config │ ├── AppConfigTests.cs │ ├── ArgumentsTests.cs │ ├── AssemblyConfigTests.cs │ ├── AsyncBenchmarksTests.cs │ ├── AttributesTests.cs │ ├── BaselineRatioColumnTest.cs │ ├── BenchmarkDotNet.IntegrationTests.csproj │ ├── BenchmarkSwitcherTest.cs │ ├── BenchmarkTestExecutor.cs │ ├── BuildTimeoutTests.cs │ ├── ConflictingNamesTests.cs │ ├── ContinuousIntegration.cs │ ├── CopyToOutputTests.cs │ ├── CustomBuildConfigurationTests.cs │ ├── CustomEngineTests.cs │ ├── Diagnosers/ │ │ ├── MockInProcessDiagnoser.cs │ │ └── ProcessMetricsTests.cs │ ├── DisassemblyDiagnoserTests.cs │ ├── DryRunTests.cs │ ├── EngineTests.cs │ ├── EnvironmentVariablesTests.cs │ ├── EventProcessorTests.cs │ ├── ExceptionDiagnoserTests.cs │ ├── ExceptionHandlingTests.cs │ ├── ExporterIOTests.cs │ ├── ExtraAttributesForEntryMethodTests.cs │ ├── FSharpTests.cs │ ├── FailingProcessSpawnTests.cs │ ├── GcModeTests.cs │ ├── GlobalSetupAttributeInvalidMethodTest.cs │ ├── InProcess.EmitTests/ │ │ ├── NaiveRunnableEmitDiff.cs │ │ ├── RunnableTestCasesHelperTypes.cs │ │ ├── Runnable_0.cs │ │ └── SampleBenchmark.cs │ ├── InProcess.EmitTests.T4/ │ │ ├── RunnableClassCaseBenchmark.cs │ │ ├── RunnableClassCaseBenchmark.tt │ │ ├── RunnableManyArgsCaseBenchmark.cs │ │ ├── RunnableManyArgsCaseBenchmark.tt │ │ ├── RunnableRefStructCaseBenchmark.cs │ │ ├── RunnableRefStructCaseBenchmark.tt │ │ ├── RunnableStructCaseBenchmark.cs │ │ ├── RunnableStructCaseBenchmark.tt │ │ ├── RunnableTaskCaseBenchmark.cs │ │ ├── RunnableTaskCaseBenchmark.tt │ │ ├── RunnableVoidCaseBenchmark.cs │ │ └── RunnableVoidCaseBenchmark.tt │ ├── InProcessDiagnoserTests.cs │ ├── InProcessEmitTest.cs │ ├── InProcessTest.cs │ ├── IntegrationTestSetupTests.cs │ ├── JitOptimizationsTests.cs │ ├── JitRuntimeValidationTest.cs │ ├── LanguageVersionTests.cs │ ├── LargeAddressAwareTest.cs │ ├── MemoryDiagnoserTests.cs │ ├── MonoTests.cs │ ├── MultipleRuntimesTest.cs │ ├── NativeAotTests.cs │ ├── ParamSourceTests.cs │ ├── ParamsTests.cs │ ├── PathTooLongTests.cs │ ├── PowerManagementApplierTests.cs │ ├── PowerRequest.cs │ ├── PowerRequestsParser.cs │ ├── PriorityTests.cs │ ├── ProcessPropertiesTests.cs │ ├── ProcessorArchitectureTest.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ ├── R2RTests.cs │ ├── ReferencesTests.cs │ ├── RoslynToolchainTest.cs │ ├── RunStrategyTests.cs │ ├── RunningEmptyBenchmarkTests.cs │ ├── SetupAndCleanupTests.cs │ ├── StandardErrorTests.cs │ ├── StatResultExtenderTests.cs │ ├── TailCallDiagnoserTests.cs │ ├── TestConfigs.cs │ ├── TestCultureInfo.cs │ ├── ThreadingDiagnoserTests.cs │ ├── ToolchainTest.cs │ ├── ValidatorsTest.cs │ ├── ValuesReturnedByBenchmarkTest.cs │ ├── WakeLockTests.cs │ ├── WasmTests.cs │ ├── Xunit/ │ │ ├── Constants.cs │ │ ├── Extensions.cs │ │ └── MisconfiguredEnvironmentException.cs │ ├── runtimeconfig.template.json │ ├── wwwroot/ │ │ └── custom-main.mjs │ └── xunit.runner.json ├── BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/ │ ├── AssemblyConfigAttribute.cs │ ├── AssemblyConfigBenchmarks.cs │ ├── BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj │ └── Properties/ │ └── AssemblyInfo.cs ├── BenchmarkDotNet.IntegrationTests.CustomPaths/ │ ├── BenchmarkDotNet.IntegrationTests.CustomPaths.csproj │ ├── BenchmarksThatReturnTypeFromCustomPathDll.cs │ ├── BenchmarksThatUseTypeFromCustomPathDll.cs │ ├── BenchmarksThatUsesFileFromOutput.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ └── ShouldGetCopied.xml ├── BenchmarkDotNet.IntegrationTests.DisabledOptimizations/ │ ├── BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj │ ├── OptimizationsDisabledInCsproj.cs │ └── Properties/ │ └── AssemblyInfo.cs ├── BenchmarkDotNet.IntegrationTests.EnabledOptimizations/ │ ├── BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj │ ├── OptimizationsEnabledInCsproj.cs │ └── Properties/ │ └── AssemblyInfo.cs ├── BenchmarkDotNet.IntegrationTests.FSharp/ │ ├── BenchmarkDotNet.IntegrationTests.FSharp.fsproj │ └── Program.fs ├── BenchmarkDotNet.IntegrationTests.ManualRunning/ │ ├── BenchmarkDotNet.IntegrationTests.ManualRunning.csproj │ ├── DotMemoryTests.cs │ ├── DotTraceTests.cs │ ├── MsBuildArgumentTests.cs │ └── xunit.runner.json ├── BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/ │ ├── BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj │ └── MultipleFrameworksTest.cs ├── BenchmarkDotNet.IntegrationTests.Static/ │ ├── BenchmarkClassWithStaticMethod.cs │ └── BenchmarkDotNet.IntegrationTests.Static.csproj ├── BenchmarkDotNet.IntegrationTests.VisualBasic/ │ ├── BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj │ └── Sample.vb ├── BenchmarkDotNet.Tests/ │ ├── Analysers/ │ │ ├── OutliersAnalyserTests.cs │ │ └── ZeroMeasurementHelperTests.cs │ ├── AppConfigGeneratorTests.cs │ ├── ArtifactFileNameHelperTests.cs │ ├── Attributes/ │ │ ├── MutatorAttributesTests.cs │ │ ├── ParamsAllValuesVerifyTests.cs │ │ └── VerifiedFiles/ │ │ ├── ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt │ │ ├── ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt │ │ ├── ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt │ │ ├── ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt │ │ ├── ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt │ │ ├── ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt │ │ └── ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt │ ├── BenchmarkDotNet.Tests.csproj │ ├── BenchmarkDotNetInfoTests.cs │ ├── BuildResultTests.cs │ ├── Builders/ │ │ └── HostEnvironmentInfoBuilder.cs │ ├── CharacteristicPresenterTests.cs │ ├── CodeGeneratorTests.cs │ ├── Columns/ │ │ ├── MetricColumnTests.cs │ │ ├── RatioColumnTest.cs │ │ ├── StatisticalTestColumnTests.cs │ │ └── TagColumnTests.cs │ ├── Common/ │ │ └── AbsoluteEqualityComparer.cs │ ├── ConfigParserTests.cs │ ├── Configs/ │ │ ├── CategoriesTests.cs │ │ ├── ConfigOptionsTests.cs │ │ ├── ConfigPerMethodTests.cs │ │ ├── ConfigUnionTests.cs │ │ ├── EnvironmentVariableTests.cs │ │ ├── ImmutableConfigTests.cs │ │ └── JobTests.cs │ ├── CorrectionsSuggesterTests.cs │ ├── CsProjGeneratorTests.cs │ ├── CsvHelperTests.cs │ ├── Detectors/ │ │ └── Cpu/ │ │ ├── CpuInfoFormatterTests.cs │ │ ├── LinuxCpuInfoParserTests.cs │ │ ├── PowershellWmiCpuInfoParserTests.cs │ │ ├── SysctlCpuInfoParserTests.cs │ │ ├── TestFiles/ │ │ │ ├── ProcCpuInfoProcessorWithDifferentCoresCount.txt │ │ │ ├── ProcCpuInfoRealOneProcessorFourCores.txt │ │ │ ├── ProcCpuInfoRealOneProcessorTwoCores.txt │ │ │ ├── SysctlRealOneProcessorFourCores.txt │ │ │ └── ryzen9-cpuinfo.txt │ │ ├── TestHelper.cs │ │ ├── VerifiedFiles/ │ │ │ └── CpuInfoFormatterTests.FormatTest.verified.txt │ │ └── WmicCpuInfoParserTests.cs │ ├── Disassemblers/ │ │ └── GithubMarkdownDisassemblyExporterMultiCorerunTest.cs │ ├── Engine/ │ │ ├── EngineActualStageTests.cs │ │ ├── EnginePilotStageTests.cs │ │ ├── EngineResultStageTests.cs │ │ ├── EngineWarmupStageTests.cs │ │ └── EnumerateStagesTests.cs │ ├── Environments/ │ │ └── HostEnvironmentInfoTests.cs │ ├── EventPipeProfilerTests.cs │ ├── Exporters/ │ │ ├── CommonExporterVerifyTests.cs │ │ ├── MarkdownExporterVerifyTests.cs │ │ ├── OpenMetricsExporterTests.cs │ │ ├── VerifiedFiles/ │ │ │ ├── CommonExporterVerifyTests.Exporters_Invariant.verified.txt │ │ │ ├── CommonExporterVerifyTests.Exporters_en-US.verified.txt │ │ │ ├── CommonExporterVerifyTests.Exporters_ru-RU.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_HideColumns_TableMarkDown.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt │ │ │ ├── MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt │ │ │ ├── OpenMetricsExporterTests.LabelsAreEscapedCorrectly.verified.txt │ │ │ ├── OpenMetricsExporterTests.ParametrizedBenchmarks_LabelExpansion.verified.txt │ │ │ └── OpenMetricsExporterTests.SingleBenchmark_ProducesHelpAndTypeOnce.verified.txt │ │ └── XmlSerializerTests.cs │ ├── FolderNameTests.cs │ ├── FrameworkVersionHelperTests.cs │ ├── FullNameProviderTests.cs │ ├── GenericBuilderTests.cs │ ├── GlobFilterTests.cs │ ├── Helpers/ │ │ └── LinuxOsReleaseHelperTests.cs │ ├── Infra/ │ │ └── VerifyHelper.cs │ ├── Jobs/ │ │ ├── JobIdGeneratorTests.cs │ │ └── JobIdTests.cs │ ├── KnownIssue.cs │ ├── LevenshteinDistanceCalculatorTests.cs │ ├── Loggers/ │ │ ├── LoggerWithPrefixTests.cs │ │ └── OutputLogger.cs │ ├── Mathematics/ │ │ ├── NumeralSystemTests.cs │ │ ├── RankTests.cs │ │ ├── RatioStatisticsTests.cs │ │ └── StatisticsTests.cs │ ├── Mocks/ │ │ ├── MockClock.cs │ │ ├── MockEngine.cs │ │ ├── MockFactory.cs │ │ ├── MockRunner.cs │ │ └── Toolchain/ │ │ └── MockToolchain.cs │ ├── MonoDisassemblyOutputParserTests.cs │ ├── Order/ │ │ ├── DefaultOrdererTests.cs │ │ └── JobOrderTests.cs │ ├── ParameterComparerTests.cs │ ├── ParameterInstanceTests.cs │ ├── ParamsSourceTests.cs │ ├── Perfonar/ │ │ ├── Infra/ │ │ │ ├── PerfonarMock.cs │ │ │ └── PerfonarTestExtensions.cs │ │ ├── PerfonarTests.cs │ │ └── VerifiedFiles/ │ │ ├── Perfonar.PerfonarIndexTest_key=default01.verified.txt │ │ ├── Perfonar.PerfonarIndexTest_key=default02.verified.txt │ │ ├── Perfonar.PerfonarIndexTest_key=default03.verified.txt │ │ ├── Perfonar.PerfonarIndexTest_key=default04.verified.txt │ │ ├── Perfonar.PerfonarIndexTest_key=default05.verified.txt │ │ ├── Perfonar.PerfonarIndexTest_key=params01.verified.txt │ │ ├── Perfonar.PerfonarIndexTest_key=sort01.verified.txt │ │ ├── Perfonar.PerfonarTableTest_key=default01.verified.txt │ │ ├── Perfonar.PerfonarTableTest_key=default02.verified.txt │ │ ├── Perfonar.PerfonarTableTest_key=default03.verified.txt │ │ ├── Perfonar.PerfonarTableTest_key=default04.verified.txt │ │ ├── Perfonar.PerfonarTableTest_key=default05.verified.txt │ │ ├── Perfonar.PerfonarTableTest_key=params01.verified.txt │ │ └── Perfonar.PerfonarTableTest_key=sort01.verified.txt │ ├── Portability/ │ │ ├── HyperVTests.cs │ │ ├── VMWareTests.cs │ │ └── VirtualBoxTests.cs │ ├── Properties/ │ │ ├── AssemblyInfo.cs │ │ └── BenchmarkDotNetInfoTests.cs │ ├── ReflectionTests.cs │ ├── Reports/ │ │ ├── ColumnTests.cs │ │ ├── DefaultColumnProvidersTests.cs │ │ ├── DisplayPrecisionManagerTests.cs │ │ ├── FakeMetricDescriptor.cs │ │ ├── RatioPrecisionTests.cs │ │ ├── RatioStyleTests.cs │ │ ├── SummaryTableTests.cs │ │ └── SummaryTests.cs │ ├── Running/ │ │ ├── BenchmarkConverterTests.BAC_Partial_DifferentFiles.cs │ │ ├── BenchmarkConverterTests.cs │ │ └── JobRuntimePropertiesComparerTests.cs │ ├── RuntimeVersionDetectionTests.cs │ ├── Serialization/ │ │ ├── DisassemblerModelSerializationTests.cs │ │ └── SummaryJsonSerializationTests.cs │ ├── SourceCodeHelperTests.cs │ ├── StringExtensionsTests.cs │ ├── SummaryStyleTests.cs │ ├── TargetFrameworkVersionParsingTestscs.cs │ ├── TestCultureInfo.cs │ ├── TheoryDataHelper.cs │ ├── TypeFilterTests.cs │ ├── Validators/ │ │ ├── CompilationValidatorTests.cs │ │ ├── DeferredExecutionValidatorTests.cs │ │ ├── ExecutionValidatorTests.cs │ │ ├── ParamsValidatorTests.cs │ │ ├── ReturnValueValidatorTests.cs │ │ ├── RuntimeValidatorTests.cs │ │ └── SetupCleanupValidatorTests.cs │ ├── XUnit/ │ │ ├── EnvRequirement.cs │ │ ├── EnvRequirementChecker.cs │ │ ├── EnvRequirementCheckerTests.cs │ │ ├── FactEnvSpecific.cs │ │ ├── SmartAssert.cs │ │ └── TheoryEnvSpecific.cs │ ├── dotMemory/ │ │ └── DotMemoryTests.cs │ ├── dotTrace/ │ │ └── DotTraceTests.cs │ └── xunit.runner.json ├── CodeCoverage.config ├── Directory.Build.props ├── runClassicTests.cmd ├── runCoreTests.cmd └── runCoreTests.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/CODE_OF_CONDUCT.md ================================================ # Code of Conduct This project has adopted the code of conduct defined by the [Contributor Covenant](http://contributor-covenant.org/) to clarify expected behavior in our community. For more information see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). ================================================ FILE: .github/CONTRIBUTING.md ================================================ # Contributions are welcome! BenchmarkDotNet is already a stable full-featured library which allows performing performance investigation on a professional level. And it continues to evolve! We add new features all the time, but we have too many new cool ideas. Any help will be appreciated. You can develop new features, fix bugs, improve the documentation, or do some other cool stuff. If you want to contribute, check out the [Contributing guide](http://benchmarkdotnet.org/Contributing.htm) and [up-for-grabs](https://github.com/dotnet/BenchmarkDotNet/issues?q=is:open+is:issue+label:up-for-grabs) issues. If you have new ideas or want to complain about bugs, feel free to [create a new issue](https://github.com/dotnet/BenchmarkDotNet/issues/new). Let's build the best tool for benchmarking together! ================================================ FILE: .github/FUNDING.yml ================================================ github: AndreyAkinshin ================================================ FILE: .github/workflows/generate-changelog.yaml ================================================ name: generate-changelog run-name: Generate changelog / ${{ github.event.head_commit.message }} on: push: branches: - master workflow_dispatch: permissions: write-all jobs: generate: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6 with: ref: master - name: Checkout changelog uses: actions/checkout@v6 with: ref: docs-changelog path: docs/_changelog - name: Fetch changelog run: ./build.cmd docs-fetch --depth 1 --preview env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Push changelog uses: JamesIves/github-pages-deploy-action@9d877eea73427180ae43cf98e8914934fe157a1a # v4.7.6 with: branch: docs-changelog folder: docs/_changelog git-config-name: Andrey Akinshin git-config-email: andrey.akinshin@gmail.com clean: true ================================================ FILE: .github/workflows/generate-coverage-report.yaml ================================================ name: generate-coverage-report run-name: Generate coverage coverage report ${{ github.event.head_commit.message }} on: workflow_dispatch: inputs: skip_integration_tests: type: boolean description: Set `true` to skip integration tests. default: true concurrency: group: ${{ github.workflow }}-${{ github.event.inputs.skip_integration_tests }}-${{ github.head_ref || github.ref || github.run_id }} cancel-in-progress: true jobs: collect-coverage: runs-on: ${{ matrix.os }} defaults: run: shell: pwsh strategy: matrix: # Note: ARM64 on linux/macos are not supported. # https://github.com/microsoft/codecoverage/blob/main/docs/supported-os.md os: [windows-latest, ubuntu-latest, macos-15-intel, windows-11-arm] steps: - uses: actions/checkout@v6 # Ensure DOTNET_ROOT environment variable set on macos-15-intel - uses: actions/setup-dotnet@v5 with: dotnet-version: | 8.x - name: Install dotnet-coverage run: dotnet tool install --global dotnet-coverage - name: Run task 'build' run: dotnet build BenchmarkDotNet.slnx -c Release - name: Start dotnet-coverage with background server mode run: dotnet coverage collect --session-id bdn_coverage --settings tests/CodeCoverage.config --server-mode --background - name: Collect Code Coverage run: | dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Tests -c Release --no-build --framework net8.0" dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net8.0" - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests if: ${{ github.event.inputs.skip_integration_tests == 'false'}} run: | dotnet coverage connect bdn_coverage 'dotnet test tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net8.0 --filter "(FullyQualifiedName!~DotMemoryTests) & (FullyQualifiedName!~DotTraceTests) & (FullyQualifiedName!~WasmIsSupported) & (FullyQualifiedName!~WasmSupportsInProcessDiagnosers)"' - name: Shutdown dotnet-coverage server. run: dotnet coverage shutdown bdn_coverage --timeout 60000 - name: Upload coverage artifact uses: actions/upload-artifact@v6 with: name: coverage-${{ matrix.os }} path: "**/*.cobertura.xml" collect-coverage-netfx: runs-on: windows-latest defaults: run: shell: pwsh steps: - uses: actions/checkout@v6 - name: Install dotnet-coverage run: dotnet tool install --global dotnet-coverage - name: Run task 'build' run: dotnet build BenchmarkDotNet.slnx -c Release - name: Start dotnet-coverage with background server mode run: dotnet coverage collect --session-id bdn_coverage --settings tests/CodeCoverage.config --server-mode --background - name: Collect Code Coverage run: | dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Tests -c Release --no-build --framework net462" dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net462" - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests if: ${{ github.event.inputs.skip_integration_tests == 'false'}} run: | dotnet coverage connect bdn_coverage 'dotnet test tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net462 --filter "(FullyQualifiedName!~DotMemoryTests) & (FullyQualifiedName!~DotTraceTests) & (FullyQualifiedName!~WasmIsSupported) & (FullyQualifiedName!~WasmSupportsInProcessDiagnosers)"' - name: Shutdown dotnet-coverage server. run: dotnet coverage shutdown bdn_coverage --timeout 60000 - name: Upload coverage artifact uses: actions/upload-artifact@v6 with: name: coverage-windows-netfx path: "**/*.cobertura.xml" generate-coverage-report: needs: [collect-coverage, collect-coverage-netfx] runs-on: ubuntu-latest defaults: run: shell: pwsh steps: - uses: actions/checkout@v6 - uses: actions/download-artifact@v6 with: path: coverage - name: Upload raw coverage data uses: actions/upload-artifact@v6 with: name: coverage path: coverage - name: Rewrite file path map to absolute path run: | $baseSourcePath = Get-Location $files = [IO.Directory]::GetFiles("coverage", "coverage.cobertura.xml", [IO.SearchOption]::AllDirectories) foreach($file in $files) { $content = [IO.File]::ReadAllText($file).Replace("/_/", [Environment]::CurrentDirectory + '/') [IO.File]::WriteAllText($file, $content) } - name: Install ReportGenerator as global tool run: dotnet tool install --global dotnet-reportgenerator-globaltool - name: Geterate reports run: | reportgenerator ` -reports:"coverage/**/coverage.cobertura.xml" ` -targetdir:"coverage_report/reports" ` -reporttypes:"Html;MarkdownSummaryGithub" - name: Create index.html run: | $html = @" Redirect

If you are not redirected automatically, click here.

"@ [IO.File]::WriteAllText("coverage_report/index.html", $html) - name: Write coverage summary run: | $markdown = Get-Content coverage_report/reports/SummaryGithub.md -Raw Write-Output $markdown >> $env:GITHUB_STEP_SUMMARY - name: Upload HTML report files uses: actions/upload-artifact@v6 with: name: coverage_report path: coverage_report ================================================ FILE: .github/workflows/generate-gh-pages.yaml ================================================ name: generate-gh-pages run-name: Generate gh-pages / ${{ github.event.head_commit.message }} on: push: branches: - docs-stable workflow_dispatch: permissions: write-all jobs: generate: runs-on: windows-latest steps: - name: Checkout uses: actions/checkout@v6 with: ref: docs-stable - name: Checkout changelog uses: actions/checkout@v6 with: ref: docs-changelog path: docs/_changelog - name: Build BenchmarkDotNet run: ./build.cmd build - name: Fetch changelog run: ./build.cmd docs-fetch --depth 2 --preview env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build documentation run: ./build.cmd docs-build - name: Upload Artifacts uses: actions/upload-artifact@v6 with: name: site path: docs/_site deploy: concurrency: ci-${{ github.ref }} needs: [generate] runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6 with: ref: docs-stable - name: Download Artifacts uses: actions/download-artifact@v7 with: name: site path: site - name: Print file tree run: tree $GITHUB_WORKSPACE - name: Deploy documentation uses: JamesIves/github-pages-deploy-action@9d877eea73427180ae43cf98e8914934fe157a1a # v4.7.6 with: branch: gh-pages folder: site git-config-name: GitHub Actions git-config-email: actions@github.com clean: true ================================================ FILE: .github/workflows/publish-nightly.yaml ================================================ name: publish-nightly run-name: Publish nightly nupkg / ${{ github.event.head_commit.message }} on: push: branches: - master workflow_dispatch: jobs: publish: runs-on: ubuntu-latest if: ${{ github.repository == 'dotnet/BenchmarkDotNet' }} steps: - uses: actions/checkout@v6 - name: Set date run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV - name: Pack run: ./build.cmd pack /p:VersionSuffix=nightly.$DATE.$GITHUB_RUN_NUMBER - name: Upload nupkg to artifacts uses: actions/upload-artifact@v6 with: name: nupkgs path: "**/*.*nupkg" - name: Publish nupkg env: MYGET_API_KEY: ${{ secrets.MYGET_API_KEY }} run: ./.dotnet/dotnet nuget push **/*.nupkg --source https://www.myget.org/F/benchmarkdotnet/api/v3/index.json --api-key $MYGET_API_KEY --timeout 600 ================================================ FILE: .github/workflows/release.yaml ================================================ name: release run-name: Release new version on: workflow_dispatch: env: DOTNET_VERSION: "8.0.410" jobs: release: runs-on: ubuntu-latest environment: deploy # Only maintainers can execute this workflow permissions: contents: write discussions: write issues: write steps: # --- Init --- - name: Checkout sources uses: actions/checkout@v6 - name: Checkout changelog uses: actions/checkout@v6 with: ref: docs-changelog path: docs/_changelog - name: Setup .NET Core uses: actions/setup-dotnet@v5 with: dotnet-version: ${{ env.DOTNET_VERSION }} - name: Read current version id: version run: echo "VERSION=$(grep -v '^$' build/versions.txt | tail -n 1)" >> $GITHUB_OUTPUT # --- Main --- - name: Tag release run: git tag v${{ steps.version.outputs.VERSION }} - name: Build run: ./build.cmd build --stable - name: Pack run: ./build.cmd pack --stable - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: nupkgs path: artifacts/*.nupkg # --- Analyzer Rules and Next Version --- - name: Move analyzer rules run: ./build.cmd move-analyzer-rules - name: Generate changelog run: ./build.cmd docs-generate --stable env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Version increment run: ./build.cmd version-increment - name: Commit changes uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 # v9.1.4 with: message: "Set next BenchmarkDotNet version: ${{ steps.version.outputs.VERSION }} and update released analyzer rules" author_name: GitHub Actions author_email: actions@github.com committer_name: GitHub Actions committer_email: actions@github.com - name: Read next version id: next_version run: echo "NEXT_VERSION=$(grep -v '^$' build/versions.txt | tail -n 1)" >> $GITHUB_OUTPUT # --- Release --- - name: Push git changes uses: ad-m/github-push-action@77c5b412c50b723d2a4fbc6d71fb5723bcd439aa # v1.0.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: master tags: true - name: Release uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 with: name: BenchmarkDotNet v${{ steps.version.outputs.VERSION }} tag_name: v${{ steps.version.outputs.VERSION }} body: | Full changelog: https://benchmarkdotnet.org/changelog/v${{ steps.version.outputs.VERSION }}.html discussion_category_name: Announcements - name: Close old milestone uses: Akkjon/close-milestone@v2.2.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: milestone_name: v${{ steps.version.outputs.VERSION }} - name: Create new milestone uses: WyriHaximus/github-action-create-milestone@bcd9e15439836d6098d353a5bcf82de46591a35d # v1.1.2 env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" with: title: v${{ steps.next_version.outputs.NEXT_VERSION }} - name: Publish to NuGet run: dotnet nuget push ./artifacts/*.nupkg --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json --skip-duplicate # --- Documentation --- - name: Fetch changelog run: ./build.cmd docs-fetch --depth 2 --preview env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Generate changelog run: ./build.cmd docs-generate --preview env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Push changelog uses: JamesIves/github-pages-deploy-action@v4 with: branch: docs-changelog folder: docs/_changelog git-config-name: GitHub Actions git-config-email: actions@github.com clean: true - name: Build documentation run: ./build.cmd docs-build - name: Deploy documentation uses: JamesIves/github-pages-deploy-action@9d877eea73427180ae43cf98e8914934fe157a1a # v4.7.6 with: branch: gh-pages folder: docs/_site git-config-name: GitHub Actions git-config-email: actions@github.com clean: true ================================================ FILE: .github/workflows/run-tests-selected.yaml ================================================ name: run-tests-selected run-name: Run selected tests (${{ inputs.runs_on }} --framework ${{ inputs.framework}} --filter ${{ inputs.filter }}) on: workflow_dispatch: inputs: runs_on: type: choice description: GitHub Actions runner image name required: true default: ubuntu-latest options: - windows-latest - ubuntu-latest - macos-latest - windows-11-arm - ubuntu-24.04-arm - macos-15-intel project: type: string description: Specify test project path required: true default: tests/BenchmarkDotNet.IntegrationTests options: - tests/BenchmarkDotNet.Tests - tests/BenchmarkDotNet.IntegrationTests - tests/BenchmarkDotNet.IntegrationTests.ManualRunning framework: type: choice description: Specify target framework required: true options: - net8.0 - net462 filter: type: string description: Test filter text (It's used for `dotnet test --filter`) Use default value when running all tests required: true default: "BenchmarkDotNet" iteration_count: type: number description: Count of test loop (It's expected to be used for flaky tests) required: true default: 1 jobs: test: name: test (${{ inputs.runs_on }} --framework ${{ inputs.framework}} --filter ${{ inputs.filter }}) runs-on: ${{ inputs.runs_on }} timeout-minutes: 60 # Explicitly set timeout. When wrong input parameter is passed. It may continue to run until it times out (Default:360 minutes)) steps: - uses: actions/checkout@v4 # Setup - name: Setup run: | mkdir artifacts - name: Install workloads run: | dotnet workload install wasm-tools dotnet workload install wasm-tools-net8 - name: Set up node uses: actions/setup-node@v6 with: node-version: "24" - name: Set up v8 shell: pwsh run: | npm install jsvu -g jsvu --os=default --engines=v8 $homeDir = $env:HOME if (-not $homeDir) { $homeDir = $env:USERPROFILE } Add-Content -Path $env:GITHUB_PATH -Value (Join-Path $homeDir ".jsvu/bin") # Build - name: Run build working-directory: ${{ github.event.inputs.project }} run: | dotnet build -c Release --framework ${{ inputs.framework }} -tl:off # Test - name: Run tests shell: pwsh working-directory: ${{ github.event.inputs.project }} run: | $PSNativeCommandUseErrorActionPreference = $true $iterationCount = ${{ inputs.iteration_count }} foreach($i in 1..$iterationCount) { Write-Output ('##[group]Executing Iteration: {0}/${{ inputs.iteration_count }}' -f $i) dotnet test -c Release --framework ${{ inputs.framework }} --filter ${{ inputs.filter }} -tl:off --no-build --logger "console;verbosity=normal" Write-Output '##[endgroup]' } # Upload artifact files that are located at `$(GITHUB_WORKSPACE)/artifacts` directory - name: Upload test results uses: actions/upload-artifact@v6 if: always() with: name: results if-no-files-found: ignore path: | artifacts/**/* ================================================ FILE: .github/workflows/run-tests.yaml ================================================ name: run-tests run-name: Run tests / ${{ github.event.head_commit.message }} on: pull_request: push: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} cancel-in-progress: true jobs: test-windows-core: strategy: matrix: os: [windows-latest, windows-11-arm] runs-on: ${{ matrix.os }} steps: - name: Add Windows Defender Exclusions shell: powershell run: | Add-MpPreference -ExclusionPath $env:GITHUB_WORKSPACE Add-MpPreference -ExclusionPath $env:TEMP - uses: actions/checkout@v6 # Setup wasm - name: Set up node uses: actions/setup-node@v6 with: node-version: "24" - name: Set up v8 run: | npm install jsvu -g jsvu --os=win64 --engines=v8 Add-Content -Path $env:GITHUB_PATH -Value "$env:USERPROFILE\.jsvu\bin" - name: Install wasm-tools workload run: ./build.cmd install-wasm-tools # Build and Test - name: Run task 'build' shell: cmd run: ./build.cmd build - name: Run task 'unit-tests' shell: cmd run: ./build.cmd unit-tests -e - name: Run task 'analyzer-tests' shell: cmd run: ./build.cmd analyzer-tests -e - name: Run task 'in-tests-core' shell: cmd run: ./build.cmd in-tests-core -e # Report test results with unique name - name: Report tests results uses: dorny/test-reporter@fe45e9537387dac839af0d33ba56eed8e24189e8 # v2.3.0 if: always() with: name: test-windows-core-${{ matrix.os }} path: "**/*.trx" reporter: dotnet-trx # Upload Artifacts with Unique Name - name: Upload test results uses: actions/upload-artifact@v6 if: always() with: name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-windows-full: strategy: matrix: os: [windows-latest, windows-11-arm] runs-on: ${{ matrix.os }} steps: - name: Add Windows Defender Exclusions shell: powershell run: | Add-MpPreference -ExclusionPath $env:GITHUB_WORKSPACE Add-MpPreference -ExclusionPath $env:TEMP - uses: actions/checkout@v6 # Setup wasm - name: Set up node uses: actions/setup-node@v6 with: node-version: "24" - name: Set up v8 run: | npm install jsvu -g jsvu --os=win64 --engines=v8 Add-Content -Path $env:GITHUB_PATH -Value "$env:USERPROFILE\.jsvu\bin" - name: Install wasm-tools workload run: ./build.cmd install-wasm-tools # Build and Test - name: Run task 'build' shell: cmd run: ./build.cmd build - name: Run task 'unit-tests' shell: cmd run: ./build.cmd unit-tests -e - name: Run task 'analyzer-tests' shell: cmd run: ./build.cmd analyzer-tests -e - name: Run task 'in-tests-full' shell: cmd run: ./build.cmd in-tests-full -e # Report test results with unique name - name: Report tests results uses: dorny/test-reporter@fe45e9537387dac839af0d33ba56eed8e24189e8 # v2.3.0 if: always() with: name: test-windows-full-${{ matrix.os }} path: "**/*.trx" reporter: dotnet-trx # Upload Artifacts with Unique Name - name: Upload test results uses: actions/upload-artifact@v6 if: always() with: name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-linux: strategy: matrix: os: [ubuntu-latest, ubuntu-24.04-arm] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 # Set up the environment - name: Set up Clang uses: egor-tensin/setup-clang@ef434b41eb33a70396fb336b1bae39c76d740c3d # v1.4 with: version: latest platform: x64 - name: Set up zlib-static run: sudo apt-get install -y libkrb5-dev - name: Set up node uses: actions/setup-node@v6 with: node-version: "24" - name: Set up v8 run: npm install jsvu -g && jsvu --os=linux64 --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH - name: Install wasm-tools workload run: ./build.cmd install-wasm-tools # Build and Test - name: Run task 'build' run: ./build.cmd build - name: Run task 'analyzer-tests' run: ./build.cmd analyzer-tests -e - name: Run task 'unit-tests' run: ./build.cmd unit-tests -e - name: Run task 'in-tests-core' run: ./build.cmd in-tests-core -e # Report test results with unique name - name: Report tests results uses: dorny/test-reporter@fe45e9537387dac839af0d33ba56eed8e24189e8 # v2.3.0 if: always() with: name: test-linux-${{ matrix.os }} path: "**/*.trx" reporter: dotnet-trx # Upload Artifacts with Unique Name - name: Upload test results uses: actions/upload-artifact@v6 if: always() with: name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-macos: name: test-macos (${{ matrix.os.arch }}) runs-on: ${{ matrix.os.runs-on }} strategy: matrix: os: - runs-on: 'macos-latest' jsvu-os: 'mac64arm' arch: 'arm64' - runs-on: 'macos-15-intel' jsvu-os: 'mac64' arch: 'x64' steps: - uses: actions/checkout@v6 - name: Set up node uses: actions/setup-node@v6 with: node-version: "24" - name: Set up v8 run: npm install jsvu -g && jsvu --os=${{ matrix.os.jsvu-os }} --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH - name: Install wasm-tools workload run: ./build.cmd install-wasm-tools # Build and Test - name: Run task 'build' run: ./build.cmd build - name: Run task 'analyzer-tests' run: ./build.cmd analyzer-tests -e - name: Run task 'unit-tests' run: ./build.cmd unit-tests -e - name: Run task 'in-tests-core' run: ./build.cmd in-tests-core -e # Report test results with unique name - name: Report tests results uses: dorny/test-reporter@fe45e9537387dac839af0d33ba56eed8e24189e8 # v2.3.0 if: always() with: name: test-macos(${{ matrix.os.arch }}) path: "**/*.trx" reporter: dotnet-trx # Upload Artifacts with Unique Name - name: Upload test results uses: actions/upload-artifact@v6 if: always() with: name: test-macos(${{ matrix.os.arch }})-trx-${{ github.run_id }} path: "**/*.trx" test-pack: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Set up Clang uses: egor-tensin/setup-clang@v1 with: version: latest platform: x64 - name: Set up zlib-static run: sudo apt-get install -y libkrb5-dev - name: Run task 'pack' run: ./build.cmd pack spellcheck-docs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: actions/setup-node@v6 name: Setup node with: node-version: "24" - name: Install cSpell run: npm install -g cspell@9.0.2 - name: Copy cSpell config run: cp ./build/cSpell.json ./cSpell.json - name: Run cSpell run: cspell --config ./cSpell.json "docs/**/*.md" --no-progress ================================================ FILE: .gitignore ================================================ # use glob syntax syntax: glob *.obj *.pdb *.user *.aps *.pch *.vspscc *.vssscc *_i.c *_p.c *.ncb *.suo *.tlb *.tlh *.bak *.cache *.ilk *.log *.lib *.sbr *.scc [Bb]in [Dd]ebug*/ obj/ [Rr]elease*/ _ReSharper*/ [Tt]humbs.db [Tt]est[Rr]esult* [Bb]uild[Ll]og.* *.[Pp]ublish.xml *.resharper *.received.txt *.orig packages/ nuget.exe docs/guidehtml docs/apihtml .idea/ *.iml **/BenchmarkDotNet.Artifacts/* **/project.lock.json tests/output/* .vs/restore.dg artifacts/* BDN.Generated BenchmarkDotNet.Samples/Properties/launchSettings.json src/BenchmarkDotNet/Disassemblers/net462/* src/BenchmarkDotNet/Disassemblers/BenchmarkDotNet.Disassembler.*.nupkg # Visual Studio 2015 cache/options directory .vs/ # VSCode directory .vscode/ # Cake tools/** .dotnet # Xamarin Resource.designer.cs # Tests TestResults ## Mac OS # General .DS_Store .AppleDouble .LSOverride # Thumbnails ._* ================================================ FILE: BenchmarkDotNet.slnx ================================================ ================================================ FILE: BenchmarkDotNet.slnx.DotSettings ================================================  True ExplicitlyExcluded ExplicitlyExcluded ExplicitlyExcluded DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW True True True DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW True NotRequired ExpressionBody ExpressionBody ExpressionBody 0 TOGETHER_SAME_LINE True NEVER False NEVER False NEVER True True 160 UseExplicitType ASCII GC <?xml version="1.0" encoding="utf-16"?><Profile name="BenchmarkDotNet"><CSCodeStyleAttributes ArrangeTypeAccessModifier="True" ArrangeTypeMemberAccessModifier="True" SortModifiers="True" RemoveRedundantParentheses="True" AddMissingParentheses="True" ArrangeBraces="True" ArrangeAttributes="True" ArrangeArgumentsStyle="True" ArrangeCodeBodyStyle="True" ArrangeVarStyle="True" /><CSFixBuiltinTypeReferences>True</CSFixBuiltinTypeReferences><CSOptimizeUsings><OptimizeUsings>True</OptimizeUsings><EmbraceInRegion>False</EmbraceInRegion><RegionName></RegionName></CSOptimizeUsings><CSReformatCode>True</CSReformatCode></Profile> IL IO OSX RT <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> True True 983040 True True True True True True True True True True True <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> True <data><IncludeFilters /><ExcludeFilters><Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="JetBrains.Annotations.*" FunctionMask="*" IsEnabled="True" /><Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="SimpleJson.*" FunctionMask="*" IsEnabled="True" /></ExcludeFilters></data> True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True ================================================ FILE: LICENSE.md ================================================ ### The MIT License Copyright (c) 2013–2025 .NET Foundation and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: NuGet.Config ================================================  ================================================ FILE: README.md ================================================
![](https://raw.githubusercontent.com/dotnet/BenchmarkDotNet/ec962b0bd6854c991d7a3ebd77037579165acb36/docs/logo/logo-wide.png)
[![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet/) [![MyGet](https://img.shields.io/myget/benchmarkdotnet/vpre/benchmarkdotnet?label=myget)](https://www.myget.org/feed/benchmarkdotnet/package/nuget/BenchmarkDotNet) [![Downloads](https://img.shields.io/nuget/dt/benchmarkdotnet.svg)](https://www.nuget.org/packages/BenchmarkDotNet/) [![Stars](https://img.shields.io/github/stars/dotnet/BenchmarkDotNet?color=brightgreen)](https://github.com/dotnet/BenchmarkDotNet/stargazers) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/dotnet/BenchmarkDotNet/blob/master/LICENSE.md) [![Twitter](https://img.shields.io/twitter/follow/BenchmarkDotNet?style=social&label=Twitter)](https://twitter.com/BenchmarkDotNet)
Features · Getting started · Documentation · Learn more about benchmarking
**BenchmarkDotNet** helps you to transform methods into benchmarks, track their performance, and share reproducible measurement experiments. It's no harder than writing unit tests! Under the hood, it performs a lot of [magic](#automation) that guarantees [reliable and precise](#reliability) results thanks to the [perfolizer](https://github.com/AndreyAkinshin/perfolizer) and [pragmastat](https://github.com/AndreyAkinshin/pragmastat) statistical engine. BenchmarkDotNet protects you from popular benchmarking mistakes and warns you if something is wrong with your benchmark design or obtained measurements. The results are presented in a [user-friendly](#friendliness) form that highlights all the important facts about your experiment. BenchmarkDotNet is already adopted by [27400+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including [.NET Runtime](https://github.com/dotnet/runtime), [.NET Compiler](https://github.com/dotnet/roslyn), [.NET Performance](https://github.com/dotnet/performance), and many others. It's [easy](#simplicity) to start writing benchmarks, check out the following example (copy-pastable version is [here](https://benchmarkdotnet.org/articles/guides/getting-started.html)): ```cs [SimpleJob(RuntimeMoniker.Net472, baseline: true)] [SimpleJob(RuntimeMoniker.NetCoreApp30)] [SimpleJob(RuntimeMoniker.NativeAot70)] [SimpleJob(RuntimeMoniker.Mono)] [RPlotExporter] public class Md5VsSha256 { private SHA256 sha256 = SHA256.Create(); private MD5 md5 = MD5.Create(); private byte[] data; [Params(1000, 10000)] public int N; [GlobalSetup] public void Setup() { data = new byte[N]; new Random(42).NextBytes(data); } [Benchmark] public byte[] Sha256() => sha256.ComputeHash(data); [Benchmark] public byte[] Md5() => md5.ComputeHash(data); } ``` BenchmarkDotNet automatically runs the benchmarks on all the runtimes, aggregates the measurements, and prints a summary table with the most important information: ```md BenchmarkDotNet=v0.12.0, OS=Windows 10.0.17763.805 (1809/October2018Update/Redstone5) Intel Core i7-7700K CPU 4.20GHz (Kaby Lake), 1 CPU, 8 logical and 4 physical cores [Host] : .NET Framework 4.7.2 (4.7.3468.0), X64 RyuJIT Net472 : .NET Framework 4.7.2 (4.7.3468.0), X64 RyuJIT NetCoreApp30 : .NET Core 3.0.0 (CoreCLR 4.700.19.46205, CoreFX 4.700.19.46214), X64 RyuJIT NativeAot70 : .NET 7.0.0-preview.4.22172.7, X64 NativeAOT Mono : Mono 6.4.0 (Visual Studio), X64 | Method | Runtime | N | Mean | Error | StdDev | Ratio | |------- |-------------- |------ |-----------:|----------:|----------:|------:| | Sha256 | .NET 4.7.2 | 1000 | 7.735 us | 0.1913 us | 0.4034 us | 1.00 | | Sha256 | .NET Core 3.0 | 1000 | 3.989 us | 0.0796 us | 0.0745 us | 0.50 | | Sha256 | NativeAOT 7.0 | 1000 | 4.091 us | 0.0811 us | 0.1562 us | 0.53 | | Sha256 | Mono | 1000 | 13.117 us | 0.2485 us | 0.5019 us | 1.70 | | | | | | | | | | Md5 | .NET 4.7.2 | 1000 | 2.872 us | 0.0552 us | 0.0737 us | 1.00 | | Md5 | .NET Core 3.0 | 1000 | 1.848 us | 0.0348 us | 0.0326 us | 0.64 | | Md5 | NativeAOT 7.0 | 1000 | 1.817 us | 0.0359 us | 0.0427 us | 0.63 | | Md5 | Mono | 1000 | 3.574 us | 0.0678 us | 0.0753 us | 1.24 | | | | | | | | | | Sha256 | .NET 4.7.2 | 10000 | 74.509 us | 1.5787 us | 4.6052 us | 1.00 | | Sha256 | .NET Core 3.0 | 10000 | 36.049 us | 0.7151 us | 1.0025 us | 0.49 | | Sha256 | NativeAOT 7.0 | 10000 | 36.253 us | 0.7076 us | 0.7571 us | 0.49 | | Sha256 | Mono | 10000 | 116.350 us | 2.2555 us | 3.0110 us | 1.58 | | | | | | | | | | Md5 | .NET 4.7.2 | 10000 | 17.308 us | 0.3361 us | 0.4250 us | 1.00 | | Md5 | .NET Core 3.0 | 10000 | 15.726 us | 0.2064 us | 0.1930 us | 0.90 | | Md5 | NativeAOT 7.0 | 10000 | 15.627 us | 0.2631 us | 0.2461 us | 0.89 | | Md5 | Mono | 10000 | 30.205 us | 0.5868 us | 0.6522 us | 1.74 | ``` The measured data can be exported to different formats (md, html, csv, xml, json, etc.) including plots: ![](https://raw.githubusercontent.com/dotnet/BenchmarkDotNet/ec962b0bd6854c991d7a3ebd77037579165acb36/docs/images/v0.12.0/rplot.png) *Supported runtimes:* .NET 5+, .NET Framework 4.6.1+, .NET Core 2.0+, Mono, NativeAOT *Supported languages:* C#, F#, Visual Basic *Supported OS:* Windows, Linux, macOS *Supported architectures:* x86, x64, ARM, ARM64, Wasm and LoongArch64 ## Features BenchmarkDotNet has tons of features that are essential in comprehensive performance investigations. Four aspects define the design of these features: *simplicity*, *automation*, *reliability*, and *friendliness*. ### Simplicity You shouldn't have to be an experienced performance engineer if you want to write benchmarks. You can design very complicated performance experiments in the declarative style using simple APIs. For example, if you want to [parameterize](https://benchmarkdotnet.org/articles/features/parameterization.html) your benchmark, mark a field or a property with `[Params(1, 2, 3)]`: BenchmarkDotNet will enumerate all of the specified values and run benchmarks for each case. If you want to compare benchmarks with each other, mark one of the benchmarks as the [baseline](https://benchmarkdotnet.org/articles/features/baselines.html) via `[Benchmark(Baseline = true)]`: BenchmarkDotNet will compare it with all of the other benchmarks. If you want to compare performance in different environments, use [jobs](https://benchmarkdotnet.org/articles/configs/jobs.html). For example, you can run all the benchmarks on .NET 8.0 and Mono via `[SimpleJob(RuntimeMoniker.Net80)]` and `[SimpleJob(RuntimeMoniker.Mono)]`. If you don't like attributes, you can call most of the APIs via the fluent style and write code like this: ```cs ManualConfig.CreateEmpty() // A configuration for our benchmarks .AddJob(Job.Default // Adding first job .WithRuntime(ClrRuntime.Net472) // .NET Framework 4.7.2 .WithPlatform(Platform.X64) // Run as x64 application .WithJit(Jit.LegacyJit) // Use LegacyJIT instead of the default RyuJIT .WithGcServer(true) // Use Server GC ).AddJob(Job.Default // Adding second job .AsBaseline() // It will be marked as baseline .WithEnvironmentVariable("Key", "Value") // Setting an environment variable .WithWarmupCount(0) // Disable warm-up stage ); ``` If you prefer command-line experience, you can configure your benchmarks via the [console arguments](https://benchmarkdotnet.org/articles/guides/console-args.html) in any console application (other types of applications are not supported). ### Automation Reliable benchmarks always include a lot of boilerplate code. Let's think about what you should do in a typical case. First, you should perform a pilot experiment and determine the best number of method invocations. Next, you should execute several warm-up iterations and ensure that your benchmark achieved a steady state. After that, you should execute the main iterations and calculate some basic statistics. If you calculate some values in your benchmark, you should use it somehow to prevent dead code elimination. If you use loops, you should care about the effect of the loop unrolling on your results (which may depend on the processor architecture). Once you get results, you should check for some special properties of the obtained performance distribution like multimodality or extremely high outliers. You should also evaluate the overhead of your infrastructure and deduct it from your results. If you want to test several environments, you should perform the measurements in each of them and manually aggregate the results. If you write this code from scratch, it's easy to make a mistake and spoil your measurements. Note that it's a shortened version of the full checklist that you should follow during benchmarking: there are a lot of additional hidden pitfalls that should be handled appropriately. Fortunately, you shouldn't worry about it because BenchmarkDotNet [will perform](https://benchmarkdotnet.org/articles/guides/how-it-works.html) this boring and time-consuming stuff for you. Moreover, the library can help you with some advanced tasks that you may want to perform during the investigation. For example, BenchmarkDotNet can measure the [managed](https://benchmarkdotnet.org/articles/configs/diagnosers.html#usage) and [native](https://benchmarkdotnet.org/articles/samples/IntroNativeMemory.html) memory traffic and print [disassembly listings](https://benchmarkdotnet.org/articles/configs/diagnosers.html#sample-introdisassembly) for your benchmarks. ### Reliability A lot of hand-written benchmarks produce wrong numbers that lead to incorrect business decisions. BenchmarkDotNet protects you from most of the benchmarking pitfalls and allows achieving high measurement precision. You shouldn't worry about the perfect number of method invocation, the number of warm-up and actual iterations: BenchmarkDotNet tries to choose the best benchmarking parameters and achieve a good trade-off between the measurement prevision and the total duration of all benchmark runs. So, you shouldn't use any magic numbers (like "We should perform 100 iterations here"), the library will do it for you based on the values of statistical metrics. BenchmarkDotNet also prevents benchmarking of non-optimized assemblies that were built using DEBUG mode because the corresponding results will be unreliable. The library will print a warning if you have an attached debugger, if you use a hypervisor (HyperV, VMware, VirtualBox), or if you have any other problems with the current environment. During 6+ years of development, we faced dozens of different problems that may spoil your measurements. Inside BenchmarkDotNet, there are a lot of heuristics, checks, hacks, and tricks that help you to increase the reliability of the results. ### Friendliness Analysis of performance data is a time-consuming activity that requires attentiveness, knowledge, and experience. BenchmarkDotNet performs the main part of this analysis for you and presents results in a user-friendly form. After the experiments, you get a summary table that contains a lot of useful data about the executed benchmarks. By default, it includes only the most important columns, but they can be [easily customized](https://benchmarkdotnet.org/articles/configs/columns.html). The column set is adaptive and depends on the benchmark definition and measured values. For example, if you mark one of the benchmarks as a [baseline](https://benchmarkdotnet.org/articles/features/baselines.html), you will get additional columns that will help you to compare all the benchmarks with the baseline. By default, it always shows the Mean column, but if we detected a vast difference between the Mean and the Median values, both columns will be presented. BenchmarkDotNet tries to find some unusual properties of your performance distributions and prints nice messages about it. For example, it will warn you in case of multimodal distribution or high outliers. In this case, you can scroll the results up and check out ASCII-style histograms for each distribution or generate beautiful png plots using `[RPlotExporter]`. BenchmarkDotNet doesn't overload you with data; it shows only the essential information depending on your results: it allows you to keep the summary small for primitive cases and extend it only for complicated cases. Of course, you can request any additional statistics and visualizations manually. If you don't customize the summary view, the default presentation will be as much user-friendly as possible. :) ## Learn more about benchmarking BenchmarkDotNet is not a silver bullet that magically makes all of your benchmarks correct and analyzes the measurements for you. Even if you use this library, you still should know how to design benchmark experiments and how to make correct conclusions based on the raw data. If you want to know more about benchmarking methodology and good practices, it's recommended to read a book by Andrey Akinshin (the BenchmarkDotNet project lead): ["Pro .NET Benchmarking"](https://aakinshin.net/prodotnetbenchmarking/). Use this in-depth guide to correctly design benchmarks, measure key performance metrics of .NET applications, and analyze results. This book presents dozens of case studies to help you understand complicated benchmarking topics. You will avoid common pitfalls, control the accuracy of your measurements, and improve the performance of your software.
## Contributions are welcome! BenchmarkDotNet is already a stable full-featured library that allows performing performance investigation on a professional level. And it continues to evolve! We add new features all the time, but we have too many new cool ideas. Any help will be appreciated. You can develop new features, fix bugs, improve the documentation, or do some other cool stuff. If you want to contribute, check out the [Contributing guide](https://benchmarkdotnet.org/articles/contributing/building.html) and [up-for-grabs](https://github.com/dotnet/BenchmarkDotNet/issues?q=is:open+is:issue+label:up-for-grabs) issues. If you have new ideas or want to complain about bugs, feel free to [create a new issue](https://github.com/dotnet/BenchmarkDotNet/issues/new). Let's build the best tool for benchmarking together! ## Code of Conduct This project has adopted the code of conduct defined by the [Contributor Covenant](https://www.contributor-covenant.org/) to clarify expected behavior in our community. For more information, see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). ================================================ FILE: build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj ================================================ Exe net10.0 $(MSBuildProjectDirectory) enable ================================================ FILE: build/BenchmarkDotNet.Build/BuildContext.cs ================================================ using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using BenchmarkDotNet.Build.Helpers; using BenchmarkDotNet.Build.Meta; using BenchmarkDotNet.Build.Options; using BenchmarkDotNet.Build.Runners; using Cake.Common; using Cake.Common.Build; using Cake.Common.Diagnostics; using Cake.Common.IO; using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.MSBuild; using Cake.Core; using Cake.Core.IO; using Cake.FileHelpers; using Cake.Frosting; namespace BenchmarkDotNet.Build; public class BuildContext : FrostingContext { public string BuildConfiguration { get; set; } = "Release"; public DotNetVerbosity BuildVerbosity { get; set; } = DotNetVerbosity.Minimal; public DirectoryPath RootDirectory { get; } public DirectoryPath BuildDirectory { get; } public DirectoryPath ArtifactsDirectory { get; } public FilePath SolutionFile { get; } public FilePath AnalyzersProjectFile { get; } public FilePath TemplatesTestsProjectFile { get; } public FilePathCollection AllPackableSrcProjects { get; } public FilePath VersionsFile { get; } public FilePath CommonPropsFile { get; } public FilePath ReadmeFile { get; } public DotNetMSBuildSettings MsBuildSettingsRestore { get; } public DotNetMSBuildSettings MsBuildSettingsBuild { get; } public DotNetMSBuildSettings MsBuildSettingsPack { get; } private bool IsCiBuild => !this.BuildSystem().IsLocalBuild; public IReadOnlyCollection NuGetPackageNames { get; } public VersionHistory VersionHistory { get; } public GitRunner GitRunner { get; } public UnitTestRunner UnitTestRunner { get; } public DocumentationRunner DocumentationRunner { get; } public BuildRunner BuildRunner { get; } public ReleaseRunner ReleaseRunner { get; } public BuildContext(ICakeContext context) : base(context) { RootDirectory = new DirectoryPath(new DirectoryInfo(Directory.GetCurrentDirectory()).Parent?.Parent?.FullName); BuildDirectory = RootDirectory.Combine("build"); ArtifactsDirectory = RootDirectory.Combine("artifacts"); var toolFileName = context.IsRunningOnWindows() ? "dotnet.exe" : "dotnet"; var toolFilePath = RootDirectory.Combine(".dotnet").CombineWithFilePath(toolFileName); context.Tools.RegisterFile(toolFilePath); SolutionFile = RootDirectory.CombineWithFilePath("BenchmarkDotNet.slnx"); AnalyzersProjectFile = RootDirectory.Combine("src").Combine("BenchmarkDotNet.Analyzers").CombineWithFilePath("BenchmarkDotNet.Analyzers.csproj"); TemplatesTestsProjectFile = RootDirectory.Combine("templates") .CombineWithFilePath("BenchmarkDotNet.Templates.csproj"); AllPackableSrcProjects = new FilePathCollection(context.GetFiles(RootDirectory.FullPath + "/src/**/*.csproj") .Where(p => !p.FullPath.Contains("Disassembler"))); VersionsFile = BuildDirectory.CombineWithFilePath("versions.txt"); CommonPropsFile = BuildDirectory.CombineWithFilePath("common.props"); ReadmeFile = RootDirectory.CombineWithFilePath("README.md"); MsBuildSettingsRestore = new DotNetMSBuildSettings(); MsBuildSettingsBuild = new DotNetMSBuildSettings(); MsBuildSettingsPack = new DotNetMSBuildSettings(); if (IsCiBuild) { System.Environment.SetEnvironmentVariable("BDN_CI_BUILD", "true"); MsBuildSettingsBuild.MaxCpuCount = 1; MsBuildSettingsBuild.WithProperty("UseSharedCompilation", "false"); } if (context.Arguments.HasArgument("msbuild")) { var msBuildParameters = context.Arguments.GetArguments().First(it => it.Key == "msbuild").Value; foreach (var msBuildParameter in msBuildParameters) { var split = msBuildParameter.Split(new[] { '=' }, 2); if (split.Length == 2) { var name = split[0]; var value = split[1]; MsBuildSettingsRestore.WithProperty(name, value); MsBuildSettingsBuild.WithProperty(name, value); MsBuildSettingsPack.WithProperty(name, value); if (name.Equals("configuration", StringComparison.OrdinalIgnoreCase)) BuildConfiguration = value; if (name.Equals("verbosity", StringComparison.OrdinalIgnoreCase)) { var parsedVerbosity = Utils.ParseVerbosity(value); if (parsedVerbosity != null) BuildVerbosity = parsedVerbosity.Value; } } } } if (KnownOptions.Stable.Resolve(this)) { const string name = "NoVersionSuffix"; const string value = "true"; MsBuildSettingsRestore.WithProperty(name, value); MsBuildSettingsBuild.WithProperty(name, value); MsBuildSettingsPack.WithProperty(name, value); } // NativeAOT build requires VS C++ tools to be added to $path via vcvars64.bat // but once we do that, dotnet restore fails with: // "Please specify a valid solution configuration using the Configuration and Platform properties" if (context.IsRunningOnWindows()) { MsBuildSettingsRestore.WithProperty("Platform", "Any CPU"); MsBuildSettingsBuild.WithProperty("Platform", "Any CPU"); } var nuGetPackageNames = new List(); nuGetPackageNames.AddRange(this .GetSubDirectories(RootDirectory.Combine("src")) .Select(directoryPath => directoryPath.GetDirectoryName()) .Where(name => !name.Contains("Disassembler", StringComparison.OrdinalIgnoreCase))); nuGetPackageNames.Add("BenchmarkDotNet.Templates"); nuGetPackageNames.Sort(); NuGetPackageNames = nuGetPackageNames; VersionHistory = new VersionHistory(this, VersionsFile); GitRunner = new GitRunner(this); UnitTestRunner = new UnitTestRunner(this); DocumentationRunner = new DocumentationRunner(this); BuildRunner = new BuildRunner(this); ReleaseRunner = new ReleaseRunner(this); } public void GenerateFile(FilePath filePath, StringBuilder content) { GenerateFile(filePath, content.ToString()); } public void GenerateFile(FilePath filePath, string content, bool reportNoChanges = false) { this.EnsureDirectoryExists(filePath.GetDirectory()); var relativePath = RootDirectory.GetRelativePath(filePath); if (this.FileExists(filePath)) { var oldContent = this.FileReadText(filePath); if (content == oldContent) { if (reportNoChanges) this.Information("[NoChanges] " + relativePath); return; } this.FileWriteText(filePath, content); this.Information("[Updated] " + relativePath); } else { this.FileWriteText(filePath, content); this.Information("[Generated] " + relativePath); } } public void RunOnlyInPushMode(Action action) { if (KnownOptions.Push.Resolve(this)) { action(); } else this.Information(" Skip because PushMode is disabled"); } } ================================================ FILE: build/BenchmarkDotNet.Build/CommandLineParser.cs ================================================ using System; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; using BenchmarkDotNet.Build.Options; using Cake.Frosting; namespace BenchmarkDotNet.Build; public class CommandLineParser { private const string ScriptName = "build.cmd"; private static readonly string CallScriptName = (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) ? ScriptName : "./" + ScriptName; public static readonly CommandLineParser Instance = new(); public string[]? Parse(string[]? args) { if (args == null || args.Length == 0 || (args.Length == 1 && Is(args[0], "help", "--help", "-h"))) { PrintHelp(); return null; } if (Is(args[0], "cake")) return args.Skip(1).ToArray(); var argsToProcess = new Queue(args); var taskName = argsToProcess.Dequeue(); if (Is(taskName, "-t", "--target") && argsToProcess.Any()) taskName = argsToProcess.Dequeue(); var taskNames = GetTaskNames(); var matchedTaskName = taskNames .FirstOrDefault(name => string.Equals(name.Replace("-", ""), taskName.Replace("-", ""), StringComparison.OrdinalIgnoreCase)); if (matchedTaskName == null) { PrintError($"'{taskName}' is not a task"); return null; } taskName = matchedTaskName; if (argsToProcess.Count == 1 && Is(argsToProcess.Peek(), "-h", "--help")) { PrintTaskHelp(taskName); return null; } var cakeArgs = new List { "--target", taskName }; while (argsToProcess.Any()) { var arg = argsToProcess.Dequeue(); if (arg.StartsWith("/p:")) { cakeArgs.Add("--msbuild"); cakeArgs.Add(arg[3..]); continue; } if (arg.StartsWith('-')) { cakeArgs.Add(arg); if (argsToProcess.Any() && !argsToProcess.Peek().StartsWith('-')) cakeArgs.Add(argsToProcess.Dequeue()); continue; } PrintError("Unknown option: " + arg); return null; } return cakeArgs.ToArray(); } private readonly IOption[] baseOptions = { KnownOptions.Verbosity, KnownOptions.Exclusive, KnownOptions.Help, KnownOptions.Stable }; private void PrintHelp() { WriteHeader("Description:"); WritePrefix(); WriteLine("BenchmarkDotNet build script"); WriteLine(); WriteHeader("Usage:"); WritePrefix(); Write(CallScriptName + " "); WriteTask(" "); WriteOption("[OPTIONS]"); WriteLine(); WriteLine(); PrintExamples(GetTasks().SelectMany(task => task.HelpInfo.Examples).ToList()); PrintOptions(baseOptions); WriteHeader("Tasks:"); var taskWidth = GetTaskNames().Max(name => name.Length) + 3; foreach (var (taskName, taskDescription, _) in GetTasks()) { if (taskName.Equals("Default", StringComparison.OrdinalIgnoreCase)) continue; WriteTask(" " + taskName.PadRight(taskWidth)); Write(taskDescription); WriteLine(); } } private void PrintTaskHelp(string taskName) { var taskType = typeof(BuildContext).Assembly .GetTypes() .Where(type => type.IsSubclassOf(typeof(FrostingTask)) && !type.IsAbstract) .First(type => Is(type.GetCustomAttribute()?.Name, taskName)); taskName = taskType.GetCustomAttribute()!.Name; var taskDescription = taskType.GetCustomAttribute()?.Description ?? ""; var helpInfo = GetHelpInfo(taskType); WriteHeader("Description:"); WritePrefix(); WriteLine(!string.IsNullOrWhiteSpace(taskDescription) ? $"Task '{taskName}': {taskDescription}" : $"Task '{taskName}'"); if (!string.IsNullOrWhiteSpace(helpInfo.Description)) foreach (var line in helpInfo.Description.Split('\n', StringSplitOptions.RemoveEmptyEntries)) { WritePrefix(); WriteLine(line.TrimEnd()); } WriteLine(); WriteHeader("Usage:"); WritePrefix(); Write(CallScriptName + " "); WriteTask(taskName + " "); WriteOption("[OPTIONS]"); WriteLine(); WriteLine(); PrintExamples(helpInfo.Examples); PrintOptions(helpInfo.Options.Concat(baseOptions).ToArray()); if (helpInfo.EnvironmentVariables.Any()) { WriteHeader("Environment variables:"); foreach (var envVar in helpInfo.EnvironmentVariables) { WritePrefix(); WriteOption(envVar.Name); WriteLine(); } } } private void PrintOptions(IOption[] options) { const string valuePlaceholder = ""; WriteLine("Options:", ConsoleColor.DarkCyan); int GetWidth(IOption option) { int width = option.CommandLineName.Length; foreach (var alias in option.Aliases) width += 1 + alias.Length; if (option is StringOption) width += 1 + valuePlaceholder.Length; return width; } const int descriptionGap = 3; var maxWidth = options.Max(GetWidth) + descriptionGap; foreach (var option in options) { var allNames = option.Aliases.Append(option.CommandLineName).OrderBy(name => name.Length); var joinName = string.Join(',', allNames); WritePrefix(); WriteOption(joinName); if (option is StringOption) { Write(" "); WriteArg(valuePlaceholder); } Write(new string(' ', maxWidth - joinName.Length - (option is StringOption ? valuePlaceholder.Length + 1 : 0))); var descriptionLines = option.Description.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); Write(descriptionLines.FirstOrDefault() ?? ""); for (int i = 1; i < descriptionLines.Length; i++) { WriteLine(); WritePrefix(); Write(new string(' ', maxWidth)); Write(descriptionLines[i]); } WriteLine(); } WritePrefix(); WriteOption("/p:"); WriteArg(""); WriteOption("="); WriteArg(valuePlaceholder); Write(new string(' ', maxWidth - "/p:=".Length - valuePlaceholder.Length)); Write("Passes custom properties to MSBuild"); WriteLine(); WriteLine(); } private void PrintExamples(IReadOnlyList examples) { if (!examples.Any()) return; WriteHeader("Examples:"); foreach (var example in examples) { WritePrefix(); Write(CallScriptName + " "); WriteTask(example.TaskName + " "); foreach (var (name, value, isMsBuild) in example.Arguments) { if (isMsBuild) { WriteOption("/p:"); WriteArg(name); WriteOption("="); WriteArg(value + " "); } else { WriteOption(name + " "); if (value != null) WriteArg(value + " "); } } WriteLine(); } WriteLine(); } private static HashSet GetTaskNames() { return GetTasks().Select(task => task.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); } private static List<(string Name, string Description, HelpInfo HelpInfo)> GetTasks() { return typeof(BuildContext).Assembly .GetTypes() .Where(type => type.IsSubclassOf(typeof(FrostingTask)) && !type.IsAbstract) .Select(type => ( Name: type.GetCustomAttribute()?.Name ?? "", Description: type.GetCustomAttribute()?.Description ?? "", HelpInfo: GetHelpInfo(type) )) .Where(task => task.Name != "") .ToList(); } private static HelpInfo GetHelpInfo(Type taskType) { return Activator.CreateInstance(taskType) is IHelpProvider helpProvider ? helpProvider.GetHelp() : new HelpInfo(); } private static bool Is(string? arg, params string[] values) => values.Any(value => value.Equals(arg, StringComparison.OrdinalIgnoreCase)); private void PrintError(string text) { Console.ForegroundColor = ConsoleColor.Red; Console.Error.WriteLine("ERROR: " + text); Console.WriteLine(); Console.ResetColor(); PrintHelp(); } private void WritePrefix() => Write(" "); private void WriteTask(string message) => Write(message, ConsoleColor.Green); private void WriteOption(string message) => Write(message, ConsoleColor.Blue); private void WriteArg(string message) => Write(message, ConsoleColor.DarkYellow); private void WriteObsolete(string message) => Write(message, ConsoleColor.Gray); private void WriteHeader(string message) { WriteLine(message, ConsoleColor.DarkCyan); } private void Write(string message, ConsoleColor? color = null) { if (color != null) Console.ForegroundColor = color.Value; Console.Write(message); if (color != null) Console.ResetColor(); } private void WriteLine(string message = "", ConsoleColor? color = null) { Write(message, color); Console.WriteLine(); } } ================================================ FILE: build/BenchmarkDotNet.Build/EnvVar.cs ================================================ using System; namespace BenchmarkDotNet.Build; public class EnvVar { public static readonly EnvVar GitHubToken = new("GITHUB_TOKEN"); public static readonly EnvVar NuGetToken = new("NUGET_TOKEN"); public string Name { get; } private EnvVar(string name) => Name = name; public string? GetValue() => Environment.GetEnvironmentVariable(Name); public void AssertHasValue() { if (string.IsNullOrEmpty(GetValue())) throw new Exception($"Environment variable '{Name}' is not specified!"); } public void SetEmpty() => Environment.SetEnvironmentVariable(Name, ""); } ================================================ FILE: build/BenchmarkDotNet.Build/Example.cs ================================================ using System.Collections.Generic; using BenchmarkDotNet.Build.Options; namespace BenchmarkDotNet.Build; public class Example { private readonly List arguments = new(); public string TaskName { get; } public IReadOnlyCollection Arguments => arguments; public Example(string taskName) { TaskName = taskName; } public Example WithMsBuildArgument(string name, string value) { arguments.Add(new Argument(name, value, true)); return this; } public Example WithArgument(BoolOption option) { arguments.Add(new Argument(option.CommandLineName, null, false)); return this; } public Example WithArgument(StringOption option, string value) { arguments.Add(new Argument(option.CommandLineName, value, false)); return this; } public record Argument(string Name, string? Value, bool IsMsBuild); } ================================================ FILE: build/BenchmarkDotNet.Build/Folder.DotSettings ================================================  <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> True True ================================================ FILE: build/BenchmarkDotNet.Build/HelpInfo.cs ================================================ using System; using BenchmarkDotNet.Build.Options; namespace BenchmarkDotNet.Build; public class HelpInfo { public string Description { get; init; } = ""; public IOption[] Options { get; init; } = Array.Empty(); public EnvVar[] EnvironmentVariables { get; init; } = Array.Empty(); public Example[] Examples { get; init; } = Array.Empty(); } ================================================ FILE: build/BenchmarkDotNet.Build/Helpers/OctokitExtensions.cs ================================================ using System; using System.Linq; using Octokit; namespace BenchmarkDotNet.Build.Helpers; public static class OctokitExtensions { public static string ToStr(this User? user, string prefix) => user != null ? $" ({prefix} [@{user.Login}]({user.HtmlUrl}))" : ""; private static string ToStr(this Author? user, string prefix) => user != null ? $" ({prefix} {user.ToLink()})" : ""; private static string ToStr(this Committer? user, string prefix) => user != null ? $" ({prefix} {user.Name})" : ""; public static string ToLink(this Author user) => $"[@{user.Login}]({user.HtmlUrl})"; public static string ToLinkWithName(this Author user, string name) => $"[@{user.Login} ({name})]({user.HtmlUrl})"; public static string ToCommitMessage(this Commit commit) { var message = commit.Message.Trim().Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) .FirstOrDefault() ?? ""; return message.Length > 80 ? message.Substring(0, 77) + "..." : message; } public static string ToLink(this GitHubCommit commit) => $"[{commit.Sha.Substring(0, 6)}]({commit.HtmlUrl})"; public static string ToByStr(this GitHubCommit commit) { if (commit.Author != null) return commit.Author.ToStr("by"); return commit.Commit.Author != null ? commit.Commit.Author.ToStr("by") : ""; } } ================================================ FILE: build/BenchmarkDotNet.Build/Helpers/Utils.cs ================================================ using System; using System.Collections.Generic; using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Cake.Common.Tools.DotNet; using Octokit; namespace BenchmarkDotNet.Build.Helpers; public static class Utils { public static string GetOs() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "linux"; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return "windows"; if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return "macos"; return "unknown"; } public static DotNetVerbosity? ParseVerbosity(string verbosity) { var lookup = new Dictionary(StringComparer.OrdinalIgnoreCase) { { "q", DotNetVerbosity.Quiet }, { "quiet", DotNetVerbosity.Quiet }, { "m", DotNetVerbosity.Minimal }, { "minimal", DotNetVerbosity.Minimal }, { "n", DotNetVerbosity.Normal }, { "normal", DotNetVerbosity.Normal }, { "d", DotNetVerbosity.Detailed }, { "detailed", DotNetVerbosity.Detailed }, { "diag", DotNetVerbosity.Diagnostic }, { "diagnostic", DotNetVerbosity.Diagnostic } }; return lookup.TryGetValue(verbosity, out var value) ? value : null; } public static GitHubClient CreateGitHubClient() { EnvVar.GitHubToken.AssertHasValue(); var client = new GitHubClient(new ProductHeaderValue("BenchmarkDotNet")); var tokenAuth = new Credentials(EnvVar.GitHubToken.GetValue()); client.Credentials = tokenAuth; return client; } public static string ApplyRegex(string content, string pattern, string newValue) { var regex = new Regex(pattern); var match = regex.Match(content); if (!match.Success) throw new Exception("Failed to apply regex"); var oldValue = match.Groups[1].Value; return content.Replace(oldValue, newValue); } } ================================================ FILE: build/BenchmarkDotNet.Build/IHelpProvider.cs ================================================ namespace BenchmarkDotNet.Build; public interface IHelpProvider { HelpInfo GetHelp(); } ================================================ FILE: build/BenchmarkDotNet.Build/Meta/Repo.cs ================================================ using System; using System.Net.Http; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace BenchmarkDotNet.Build.Meta; public static class Repo { public const string Owner = "dotnet"; public const string Name = "BenchmarkDotNet"; private const string HttpsUrlBase = $"https://github.com/{Owner}/{Name}"; public const string SshGitUrl = $"git@github.com:{Owner}/{Name}.git"; public const string ChangelogBranch = "docs-changelog"; public const string DocsStableBranch = "docs-stable"; public const string MasterBranch = "master"; public const string MaintainerAuthorName = "GitHub Actions"; public const string MaintainerAuthorEmail = "actions@github.com"; public static async Task GetDependentProjectsNumber() { using var httpClient = new HttpClient(); const string url = $"{HttpsUrlBase}/network/dependents"; var response = await httpClient.GetAsync(new Uri(url)); var dependentsPage = await response.Content.ReadAsStringAsync(); var match = new Regex(@"([0-9\,]+)[\n\r\s]+Repositories").Match(dependentsPage); var number = int.Parse(match.Groups[1].Value.Replace(",", "")); number = number / 100 * 100; return number; } } ================================================ FILE: build/BenchmarkDotNet.Build/Meta/VersionHistory.cs ================================================ using System.Linq; using Cake.Core.IO; using Cake.FileHelpers; namespace BenchmarkDotNet.Build.Meta; public class VersionHistory { public string FirstCommit { get; } public string[] StableVersions { get; } public string CurrentVersion { get; } public VersionHistory(BuildContext context, FilePath versionFilePath) { var lines = context.FileReadLines(versionFilePath).Where(line => !string.IsNullOrWhiteSpace(line)).ToArray(); FirstCommit = lines.First(); CurrentVersion = lines.Last(); StableVersions = lines.Skip(1).SkipLast(1).ToArray(); } } ================================================ FILE: build/BenchmarkDotNet.Build/Options/BoolOption.cs ================================================ using System; namespace BenchmarkDotNet.Build.Options; public class BoolOption : Option { public BoolOption(string commandLineName) : base(commandLineName) { } public override bool Resolve(BuildContext context) { if (!HasArgument(context)) return false; var value = GetArgument(context); if (value == null) return true; return !value.Equals(false.ToString(), StringComparison.OrdinalIgnoreCase); } public void AssertTrue(BuildContext context) { var value = Resolve(context); if (!value) throw new Exception($"{CommandLineName} is not specified"); } } ================================================ FILE: build/BenchmarkDotNet.Build/Options/IOption.cs ================================================ namespace BenchmarkDotNet.Build.Options; public interface IOption { string CommandLineName { get; } string Description { get; } string[] Aliases { get; } } ================================================ FILE: build/BenchmarkDotNet.Build/Options/KnownOptions.cs ================================================ namespace BenchmarkDotNet.Build.Options; public static class KnownOptions { public static readonly StringOption Verbosity = new("--verbosity" ) { Description = "Specifies the amount of information to be displayed\n" + "(Quiet, Minimal, Normal, Verbose, Diagnostic)", Aliases = new[] { "-v" } }; public static readonly BoolOption Exclusive = new("--exclusive") { Description = "Executes the target task without any dependencies", Aliases = new[] { "-e" } }; public static readonly BoolOption DocsPreview = new("--preview") { Description = "When specified, documentation changelog includes the upcoming version", Aliases = new[] { "-p" } }; public static readonly StringOption DocsDepth = new("--depth") { Description = "The number of last stable versions that requires changelog regenerations\n" + "Use 'all' for all values. The default is zero.", Aliases = new[] { "-d" } }; public static readonly BoolOption ForceClone = new("--force-clone") { Description = "Forces re-cloning of the changelog repository, deleting any existing directory.", Aliases = new[] { "-f" } }; public static readonly BoolOption Help = new("--help") { Description = "Prints help information", Aliases = new[] { "-h" } }; public static readonly BoolOption Stable = new("--stable") { Description = "Removes VersionSuffix in MSBuild settings", Aliases = new[] { "-s" } }; public static readonly StringOption NextVersion = new("--next-version") { Description = "Specifies next version number", Aliases = new[] { "-n" } }; public static readonly BoolOption Push = new("--push") { Description = "When specified, the task actually perform push to GitHub and nuget.org" }; } ================================================ FILE: build/BenchmarkDotNet.Build/Options/Option.cs ================================================ using System; using System.Collections.Generic; using System.Linq; using Cake.Common; using Cake.Core; namespace BenchmarkDotNet.Build.Options; public abstract class Option : IOption { public string CommandLineName { get; } public string Description { get; init; } = ""; public string[] Aliases { get; init; } = Array.Empty(); private IEnumerable AllNames { get { yield return CommandLineName; foreach (var alias in Aliases) yield return alias; } } private IEnumerable AllStrippedNames => AllNames.Select(name => name.TrimStart('-')); protected Option(string commandLineName) { CommandLineName = commandLineName; } public abstract T Resolve(BuildContext context); protected bool HasArgument(BuildContext context) => AllStrippedNames.Any(context.HasArgument); protected string? GetArgument(BuildContext context) => AllStrippedNames .Where(context.HasArgument) .Select(name => context.Arguments.GetArgument(name)) .FirstOrDefault(); } ================================================ FILE: build/BenchmarkDotNet.Build/Options/StringOption.cs ================================================ using System; namespace BenchmarkDotNet.Build.Options; public class StringOption : Option { public StringOption(string commandLineName) : base(commandLineName) { } public override string Resolve(BuildContext context) { if (!HasArgument(context)) return ""; var value = GetArgument(context); if (value == null || string.IsNullOrWhiteSpace(value)) return ""; return value.Trim(); } public string AssertHasValue(BuildContext context) { var value = Resolve(context); if (string.IsNullOrWhiteSpace(value)) throw new Exception($"{CommandLineName} is not specified"); return value; } } ================================================ FILE: build/BenchmarkDotNet.Build/Program.cs ================================================ using BenchmarkDotNet.Build.Meta; using BenchmarkDotNet.Build.Options; using Cake.Common; using Cake.Frosting; namespace BenchmarkDotNet.Build; public static class Program { public static int Main(string[] args) { var cakeArgs = CommandLineParser.Instance.Parse(args); return cakeArgs == null ? 0 : new CakeHost().UseContext().Run(cakeArgs); } } [TaskName(Name)] [TaskDescription("Pack Weaver")] public class PackWeaverTask : FrostingTask, IHelpProvider { private const string Name = "pack-weaver"; public override void Run(BuildContext context) => context.BuildRunner.PackWeaver(); public HelpInfo GetHelp() { return new HelpInfo { Examples = new[] { new Example(Name) } }; } } [TaskName(Name)] [TaskDescription("Restore NuGet packages")] [IsDependentOn(typeof(PackWeaverTask))] public class RestoreTask : FrostingTask, IHelpProvider { private const string Name = "restore"; public override void Run(BuildContext context) => context.BuildRunner.Restore(); public HelpInfo GetHelp() { return new HelpInfo { Examples = [ new Example(Name) ] }; } } [TaskName(Name)] [TaskDescription("Build BenchmarkDotNet.slnx solution")] [IsDependentOn(typeof(RestoreTask))] public class BuildTask : FrostingTask, IHelpProvider { private const string Name = "build"; public override void Run(BuildContext context) => context.BuildRunner.Build(); public HelpInfo GetHelp() { return new HelpInfo { Examples = [ new Example(Name).WithMsBuildArgument("Configuration", "Debug") ] }; } } [TaskName(Name)] [TaskDescription("Install wasm-tools workload")] public class InstallWasmToolsWorkload : FrostingTask, IHelpProvider { private const string Name = "install-wasm-tools"; public override void Run(BuildContext context) { context.BuildRunner.InstallWorkload("wasm-tools-net8"); context.BuildRunner.InstallWorkload("wasm-tools"); } public HelpInfo GetHelp() { return new HelpInfo { Examples = [ new Example(Name) ] }; } } [TaskName(Name)] [TaskDescription("Run unit tests (fast)")] [IsDependentOn(typeof(BuildTask))] public class UnitTestsTask : FrostingTask, IHelpProvider { private const string Name = "unit-tests"; public override void Run(BuildContext context) => context.UnitTestRunner.RunUnitTests(); public HelpInfo GetHelp() { return new HelpInfo { Examples = [ new Example(Name) .WithArgument(KnownOptions.Exclusive) .WithArgument(KnownOptions.Verbosity, "Diagnostic") ] }; } } [TaskName(Name)] [TaskDescription("Run analyzer tests")] [IsDependentOn(typeof(BuildTask))] public class AnalyzerTestsTask : FrostingTask, IHelpProvider { private const string Name = "analyzer-tests"; public override void Run(BuildContext context) => context.UnitTestRunner.RunAnalyzerTests(); public HelpInfo GetHelp() { return new HelpInfo { Examples = [ new Example(Name) .WithArgument(KnownOptions.Exclusive) .WithArgument(KnownOptions.Verbosity, "Diagnostic") ] }; } } [TaskName(Name)] [TaskDescription("Run integration tests using .NET Framework 4.6.2+ (slow)")] [IsDependentOn(typeof(BuildTask))] public class InTestsFullTask : FrostingTask, IHelpProvider { private const string Name = "in-tests-full"; public override bool ShouldRun(BuildContext context) => context.IsRunningOnWindows(); public override void Run(BuildContext context) => context.UnitTestRunner.RunInTests("net462"); public HelpInfo GetHelp() => new(); } [TaskName(Name)] [TaskDescription("Run integration tests using .NET 8 (slow)")] [IsDependentOn(typeof(BuildTask))] public class InTestsCoreTask : FrostingTask, IHelpProvider { private const string Name = "in-tests-core"; public override void Run(BuildContext context) => context.UnitTestRunner.RunInTests("net8.0"); public HelpInfo GetHelp() => new(); } [TaskName(Name)] [TaskDescription("Run all unit, analyzer, and integration tests (slow)")] [IsDependentOn(typeof(UnitTestsTask))] [IsDependentOn(typeof(AnalyzerTestsTask))] [IsDependentOn(typeof(InTestsFullTask))] [IsDependentOn(typeof(InTestsCoreTask))] public class AllTestsTask : FrostingTask, IHelpProvider { private const string Name = "all-tests"; public HelpInfo GetHelp() => new(); } [TaskName(Name)] [TaskDescription("Build BenchmarkDotNet.Analyzers")] public class BuildAnalyzersTask : FrostingTask, IHelpProvider { private const string Name = "build-analyzers"; public override void Run(BuildContext context) => context.BuildRunner.BuildAnalyzers(); public HelpInfo GetHelp() { return new HelpInfo { Examples = [ new Example(Name) ] }; } } [TaskName(Name)] [TaskDescription("Move updated analyzer rules from unshipped to shipped file")] public class MoveAnalyzerRulesTask : FrostingTask, IHelpProvider { private const string Name = "move-analyzer-rules"; public override void Run(BuildContext context) => context.DocumentationRunner.MoveAnalyzerRules(); public HelpInfo GetHelp() { return new HelpInfo { Examples = [ new Example(Name) ] }; } } [TaskName(Name)] [TaskDescription("Pack Nupkg packages")] [IsDependentOn(typeof(BuildTask))] [IsDependentOn(typeof(BuildAnalyzersTask))] public class PackTask : FrostingTask, IHelpProvider { public const string Name = "pack"; public override void Run(BuildContext context) => context.BuildRunner.Pack(); public HelpInfo GetHelp() { return new HelpInfo { Examples = [ new Example(Name) .WithMsBuildArgument("VersionPrefix", "0.1.1729") .WithMsBuildArgument("VersionSuffix", "preview"), new Example(Name).WithArgument(KnownOptions.Stable) ] }; } } [TaskName(Name)] [TaskDescription("Fetch changelog files")] public class DocsFetchTask : FrostingTask, IHelpProvider { private const string Name = "docs-fetch"; public override void Run(BuildContext context) => context.DocumentationRunner.Fetch(); public HelpInfo GetHelp() { return new HelpInfo { Description = $"This task updates the following files:\n" + $"* Clones or fetches branch 'docs-changelog' to docs/_changelog\n" + $"* Last changelog footer (if {KnownOptions.Stable.CommandLineName} is specified)\n" + $"* All changelog details in docs/_changelog\n" + $" (This dir is a cloned version of this repo from branch {Repo.ChangelogBranch})", Options = [KnownOptions.DocsPreview, KnownOptions.DocsDepth, KnownOptions.ForceClone], EnvironmentVariables = [EnvVar.GitHubToken], Examples = [ new Example(Name) .WithArgument(KnownOptions.DocsDepth, "3") .WithArgument(KnownOptions.DocsPreview) .WithArgument(KnownOptions.ForceClone) ] }; } } [TaskName(Name)] [TaskDescription("Generate auxiliary documentation files")] public class DocsGenerateTask : FrostingTask, IHelpProvider { private const string Name = "docs-generate"; public override void Run(BuildContext context) => context.DocumentationRunner.Generate(); public HelpInfo GetHelp() { return new HelpInfo { Options = [KnownOptions.DocsPreview, KnownOptions.Stable], Examples = [ new Example(Name).WithArgument(KnownOptions.DocsPreview), new Example(Name).WithArgument(KnownOptions.Stable) ] }; } } [TaskName(Name)] [TaskDescription("Build final documentation")] [IsDependentOn(typeof(DocsGenerateTask))] public class DocsBuildTask : FrostingTask, IHelpProvider { private const string Name = "docs-build"; public override void Run(BuildContext context) => context.DocumentationRunner.Build(); public HelpInfo GetHelp() => new() { Description = "The 'build' task should be run manually to build api docs", Options = [KnownOptions.DocsPreview], Examples = [ new Example(Name).WithArgument(KnownOptions.DocsPreview) ] }; } [TaskName(Name)] [TaskDescription("Increments the current version")] public class VersionIncrementTask : FrostingTask, IHelpProvider { private const string Name = "version-increment"; public override void Run(BuildContext context) { context.ReleaseRunner.VersionIncrement(); context.BuildRunner.PackWeaver(); } public HelpInfo GetHelp() => new() { Options = [KnownOptions.NextVersion], Examples = [ new Example(Name), new Example(Name) .WithArgument(KnownOptions.NextVersion, "0.1.1729") ] }; } [TaskName(Name)] [TaskDescription("Release new version")] [IsDependentOn(typeof(BuildTask))] [IsDependentOn(typeof(PackTask))] [IsDependentOn(typeof(DocsFetchTask))] [IsDependentOn(typeof(DocsGenerateTask))] [IsDependentOn(typeof(DocsBuildTask))] public class ReleaseTask : FrostingTask, IHelpProvider { public const string Name = "release"; public override void Run(BuildContext context) => context.ReleaseRunner.Run(); public HelpInfo GetHelp() => new() { Options = [KnownOptions.NextVersion, KnownOptions.Push], EnvironmentVariables = [EnvVar.GitHubToken, EnvVar.NuGetToken], Examples = [ new Example(Name) .WithArgument(KnownOptions.Stable) .WithArgument(KnownOptions.NextVersion, "0.1.1729") .WithArgument(KnownOptions.Push), new Example(Name) .WithArgument(KnownOptions.Stable) .WithArgument(KnownOptions.Push) ] }; } ================================================ FILE: build/BenchmarkDotNet.Build/Runners/BuildRunner.cs ================================================ using Cake.Common.Build; using Cake.Common.Diagnostics; using Cake.Common.IO; using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.Build; using Cake.Common.Tools.DotNet.Pack; using Cake.Common.Tools.DotNet.Restore; using Cake.Common.Tools.DotNet.Workload.Install; using Cake.Core; using Cake.Core.IO; using System; using System.IO; using System.Linq; namespace BenchmarkDotNet.Build.Runners; public class BuildRunner { private readonly BuildContext context; private readonly bool isFullPack; public BuildRunner(BuildContext context) { this.context = context; isFullPack = context.Arguments.GetArgument("target") is PackTask.Name or ReleaseTask.Name; } private void MaybeAppendArgument(DotNetSettings settings) { if (isFullPack) { settings.ArgumentCustomization = args => args.Append("-p:IsFullPack=true"); } } public void PackWeaver() { var weaverPath = context.AllPackableSrcProjects.Single(p => p.GetFilename() == "BenchmarkDotNet.Weaver.csproj"); var outputPackageDir = weaverPath.GetDirectory().Combine("packages"); if (!isFullPack) { // Delete old package. foreach (var file in Directory.EnumerateFiles(outputPackageDir.FullPath)) { File.Delete(file); } } var restoreSettings = new DotNetRestoreSettings { MSBuildSettings = context.MsBuildSettingsRestore, }; MaybeAppendArgument(restoreSettings); context.DotNetRestore(weaverPath.GetDirectory().FullPath, restoreSettings); context.Information("BuildSystemProvider: " + context.BuildSystem().Provider); var buildSettings = new DotNetBuildSettings { NoRestore = true, DiagnosticOutput = true, MSBuildSettings = context.MsBuildSettingsBuild, Configuration = context.BuildConfiguration, Verbosity = context.BuildVerbosity }; MaybeAppendArgument(buildSettings); context.DotNetBuild(weaverPath.FullPath, buildSettings); var packSettings = new DotNetPackSettings { OutputDirectory = outputPackageDir, MSBuildSettings = context.MsBuildSettingsPack, Configuration = context.BuildConfiguration }; MaybeAppendArgument(packSettings); context.DotNetPack(weaverPath.FullPath, packSettings); } public void Restore() { var restoreSettings = new DotNetRestoreSettings { MSBuildSettings = context.MsBuildSettingsRestore, }; MaybeAppendArgument(restoreSettings); context.DotNetRestore(context.SolutionFile.FullPath, restoreSettings); } public void InstallWorkload(string workloadId) { context.DotNetWorkloadInstall(workloadId, new DotNetWorkloadInstallSettings { IncludePreviews = true, NoCache = true }); } public void Build() { context.Information("BuildSystemProvider: " + context.BuildSystem().Provider); var buildSettings = new DotNetBuildSettings { NoRestore = true, DiagnosticOutput = true, MSBuildSettings = context.MsBuildSettingsBuild, Configuration = context.BuildConfiguration, Verbosity = context.BuildVerbosity }; MaybeAppendArgument(buildSettings); context.DotNetBuild(context.SolutionFile.FullPath, buildSettings); } public void BuildProjectSilent(FilePath projectFile) { var buildSettings = new DotNetBuildSettings { NoRestore = false, DiagnosticOutput = false, MSBuildSettings = context.MsBuildSettingsBuild, Configuration = context.BuildConfiguration, Verbosity = DotNetVerbosity.Quiet }; MaybeAppendArgument(buildSettings); context.DotNetBuild(projectFile.FullPath, buildSettings); } public void BuildAnalyzers() { context.Information("BuildSystemProvider: " + context.BuildSystem().Provider); string[] mccVersions = ["2.8", "3.8", "4.8"]; foreach (string version in mccVersions) { context.DotNetBuild(context.AnalyzersProjectFile.FullPath, new DotNetBuildSettings { NoRestore = true, DiagnosticOutput = true, MSBuildSettings = context.MsBuildSettingsBuild, Configuration = context.BuildConfiguration, Verbosity = context.BuildVerbosity, ArgumentCustomization = args => args.Append($"-p:MccVersion={version}") }); } } public void Pack() { context.CleanDirectory(context.ArtifactsDirectory); var settingsSrc = new DotNetPackSettings { OutputDirectory = context.ArtifactsDirectory, ArgumentCustomization = args => args.Append("--include-symbols").Append("-p:SymbolPackageFormat=snupkg").Append("-p:IsFullPack=true"), MSBuildSettings = context.MsBuildSettingsPack, Configuration = context.BuildConfiguration }; foreach (var project in context.AllPackableSrcProjects) context.DotNetPack(project.FullPath, settingsSrc); var settingsTemplate = new DotNetPackSettings { OutputDirectory = context.ArtifactsDirectory, MSBuildSettings = context.MsBuildSettingsPack, Configuration = context.BuildConfiguration }; context.DotNetPack(context.TemplatesTestsProjectFile.FullPath, settingsTemplate); } } ================================================ FILE: build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs ================================================ using System; using System.Globalization; using System.IO; using System.Linq; using System.Text; using BenchmarkDotNet.Build.Meta; using BenchmarkDotNet.Build.Options; using Cake.Common.Diagnostics; using Cake.Common.IO; using Cake.Core.IO; using Cake.FileHelpers; namespace BenchmarkDotNet.Build.Runners.Changelog; public class ChangelogBuilder { private readonly BuildContext context; private readonly bool preview; private readonly string depth; private readonly bool forceClone; /// /// Directory with original changelog part files from branch 'docs-changelog' /// public DirectoryPath SrcDirectory { get; } /// /// Final changelog files to be used by docfx /// public DirectoryPath DocfxDirectory { get; } public ChangelogBuilder(BuildContext context) { this.context = context; preview = KnownOptions.DocsPreview.Resolve(context); depth = KnownOptions.DocsDepth.Resolve(context); forceClone = KnownOptions.ForceClone.Resolve(context); var docsDirectory = context.RootDirectory.Combine("docs"); SrcDirectory = docsDirectory.Combine("_changelog"); DocfxDirectory = docsDirectory.Combine("changelog"); } public void Fetch() { EnvVar.GitHubToken.AssertHasValue(); EnsureSrcDirectoryExist(forceClone); var history = context.VersionHistory; var stableVersionCount = history.StableVersions.Length; if (depth.Equals("all", StringComparison.OrdinalIgnoreCase)) { FetchDetails( history.StableVersions.First(), history.FirstCommit); for (var i = 1; i < stableVersionCount; i++) FetchDetails( history.StableVersions[i], history.StableVersions[i - 1]); } else if (depth != "") { if (!int.TryParse(depth, CultureInfo.InvariantCulture, out var depthValue)) throw new InvalidDataException($"Failed to parse the depth value: '{depth}'"); for (var i = Math.Max(stableVersionCount - depthValue, 1); i < stableVersionCount; i++) FetchDetails( history.StableVersions[i], history.StableVersions[i - 1]); } if (preview) FetchDetails( history.CurrentVersion, history.StableVersions.Last(), "HEAD"); context.GitRunner.AddAll(SrcDirectory); context.GitRunner.Commit("Update changelog", SrcDirectory); } private void FetchDetails(string version, string versionPrevious, string lastCommit = "") { EnsureSrcDirectoryExist(); context.Information($"Downloading changelog details for v{version}"); var detailsDirectory = SrcDirectory.Combine("details"); ChangelogDetailsBuilder.Run(context, detailsDirectory, version, versionPrevious, lastCommit); } public void Generate() { GenerateLastHeader(); GenerateLastFooter(); foreach (var version in context.VersionHistory.StableVersions) GenerateVersion(version); if (preview) GenerateVersion(context.VersionHistory.CurrentVersion); GenerateIndex(); GenerateFull(); GenerateToc(); } public void GenerateLastHeader() { var fileName = "v" + context.VersionHistory.CurrentVersion + ".md"; var filePath = SrcDirectory.Combine("header").CombineWithFilePath(fileName); if (!context.FileExists(filePath)) { var relativePath = context.RootDirectory.GetRelativePath(filePath); context.FileWriteText(filePath, ""); context.Information("[Created] " + relativePath); } } public void GenerateLastFooter() { var version = context.VersionHistory.CurrentVersion; var previousVersion = context.VersionHistory.StableVersions.Last(); var date = KnownOptions.Stable.Resolve(context) ? DateTime.Now.ToString("MMMM dd, yyyy", CultureInfo.InvariantCulture) : "TBA"; var content = new StringBuilder(); content.AppendLine($"_Date: {date}_"); content.AppendLine(""); content.AppendLine( $"_Milestone: [v{version}](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av{version})_"); content.AppendLine( $"([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v{previousVersion}...v{version}))"); content.AppendLine(""); content.AppendLine("_NuGet Packages:_"); foreach (var packageName in context.NuGetPackageNames) content.AppendLine($"* https://www.nuget.org/packages/{packageName}/{version}"); var fileName = "v" + context.VersionHistory.CurrentVersion + ".md"; var filePath = SrcDirectory.Combine("footer").CombineWithFilePath(fileName); context.GenerateFile(filePath, content); } private void GenerateVersion(string version) { EnsureSrcDirectoryExist(); var md = $"v{version}.md"; var header = SrcDirectory.Combine("header").CombineWithFilePath(md); var footer = SrcDirectory.Combine("footer").CombineWithFilePath(md); var details = SrcDirectory.Combine("details").CombineWithFilePath(md); var release = DocfxDirectory.CombineWithFilePath(md); var content = new StringBuilder(); content.AppendLine("---"); content.AppendLine("uid: changelog.v" + version); content.AppendLine("---"); content.AppendLine(""); content.AppendLine("# BenchmarkDotNet v" + version); content.AppendLine(""); content.AppendLine(""); if (context.FileExists(header)) { content.AppendLine(context.FileReadText(header)); content.AppendLine(""); content.AppendLine(""); } if (context.FileExists(details)) { content.AppendLine(context.FileReadText(details)); content.AppendLine(""); content.AppendLine(""); } if (context.FileExists(footer)) { content.AppendLine("## Additional details"); content.AppendLine(""); content.AppendLine(context.FileReadText(footer)); } context.GenerateFile(release, content.ToString()); } private void GenerateIndex() { var content = new StringBuilder(); content.AppendLine("---"); content.AppendLine("uid: changelog"); content.AppendLine("---"); content.AppendLine(""); content.AppendLine("# ChangeLog"); content.AppendLine(""); if (preview) content.AppendLine($"* @changelog.v{context.VersionHistory.CurrentVersion}"); foreach (var version in context.VersionHistory.StableVersions.Reverse()) content.AppendLine($"* @changelog.v{version}"); content.AppendLine("* @changelog.full"); context.GenerateFile(DocfxDirectory.CombineWithFilePath("index.md"), content); } private void GenerateFull() { var content = new StringBuilder(); content.AppendLine("---"); content.AppendLine("uid: changelog.full"); content.AppendLine("---"); content.AppendLine(""); content.AppendLine("# Full ChangeLog"); content.AppendLine(""); if (preview) content.AppendLine( $"[!include[v{context.VersionHistory.CurrentVersion}](v{context.VersionHistory.CurrentVersion}.md)]"); foreach (var version in context.VersionHistory.StableVersions.Reverse()) content.AppendLine($"[!include[v{version}](v{version}.md)]"); context.GenerateFile(DocfxDirectory.CombineWithFilePath("full.md"), content); } private void GenerateToc() { var content = new StringBuilder(); if (preview) { content.AppendLine($"- name: v{context.VersionHistory.CurrentVersion}"); content.AppendLine($" href: v{context.VersionHistory.CurrentVersion}.md"); } foreach (var version in context.VersionHistory.StableVersions.Reverse()) { content.AppendLine($"- name: v{version}"); content.AppendLine($" href: v{version}.md"); } content.AppendLine("- name: Full ChangeLog"); content.AppendLine(" href: full.md"); context.GenerateFile(DocfxDirectory.CombineWithFilePath("toc.yml"), content); } private void EnsureSrcDirectoryExist(bool forceClone = false) { void Log(string message) => context.Information($"[Changelog] {message}"); if (context.DirectoryExists(SrcDirectory) && forceClone) { Log($"Directory '{SrcDirectory}' already exists and forceClean is specified. " + $"Deleting the current directory..."); context.DeleteDirectory( SrcDirectory, new DeleteDirectorySettings { Force = true, Recursive = true }); Log($"Directory '{SrcDirectory}' deleted successfully."); } if (!context.DirectoryExists(SrcDirectory)) { Log($"Cloning branch '{Repo.ChangelogBranch}' from '{Repo.SshGitUrl}' to '{SrcDirectory}'."); context.GitRunner.Clone(SrcDirectory, Repo.SshGitUrl, Repo.ChangelogBranch); Log($"Clone completed: '{Repo.ChangelogBranch}' -> '{SrcDirectory}'."); } else if (forceClone) { Log($"Fetching branch '{Repo.ChangelogBranch}' from '{Repo.SshGitUrl}' to '{SrcDirectory}'."); context.GitRunner.Pull(SrcDirectory); Log($"Fetch completed: '{Repo.ChangelogBranch}' -> '{SrcDirectory}'."); } } } ================================================ FILE: build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogDetailsBuilder.cs ================================================ using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Text; using System.Threading.Tasks; using BenchmarkDotNet.Build.Helpers; using BenchmarkDotNet.Build.Meta; using Cake.Common.Diagnostics; using Cake.Core.IO; using Octokit; namespace BenchmarkDotNet.Build.Runners.Changelog; public static class ChangelogDetailsBuilder { private class Config(string currentVersion, string previousVersion, string lastCommit) { public string CurrentVersion { get; } = currentVersion; public string PreviousVersion { get; } = previousVersion; public string LastCommit { get; } = lastCommit; public void Deconstruct(out string currentMilestone, out string previousMilestone, out string lastCommit) { currentMilestone = CurrentVersion; previousMilestone = PreviousVersion; lastCommit = LastCommit; } } private class MarkdownBuilder { private static IReadOnlyList? allMilestones; private static readonly Dictionary AuthorNames = new(); private readonly Config config; private readonly StringBuilder builder; public static async Task Build(Config config) { return await new MarkdownBuilder(config).Build(); } private MarkdownBuilder(Config config) { this.config = config; builder = new StringBuilder(); } private async Task Build() { var (currentVersion, previousVersion, lastCommit) = config; if (string.IsNullOrEmpty(lastCommit)) lastCommit = $"v{currentVersion}"; var client = Utils.CreateGitHubClient(); if (currentVersion == "_") { var allContributors = await client.Repository.GetAllContributors(Repo.Owner, Repo.Name); builder.AppendLine("# All contributors"); builder.AppendLine(); foreach (var contributor in allContributors) { var user = await client.User.Get(contributor.Login); var name = user?.Name; builder.AppendLine("* " + (string.IsNullOrEmpty(name) ? contributor.ToLink() : contributor.ToLinkWithName(name))); } return builder.ToString(); } if (allMilestones == null) { var milestoneRequest = new MilestoneRequest { State = ItemStateFilter.All }; allMilestones = await client.Issue.Milestone.GetAllForRepository(Repo.Owner, Repo.Name, milestoneRequest); } IReadOnlyList allIssues = Array.Empty(); var targetMilestone = allMilestones.FirstOrDefault(m => m.Title == $"v{currentVersion}"); if (targetMilestone != null) { var issueRequest = new RepositoryIssueRequest { State = ItemStateFilter.Closed, Milestone = targetMilestone.Number.ToString() }; allIssues = await client.Issue.GetAllForRepository(Repo.Owner, Repo.Name, issueRequest); } var issues = allIssues .Where(issue => issue.PullRequest == null) .OrderBy(issue => issue.Number) .ToList(); var pullRequests = allIssues .Where(issue => issue.PullRequest != null) .OrderBy(issue => issue.Number) .ToList(); var compare = await client.Repository.Commit.Compare(Repo.Owner, Repo.Name, $"v{previousVersion}", lastCommit); var commits = compare.Commits; foreach (var contributor in commits.Select(commit => commit.Author)) if (contributor != null && !AuthorNames.ContainsKey(contributor.Login)) { var user = await client.User.Get(contributor.Login); var name = user?.Name; AuthorNames[contributor.Login] = string.IsNullOrWhiteSpace(name) ? contributor.Login : name; } string PresentContributor(GitHubCommit commit) { if (commit.Author != null) return $"{AuthorNames[commit.Author.Login]} ({commit.Author.ToLink()})".Trim(); return commit.Commit.Author.Name; } var contributors = compare.Commits .Select(PresentContributor) .OrderBy(it => it) .Distinct() .ToImmutableList(); var milestoneHtmlUlr = $"https://github.com/{Repo.Owner}/{Repo.Name}/issues?q=milestone:v{currentVersion}"; builder.AppendLine("## Milestone details"); builder.AppendLine(); builder.AppendLine($"In the [v{currentVersion}]({milestoneHtmlUlr}) scope, "); builder.Append(issues.Count + " issues were resolved and "); builder.AppendLine(pullRequests.Count + " pull requests were merged."); builder.AppendLine($"This release includes {commits.Count} commits by {contributors.Count} contributors."); builder.AppendLine(); AppendList("Resolved issues", issues, issue => $"[#{issue.Number}]({issue.HtmlUrl}) {issue.Title.Trim()}{issue.Assignee.ToStr("assignee:")}"); AppendList("Merged pull requests", pullRequests, pr => $"[#{pr.Number}]({pr.HtmlUrl}) {pr.Title.Trim()}{pr.User.ToStr("by")}"); AppendList("Commits", commits, commit => $"{commit.ToLink()} {commit.Commit.ToCommitMessage()}{commit.ToByStr()}"); AppendList("Contributors", contributors, it => it, "Thank you very much!"); return builder.ToString(); } private void AppendList(string title, IReadOnlyList items, Func format, string? conclusion = null) { builder.AppendLine($"## {title} ({items.Count})"); builder.AppendLine(); foreach (var item in items) builder.AppendLine("* " + format(item)); if (!string.IsNullOrWhiteSpace(conclusion)) { builder.AppendLine(); builder.AppendLine(conclusion); } builder.AppendLine(); } } public static void Run(BuildContext context, DirectoryPath path, string currentVersion, string previousVersion, string lastCommit) { try { var config = new Config(currentVersion, previousVersion, lastCommit); var releaseNotes = MarkdownBuilder.Build(config).Result; context.GenerateFile(path.Combine($"v{config.CurrentVersion}.md").FullPath, releaseNotes, true); } catch (Exception e) { context.Error(e.ToString()); } } } ================================================ FILE: build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs ================================================ using System.IO; using System.Linq; using System.Text; using BenchmarkDotNet.Build.Helpers; using BenchmarkDotNet.Build.Meta; using BenchmarkDotNet.Build.Options; using BenchmarkDotNet.Build.Runners.Changelog; using Cake.Common.Diagnostics; using Cake.Common.IO; using Cake.Core.IO; using Cake.FileHelpers; namespace BenchmarkDotNet.Build.Runners; public class DocumentationRunner { private readonly BuildContext context; private readonly ChangelogBuilder changelogBuilder; private readonly DirectoryPath docsGeneratedDirectory; private readonly FilePath docfxJsonFile; private readonly FilePath redirectFile; private readonly FilePath readmeFile; private readonly FilePath rootIndexFile; private readonly FilePath analyzersShippedFile; private readonly FilePath analyzersUnshippedFile; private readonly FilePath analyzersPageFile; public DirectoryPath ChangelogSrcDirectory => changelogBuilder.SrcDirectory; public DocumentationRunner(BuildContext context) { this.context = context; changelogBuilder = new ChangelogBuilder(context); var docsDirectory = context.RootDirectory.Combine("docs"); docsGeneratedDirectory = docsDirectory.Combine("_site"); redirectFile = docsDirectory.Combine("_redirects").CombineWithFilePath("_redirects"); docfxJsonFile = docsDirectory.CombineWithFilePath("docfx.json"); readmeFile = context.RootDirectory.CombineWithFilePath("README.md"); rootIndexFile = docsDirectory.CombineWithFilePath("index.md"); var analyzersDirectory = context.RootDirectory.Combine("src").Combine("BenchmarkDotNet.Analyzers"); analyzersShippedFile = analyzersDirectory.CombineWithFilePath("AnalyzerReleases.Shipped.md"); analyzersUnshippedFile = analyzersDirectory.CombineWithFilePath("AnalyzerReleases.Unshipped.md"); analyzersPageFile = docsDirectory.Combine("articles").CombineWithFilePath("analyzers.md"); } public void MoveAnalyzerRules() { if (new FileInfo(analyzersUnshippedFile.FullPath).Length == 0) { return; } string tempFile = System.IO.Path.GetTempFileName(); using (var writer = new StreamWriter(tempFile)) { writer.WriteLine($"## v{context.VersionHistory.CurrentVersion}"); CopyLines(writer, analyzersUnshippedFile); writer.WriteLine(); writer.WriteLine(); CopyLines(writer, analyzersShippedFile); } File.Delete(analyzersShippedFile.FullPath); File.Move(tempFile, analyzersShippedFile.FullPath); File.WriteAllText(analyzersUnshippedFile.FullPath, string.Empty); } public void Fetch() { EnvVar.GitHubToken.AssertHasValue(); changelogBuilder.Fetch(); } public void Generate() { GenerateAnalyzersPage(); changelogBuilder.Generate(); UpdateReadme(); GenerateIndexMd(); } public void Build() { RunDocfx(); GenerateRedirects(); } private void UpdateReadme() { var content = Utils.ApplyRegex( context.FileReadText(context.ReadmeFile), @"\[(\d+)\+ GitHub projects\]", Repo.GetDependentProjectsNumber().Result.ToString() ); context.GenerateFile(context.ReadmeFile, content, true); } private void RunDocfx() { context.Information($"Running docfx for '{docfxJsonFile}'"); var currentDirectory = Directory.GetCurrentDirectory(); Directory.SetCurrentDirectory(docfxJsonFile.GetDirectory().FullPath); Docfx.Dotnet.DotnetApiCatalog.GenerateManagedReferenceYamlFiles(docfxJsonFile.FullPath).Wait(); Docfx.Docset.Build(docfxJsonFile.FullPath).Wait(); Directory.SetCurrentDirectory(currentDirectory); } private void GenerateIndexMd() { var content = new StringBuilder(); content.AppendLine("---"); content.AppendLine("title: Home"); content.AppendLine("---"); content.Append(context.FileReadText(readmeFile)); context.GenerateFile(rootIndexFile, content); } private void GenerateAnalyzersPage() { context.EnsureDirectoryExists(analyzersPageFile.GetDirectory()); using var writer = new StreamWriter(analyzersPageFile.FullPath); writer.WriteLine($"# Roslyn Analyzers for C#"); writer.WriteLine(); CopyLines(writer, analyzersShippedFile); } private static void CopyLines(StreamWriter writer, FilePath filePath) { using var reader = new StreamReader(filePath.FullPath); while (reader.ReadLine() is { } line) { writer.WriteLine(); writer.Write(line); } } private void GenerateRedirects() { if (!context.FileExists(redirectFile)) { context.Error($"Redirect file '{redirectFile}' does not exist"); return; } context.EnsureDirectoryExists(docsGeneratedDirectory); var redirects = context.FileReadLines(redirectFile) .Select(line => line.Split(' ')) .Select(parts => (source: parts[0], target: parts[1])) .ToList(); foreach (var (source, target) in redirects) { var fileName = source.StartsWith("/") || source.StartsWith("\\") ? source[1..] : source; var fullFilePath = docsGeneratedDirectory.CombineWithFilePath(fileName); var content = $"" + $"" + $"" + $"{target}" + $"" + $"" + $"" + $"" + $""; context.EnsureDirectoryExists(fullFilePath.GetDirectory()); context.GenerateFile(fullFilePath, content); } } } ================================================ FILE: build/BenchmarkDotNet.Build/Runners/GitRunner.cs ================================================ using System; using BenchmarkDotNet.Build.Meta; using Cake.Common; using Cake.Common.Diagnostics; using Cake.Core.IO; using Cake.Git; namespace BenchmarkDotNet.Build.Runners; // Cake.Git 3.0.0 may experience the following issues on macOS: // > Error: System.TypeInitializationException: The type initializer for 'LibGit2Sharp.Core.NativeMethods' threw an exception. // > ---> System.DllNotFoundException: Unable to load shared library 'git2-6777db8' or one of its dependencies. In order to help diagnose loading problems, consider setting the DYLD_PRINT_LIBRARIES environment variable // In order to workaround this problem, we provide command-line fallbacks for all the used commands. public class GitRunner { private BuildContext context; public GitRunner(BuildContext context) { this.context = context; } public void Clone(DirectoryPath workDirectoryPath, string sourceUrl, string branchName) { context.Information($"[GitClone]"); context.Information($" Repo: {sourceUrl}"); context.Information($" Branch: {branchName}"); context.Information($" Path: {workDirectoryPath}"); var settings = new GitCloneSettings { Checkout = true, BranchName = branchName }; RunCommand( () => context.GitClone(sourceUrl, workDirectoryPath, settings), $"clone -b {branchName} {sourceUrl} {workDirectoryPath}"); } public void Pull(DirectoryPath workDirectoryPath) { context.Information($"[GitPull]"); context.Information($" Path: {workDirectoryPath}"); RunCommand( () => context.GitPull(workDirectoryPath, null, null), $"-C {workDirectoryPath} pull"); } public void Tag(string tagName) { context.Information("[GitTag]"); context.Information($" Path: {context.RootDirectory}"); context.Information($" TagName: {tagName}"); RunCommand( () => context.GitTag(context.RootDirectory, tagName), $"tag {tagName}"); } public void BranchMove(string branchName, string target) { context.Information("[GitBranchMove]"); context.Information($" Branch: {branchName}"); context.Information($" Target: {target}"); RunCommand($"branch -f {branchName} {target}"); } public void AddAll(DirectoryPath? repoDirectory = null) { var repoFlag = repoDirectory != null ? $"-C {repoDirectory}" : ""; context.Information("[GitAddAll]"); RunCommand($"{repoFlag} add -A"); } public void Commit(string message, DirectoryPath? repoDirectory = null) { context.Information("[GitCommit]"); context.Information($" Message: {message}"); var repoFlag = repoDirectory != null ? $"-C {repoDirectory}" : ""; RunCommand($"{repoFlag} " + $"-c user.name=\"{Repo.MaintainerAuthorName}\" " + $"-c user.email=\"{Repo.MaintainerAuthorEmail}\" " + $"commit " + $"--author=\"{Repo.MaintainerAuthorName} <{Repo.MaintainerAuthorEmail}>\" " + $"--all " + $"--message \"{message}\"".Trim()); } public void Push(string target, bool force = false, DirectoryPath? repoDirectory = null) { context.Information("[GitPush]"); context.Information($" Target: {target}"); context.Information($" Force: {force}"); context.RunOnlyInPushMode(() => { var forceFlag = force ? " --force" : ""; var repoFlag = repoDirectory != null ? $"-C {repoDirectory}" : ""; RunCommand($"{repoFlag} push origin {target}{forceFlag}".Trim()); }); } private void RunCommand(string commandLineArgs) => RunCommand(null, commandLineArgs); private void RunCommand(Action? call, string commandLineArgs) { try { if (call == null) throw new NotImplementedException(); call(); context.Information(" Success"); } catch (Exception e) { if (e is not NotImplementedException) context.Warning($" Failed to perform operation via API ({e.Message})"); try { context.Information($" Run command in terminal: 'git {commandLineArgs}'"); context.StartProcess("git", new ProcessSettings { Arguments = commandLineArgs, WorkingDirectory = context.RootDirectory }); context.Information(" Success"); } catch (Exception e2) { throw new Exception($"Failed to run 'git ${commandLineArgs}'", e2); } } } } ================================================ FILE: build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs ================================================ using System; using System.Linq; using System.Text; using System.Threading.Tasks; using BenchmarkDotNet.Build.Helpers; using BenchmarkDotNet.Build.Meta; using BenchmarkDotNet.Build.Options; using Cake.Common.Diagnostics; using Cake.Common.IO; using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.NuGet.Push; using Cake.FileHelpers; using Octokit; namespace BenchmarkDotNet.Build.Runners; public class ReleaseRunner { private readonly BuildContext context; public ReleaseRunner(BuildContext context) { this.context = context; } public void VersionIncrement() { var currentVersion = context.VersionHistory.CurrentVersion; var nextVersion = KnownOptions.NextVersion.Resolve(context); if (nextVersion == "") { var version = Version.Parse(currentVersion); nextVersion = $"{version.Major}.{version.Minor}.{version.Build + 1}"; context.Information($"Evaluated NextVersion: {nextVersion}"); } UpdateVersionsTxt(nextVersion); UpdateCommonPropsVersion(nextVersion); UpdateTemplateVersion(nextVersion); } public void Run() { KnownOptions.Stable.AssertTrue(context); EnvVar.GitHubToken.AssertHasValue(); if (KnownOptions.Push.Resolve(context)) EnvVar.NuGetToken.AssertHasValue(); else EnvVar.NuGetToken.SetEmpty(); var currentVersion = context.VersionHistory.CurrentVersion; var tag = "v" + currentVersion; var nextVersion = KnownOptions.NextVersion.Resolve(context); if (nextVersion == "") { var version = Version.Parse(currentVersion); nextVersion = $"{version.Major}.{version.Minor}.{version.Build + 1}"; context.Information($"Evaluated NextVersion: {nextVersion}"); } context.GitRunner.Tag(tag); // Upgrade current version and commit changes UpdateVersionsTxt(nextVersion); UpdateCommonPropsVersion(nextVersion); context.Information($"Building {context.TemplatesTestsProjectFile}"); context.BuildRunner.BuildProjectSilent(context.TemplatesTestsProjectFile); context.GitRunner.Commit($"Set next BenchmarkDotNet version: {nextVersion}"); UpdateMilestones(nextVersion).Wait(); context.GitRunner.BranchMove(Repo.DocsStableBranch, "HEAD"); context.GitRunner.Push(Repo.MasterBranch); context.GitRunner.Push(Repo.DocsStableBranch, true, context.DocumentationRunner.ChangelogSrcDirectory); context.GitRunner.Push(tag); PushNupkg(); PublishGitHubRelease(); } private void UpdateVersionsTxt(string versionToAppend) { var content = context.FileReadText(context.VersionsFile).Trim(); context.GenerateFile(context.VersionsFile, $"{content}\n{versionToAppend}"); context.Information($"Added v{versionToAppend} to {context.VersionsFile}"); } private void UpdateCommonPropsVersion(string newCurrentVersion) { var content = Utils.ApplyRegex( context.FileReadText(context.CommonPropsFile), @"([\d\.]+)", newCurrentVersion); context.GenerateFile(context.CommonPropsFile, content); context.Information($"Set v{newCurrentVersion} as VersionPrefix in {context.CommonPropsFile}"); } private void UpdateTemplateVersion(string newCurrentVersion) { context.BuildRunner.BuildProjectSilent(context.TemplatesTestsProjectFile); context.Information($"Set v{newCurrentVersion} in templates from {context.TemplatesTestsProjectFile}"); } private async Task UpdateMilestones(string nextVersion) { var currentVersion = context.VersionHistory.CurrentVersion; var client = Utils.CreateGitHubClient(); var allMilestones = await client.Issue.Milestone.GetAllForRepository(Repo.Owner, Repo.Name); var currentMilestone = allMilestones.First(milestone => milestone.Title == $"v{currentVersion}"); context.Information($"[GitHub] Close milestone v{currentVersion}"); context.RunOnlyInPushMode(() => { var milestoneUpdate = new MilestoneUpdate { State = ItemState.Closed, DueOn = DateTimeOffset.Now }; client.Issue.Milestone.Update(Repo.Owner, Repo.Name, currentMilestone.Number, milestoneUpdate).Wait(); }); context.Information($"[GitHub] Create milestone v{nextVersion}"); context.RunOnlyInPushMode(() => { client.Issue.Milestone.Create(Repo.Owner, Repo.Name, new NewMilestone($"v{nextVersion}")).Wait(); }); } private void PushNupkg() { var nuGetToken = EnvVar.NuGetToken.GetValue(); var files = context .GetFiles(context.ArtifactsDirectory.CombineWithFilePath("*.nupkg").FullPath) .OrderBy(file => file.FullPath); var settings = new DotNetNuGetPushSettings { ApiKey = nuGetToken, SymbolApiKey = nuGetToken, Source = "https://api.nuget.org/v3/index.json" }; foreach (var file in files) { context.Information($"Push: {file}"); context.RunOnlyInPushMode(() => context.DotNetNuGetPush(file, settings)); } } private void PublishGitHubRelease() { var version = context.VersionHistory.CurrentVersion; var tag = $"v{version}"; var notesFile = context.DocumentationRunner .ChangelogSrcDirectory .Combine("header") .CombineWithFilePath($"{tag}.md"); var notes = $"Full changelog: https://benchmarkdotnet.org/changelog/{tag}.html\n\n" + PreprocessMarkdown(context.FileReadText(notesFile)); context.Information($"[GitHub] Creating release '{version}'"); var client = Utils.CreateGitHubClient(); context.RunOnlyInPushMode(() => { client.Repository.Release.Create(Repo.Owner, Repo.Name, new NewRelease(tag) { Name = version, Draft = false, Prerelease = false, GenerateReleaseNotes = false, DiscussionCategoryName = "Announcements", Body = notes }).Wait(); context.Information(" Success"); }); } private static string PreprocessMarkdown(string content) { var lines = content.Split("\n"); var newContent = new StringBuilder(); for (var i = 0; i < lines.Length; i++) { newContent.Append(lines[i]); if (i == lines.Length - 1) continue; if (!lines[i].EndsWith(" ") && lines[i + 1].StartsWith(" ")) continue; newContent.Append("\n"); } return newContent.ToString(); } } ================================================ FILE: build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs ================================================ using BenchmarkDotNet.Build.Helpers; using Cake.Common; using Cake.Common.Diagnostics; using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.Test; using Cake.Core.IO; using System.Runtime.InteropServices; namespace BenchmarkDotNet.Build.Runners; public class UnitTestRunner(BuildContext context) { private FilePath UnitTestsProjectFile { get; } = context.RootDirectory .Combine("tests") .Combine("BenchmarkDotNet.Tests") .CombineWithFilePath("BenchmarkDotNet.Tests.csproj"); private FilePath ExporterTestsProjectFile { get; } = context.RootDirectory .Combine("tests") .Combine("BenchmarkDotNet.Exporters.Plotting.Tests") .CombineWithFilePath("BenchmarkDotNet.Exporters.Plotting.Tests.csproj"); private FilePath AnalyzerTestsProjectFile { get; } = context.RootDirectory .Combine("tests") .Combine("BenchmarkDotNet.Analyzers.Tests") .CombineWithFilePath("BenchmarkDotNet.Analyzers.Tests.csproj"); private FilePath IntegrationTestsProjectFile { get; } = context.RootDirectory .Combine("tests") .Combine("BenchmarkDotNet.IntegrationTests") .CombineWithFilePath("BenchmarkDotNet.IntegrationTests.csproj"); private DirectoryPath TestOutputDirectory { get; } = context.RootDirectory .Combine("TestResults"); private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tfm) { var settings = new DotNetTestSettings { Configuration = context.BuildConfiguration, Framework = tfm, NoBuild = true, NoRestore = true, Loggers = new[] { "trx", $"trx;LogFileName={logFile.FullPath}", "console;verbosity=detailed" }, EnvironmentVariables = { ["Platform"] = "" // force the tool to not look for the .dll in platform-specific directory } }; return settings; } private void RunTests(FilePath projectFile, string alias, string tfm) { var os = Utils.GetOs(); var arch = RuntimeInformation.OSArchitecture.ToString().ToLower(); var trxFileName = $"{os}({arch})-{alias}-{tfm}.trx"; var trxFile = TestOutputDirectory.CombineWithFilePath(trxFileName); var settings = GetTestSettingsParameters(trxFile, tfm); context.Information($"Run tests for {projectFile} ({tfm}), result file: '{trxFile}'"); context.DotNetTest(projectFile.FullPath, settings); } private void RunUnitTests(string tfm) { RunTests(UnitTestsProjectFile, "unit", tfm); RunTests(ExporterTestsProjectFile, "exporters", tfm); } public void RunUnitTests() { string[] targetFrameworks = context.IsRunningOnWindows() ? ["net462", "net8.0"] : ["net8.0"]; foreach (var targetFramework in targetFrameworks) RunUnitTests(targetFramework); } public void RunAnalyzerTests() { string[] targetFrameworks = context.IsRunningOnWindows() ? ["net462", "net8.0"] : ["net8.0"]; foreach (var targetFramework in targetFrameworks) RunTests(AnalyzerTestsProjectFile, "analyzer", targetFramework); } public void RunInTests(string tfm) => RunTests(IntegrationTestsProjectFile, "integration", tfm); } ================================================ FILE: build/CodingStyle.ruleset ================================================  ================================================ FILE: build/build.bat ================================================ @ECHO OFF PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0build.ps1' %*; exit $LASTEXITCODE" ================================================ FILE: build/build.ps1 ================================================ #!/usr/bin/env pwsh $DotNetInstallerUri = 'https://dot.net/v1/dotnet-install.ps1'; $BuildPath = Split-Path $MyInvocation.MyCommand.Path -Parent $PSScriptRoot = Split-Path $PSScriptRoot -Parent if ($PSVersionTable.PSEdition -ne 'Core') { # Attempt to set highest encryption available for SecurityProtocol. # PowerShell will not set this by default (until maybe .NET 4.6.x). This # will typically produce a message for PowerShell v2 (just an info # message though) try { # Set TLS 1.2 (3072), then TLS 1.1 (768), then TLS 1.0 (192), finally SSL 3.0 (48) # Use integers because the enumeration values for TLS 1.2 and TLS 1.1 won't # exist in .NET 4.0, even though they are addressable if .NET 4.5+ is # installed (.NET 4.5 is an in-place upgrade). [System.Net.ServicePointManager]::SecurityProtocol = 3072 -bor 768 -bor 192 -bor 48 } catch { Write-Output 'Unable to set PowerShell to use TLS 1.2 and TLS 1.1 due to old .NET Framework installed. If you see underlying connection closed or trust errors, you may need to upgrade to .NET Framework 4.5+ and PowerShell v3' } } ########################################################################### # INSTALL .NET CORE CLI ########################################################################### $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 Function Remove-PathVariable([string]$VariableToRemove) { $SplitChar = ';' if ($IsMacOS -or $IsLinux) { $SplitChar = ':' } $path = [Environment]::GetEnvironmentVariable("PATH", "User") if ($path -ne $null) { $newItems = $path.Split($SplitChar, [StringSplitOptions]::RemoveEmptyEntries) | Where-Object { "$($_)" -inotlike $VariableToRemove } [Environment]::SetEnvironmentVariable("PATH", [System.String]::Join($SplitChar, $newItems), "User") } $path = [Environment]::GetEnvironmentVariable("PATH", "Process") if ($path -ne $null) { $newItems = $path.Split($SplitChar, [StringSplitOptions]::RemoveEmptyEntries) | Where-Object { "$($_)" -inotlike $VariableToRemove } [Environment]::SetEnvironmentVariable("PATH", [System.String]::Join($SplitChar, $newItems), "Process") } } $InstallPath = Join-Path $PSScriptRoot ".dotnet" $SdkPath = Join-Path $BuildPath "sdk" $GlobalJsonPath = Join-Path $SdkPath "global.json" if (!(Test-Path $InstallPath)) { New-Item -Path $InstallPath -ItemType Directory -Force | Out-Null; $ScriptPath = Join-Path $InstallPath 'dotnet-install.ps1' (New-Object System.Net.WebClient).DownloadFile($DotNetInstallerUri, $ScriptPath); & $ScriptPath -JSonFile $GlobalJsonPath -InstallDir $InstallPath; # Install .NET 8 SDK & $ScriptPath -Channel 8.0 -InstallDir $InstallPath -NoPath; } Remove-PathVariable "$InstallPath" $env:PATH = "$InstallPath;$env:PATH" $env:DOTNET_ROOT=$InstallPath ########################################################################### # RUN BUILD SCRIPT ########################################################################### & dotnet run --configuration Release --project build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj -- $args exit $LASTEXITCODE; ================================================ FILE: build/build.sh ================================================ #!/usr/bin/env bash # Define variables PROJECT_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd ) ########################################################################### # INSTALL .NET CORE CLI ########################################################################### export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 export DOTNET_CLI_TELEMETRY_OPTOUT=1 export DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 if [ ! -d "$PROJECT_ROOT/.dotnet" ]; then mkdir "$PROJECT_ROOT/.dotnet" curl -Lsfo "$PROJECT_ROOT/.dotnet/dotnet-install.sh" https://dot.net/v1/dotnet-install.sh bash "$PROJECT_ROOT/.dotnet/dotnet-install.sh" --jsonfile ./build/sdk/global.json --install-dir .dotnet --no-path # Install .NET 8 SDK bash "$PROJECT_ROOT/.dotnet/dotnet-install.sh" --channel 8.0 --install-dir .dotnet --no-path fi export PATH="$PROJECT_ROOT/.dotnet":$PATH export DOTNET_ROOT="$PROJECT_ROOT/.dotnet" ########################################################################### # RUN BUILD SCRIPT ########################################################################### dotnet run --configuration Release --project "$PROJECT_ROOT/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj" -- "$@" ================================================ FILE: build/cSpell.json ================================================ { "version": "0.2", "language": "en", "words": [ "Alloc", "analyse", "analyser", "Analysers", "Autofac", "bitness", "corlib", "Cygwin", "Diagnoser", "diagnosers", "diagsession", "disassemblers", "disassm", "Jits", "Jitting", "LINQ", "microbenchmarking", "microbenchmarks", "Mispredict", "Mispredictions", "msbuild", "Multimodal", "multimodality", "netcoreapp", "powerplans", "Pseudocode", "runtimes", "Serilog", "slnx", "vsprofiler", "vstest", "Tailcall", "toolchains", "unmanaged" ], "ignoreWords": [ "Akinshin", "Andrey", "Cassell", "Expecto", "Jint", "JITted", "LoongArch64", "macrobenchmark", "MediatR", "Nagórski's", "Newtonsoft", "NodaTime", "Npgsql", "Sitnik's", "Sitnik", "Stepanov", "Yegor", "Wojciech", "Avalonia", "Gitter" ], "patterns": [ { "name": "Markdown links", "pattern": "\\((.*)\\)", "description": "" }, { "name": "Markdown code blocks", "pattern": "/^(\\s*`{3,}).*[\\s\\S]*?^\\1/gmx", "description": "Taken from the cSpell example at https://cspell.org/configuration/patterns/#verbose-regular-expressions" }, { "name": "Inline code blocks", "pattern": "\\`([^\\`\\r\\n]+?)\\`", "description": "https://stackoverflow.com/questions/41274241/how-to-capture-inline-markdown-code-but-not-a-markdown-code-fence-with-regex" }, { "name": "Link contents", "pattern": "\\", "description": "" }, { "name": "Snippet references", "pattern": "-- snippet:(.*)", "description": "" }, { "name": "Snippet references 2", "pattern": "\\<\\[sample:(.*)", "description": "another kind of snippet reference" }, { "name": "Multi-line code blocks", "pattern": "/^\\s*```[\\s\\S]*?^\\s*```/gm" }, { "name": "HTML Tags", "pattern": "<[^>]*>", "description": "Reference: https://stackoverflow.com/questions/11229831/regular-expression-to-remove-html-tags-from-a-string" } ], "ignoreRegExpList": [ "Markdown links", "Markdown code blocks", "Inline code blocks", "Link contents", "Snippet references", "Snippet references 2", "Multi-line code blocks", "HTML Tags" ], "ignorePaths": [ "docs/_changelog/**/*.md", "docs/articles/team.md" ] } ================================================ FILE: build/common.props ================================================ BenchmarkDotNet Powerful .NET library for benchmarking .NET Foundation and contributors en-US .NET Foundation and contributors benchmark;benchmarking;performance package-icon.png https://github.com/dotnet/BenchmarkDotNet MIT git https://github.com/dotnet/BenchmarkDotNet true true false false true True false $(MSBuildThisFileDirectory)CodingStyle.ruleset true NU1900 enable true $(MSBuildThisFileDirectory)strongNameKey.snk true true 14.0 0.16.0 develop ci $(VersionPrefix) $(VersionPrefix) $(Version) $(Version) -3 ================================================ FILE: build/common.targets ================================================ ================================================ FILE: build/sdk/global.json ================================================ { "sdk": { "version": "10.0.103", "rollForward": "disable" } } ================================================ FILE: build/versions.txt ================================================ 6eda98ab1e83a0d185d09ff8b24c795711af8db1 0.7.0 0.7.1 0.7.2 0.7.3 0.7.4 0.7.5 0.7.6 0.7.7 0.7.8 0.8.0 0.8.1 0.8.2 0.9.0 0.9.1 0.9.2 0.9.3 0.9.4 0.9.5 0.9.6 0.9.7 0.9.8 0.9.9 0.10.0 0.10.1 0.10.2 0.10.3 0.10.4 0.10.5 0.10.6 0.10.7 0.10.8 0.10.9 0.10.10 0.10.11 0.10.12 0.10.13 0.10.14 0.11.0 0.11.1 0.11.2 0.11.3 0.11.4 0.11.5 0.12.0 0.12.1 0.13.0 0.13.1 0.13.2 0.13.3 0.13.4 0.13.5 0.13.6 0.13.7 0.13.8 0.13.9 0.13.10 0.13.11 0.13.12 0.14.0 0.15.0 0.15.1 0.15.2 0.15.3 0.15.4 0.15.5 0.15.6 0.15.7 0.15.8 0.16.0 ================================================ FILE: build.cmd ================================================ :<<"::CMDLITERAL" @CALL build\build.bat %* @GOTO :EOF ::CMDLITERAL "$(cd "$(dirname "$0")"; pwd)/build/build.sh" "$@" ================================================ FILE: docs/.gitignore ================================================ ############### # folder # ############### /**/DROP/ /**/TEMP/ /**/packages/ /**/bin/ /**/obj/ _site _exported_templates docfx-bin api/* !api/index.md /index.md _changelog changelog # Autogenerated by `build.cmd docs-generate` articles/analyzers.md ================================================ FILE: docs/_redirects/_redirects ================================================ /Advancedfeatures.htm /articles/overview.html /HowItWorks.htm /Internals/HowItWorks.htm /RulesOfBenchmarking.htm /Guides/GoodPractices.htm /Overview.htm /articles/overview.html /Configs.htm /articles/configs/configs.html /HowToRun.htm /Guides/HowToRun.htm /index.htm /index.html /Contributing.htm /articles/contributing/building.html /Team.htm /articles/team.html /Internals.htm /articles/guides/how-it-works.html /Guides.htm /articles/guides/getting-started.html /FAQ.htm /articles/faq.html /NuGet.htm /Guides/NuGet.htm /license.htm /articles/license.html /GettingStarted.htm /Guides/GettingStarted.htm /Advanced/Baseline.htm /articles/features/baselines.html /Advanced/STAThread.htm /articles/samples/IntroStaThread.html /Advanced/EnvironmentVariables.htm /articles/samples/IntroEnvVars.html /Advanced/Params.htm /articles/features/parameterization.html /Advanced/CustomizingMono.htm /articles/guides/customizing-runtime.html /Advanced/CustomMonoPath.htm /articles/guides/customizing-runtime.html /Advanced/Arguments.htm /articles/features/parameterization.html /Advanced/Percentiles.htm /articles/features/statistics.html /Advanced/SetupAndCleanup.htm /articles/features/setup-and-cleanup.html /Contributing/Debugging.htm /articles/contributing/debugging.html /Contributing/Disassembler.htm /articles/contributing/disassembler.html /Contributing/RunningTests.htm /articles/contributing/running-tests.html /Contributing/Miscellaneous.htm /articles/contributing/miscellaneous.html /Contributing/Development.htm /articles/contributing/development.html /Contributing/Building.htm /articles/contributing/building.html /Internals/HowItWorks.htm /articles/guides/how-it-works.html /Guides/GoodPractices.htm /articles/guides/good-practices.html /Guides/HowToRun.htm /articles/guides/how-to-run.html /Guides/NuGet.htm /articles/guides/nuget.html /Guides/ChoosingRunStrategy.htm /articles/guides/choosing-run-strategy.html /Guides/GettingStarted.htm /articles/guides/getting-started.html /Configs/Columns.htm /articles/configs/columns.html /Configs/Configs.htm /articles/configs/configs.html /Configs/Diagnosers.htm /articles/configs/diagnosers.html /Configs/Exporters.htm /articles/configs/exporters.html /Configs/Filters.htm /articles/configs/filters.html /Configs/OrderProvider.htm /articles/configs/orderers.html /Configs/Analyzers.htm /articles/configs/analysers.html /Configs/Toolchains.htm /articles/configs/toolchains.html /Configs/Loggers.htm /articles/configs/loggers.html /Configs/Validators.htm /articles/configs/validators.html /Configs/Jobs.htm /articles/configs/jobs.html /articles/samples/IntroXamarin.html /articles/samples/IntroMaui.html ================================================ FILE: docs/api/index.md ================================================ # BenchmarkDotNet API Reference ================================================ FILE: docs/articles/configs/analysers.md ================================================ --- uid: docs.analysers name: Analysers --- # Analysers An **analyser** can analyse summary of your benchmarks and produce some useful warnings. For example, `EnvironmentAnalyser` warns you, if you build your application in the DEBUG mode or run it with an attached debugger. ================================================ FILE: docs/articles/configs/columns.md ================================================ --- uid: docs.columns name: Columns --- # Columns A *column* is a column in the summary table. ## Default columns In this section, default columns (which be added to the Summary table by default) are presented. Some of columns are optional, i.e. they can be omitted (it depends on the measurements from the summary). ### Target There are 3 default columns which describes the target benchmark: `Namespace`, `Type`, `Method`. `Namespace` and `Type` will be omitted when all the benchmarks have the same namespace or type name. `Method` column always be a part of the summary table. ### Job There are many different job characteristics, but the summary includes only characteristics which has at least one non-default value. ### Statistics There are also a lot of different statistics which can be considered. It will be really hard to analyse the summary table, if all of the available statistics will be shown. Fortunately, BenchmarkDotNet has some heuristics for statistics columns and shows only important columns. For example, if all of the standard deviations are zero (we run our benchmarks against Dry job), this column will be omitted. The standard error will be shown only for cases when we are failed to achieve required accuracy level. Only `Mean` will be always shown. If the distribution looks strange, BenchmarkDotNet could also print additional columns like `Median` or `P95` (95th percentile). If you need specific statistics, you always could add them manually. ### Params If you have `params`, the corresponded columns will be automatically added. ### Diagnosers If you turned on diagnosers which providers additional columns, they will be also included in the summary page. ## Custom columns Of course, you can define own custom columns and use it everywhere. Here is the definition of `TagColumn`: [!code-csharp[IntroTagColumn.cs](../../../src/BenchmarkDotNet/Columns/TagColumn.cs)] --- [!include[IntroTagColumn](../samples/IntroTagColumn.md)] ================================================ FILE: docs/articles/configs/configoptions.md ================================================ --- #cspell:ignore configoptions uid: docs.configoptions name: Configoptions --- # Config Options The config options let you customize some behavior of BenchmarkDotNet - mainly regarding the output. Available config options are: * `ConfigOptions.Default` - No configuration option is set - this is the default. * `ConfigOptions.KeepBenchmarkFiles` - All auto-generated files should be kept after running the benchmarks (by default they are removed). * `ConfigOptions.JoinSummary` - All benchmarks results should be joined into a single summary (by default we have a summary per type). * `ConfigOptions.StopOnFirstError` - Benchmarking should be stopped after the first error (by default it's not). * `ConfigOptions.DisableOptimizationsValidator` - Mandatory optimizations validator should be entirely turned off. * `ConfigOptions.DontOverwriteResults` - The exported result files should not be overwritten (by default they are overwritten). * `ConfigOptions.DisableLogFile` - Disables the log file written on disk. All of these options could be combined and are available as CLI (Comand Line Interface) option (except `DisableOptimizationsValidator`), see [Console Arguments](xref:docs.console-args) for further information how to use the CLI. Any of these options could be used either in `object style config` or `fluent style config`: ### Object style config ```cs public class Config : ManualConfig { public Config() { // Using the WithOptions() factory method: this.WithOptions(ConfigOptions.JoinSummary) .WithOptions(ConfigOptions.DisableLogFile); // Or (The ConfigOptions Enum is defined as a BitField) this.WithOptions(ConfigOptions.JoinSummary | ConfigOptions.DisableLogFile); } } ``` ### Fluent style config ```cs public static void Run() { BenchmarkRunner .Run( ManualConfig .Create(DefaultConfig.Instance) .WithOptions(ConfigOptions.JoinSummary) .WithOptions(ConfigOptions.DisableLogFile) // or .WithOptions(ConfigOptions.JoinSummary | ConfigOptions.DisableLogFile)); } ``` ================================================ FILE: docs/articles/configs/configs.md ================================================ --- uid: docs.configs name: Configs --- # Configs Config is a set of so called `jobs`, `columns`, `exporters`, `loggers`, `diagnosers`, `analysers`, `validators` that help you to build your benchmark. ## Built-in configuration There are two built-in ways to set your config: ### Object style ```cs [Config(typeof(Config))] public class MyClassWithBenchmarks { private class Config : ManualConfig { public Config() { AddJob(new Job1(), new Job2()); AddColumn(new Column1(), new Column2()); AddColumnProvider(new ColumnProvider1(), new ColumnProvider2()); AddExporter(new Exporter1(), new Exporter2()); AddLogger(new Logger1(), new Logger2()); AddDiagnoser(new Diagnoser1(), new Diagnoser2()); AddAnalyser(new Analyser1(), new Analyser2()); AddValidator(new Validator2(),new Validator2()); AddHardwareCounters(HardwareCounter enum1, HardwareCounter enum2); AddFilter(new Filter1(), new Filter2()); AddLogicalGroupRules(BenchmarkLogicalGroupRule enum1, BenchmarkLogicalGroupRule enum2); } } [Benchmark] public void Benchmark1() { } [Benchmark] public void Benchmark2() { } } ``` --- [!include[IntroConfigSource](../samples/IntroConfigSource.md)] [!include[IntroConfigUnion](../samples/IntroConfigUnion.md)] [!include[IntroFluentConfigBuilder](../samples/IntroFluentConfigBuilder.md)] ================================================ FILE: docs/articles/configs/diagnosers.md ================================================ --- uid: docs.diagnosers name: Diagnosers --- # Diagnosers A **diagnoser** can attach to your benchmark and get some useful info. The current Diagnosers are: - GC and Memory Allocation (`MemoryDiagnoser`) which is cross platform, built-in and **is not enabled by default anymore**. Please see Adam Sitnik's [blog post](https://adamsitnik.com/the-new-Memory-Diagnoser/) for all the details. - JIT Stats Diagnoser. You can find this diagnoser in a separate package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) - JIT Inlining Events (`InliningDiagnoser`). You can find this diagnoser in a separate package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) - JIT Tail Call Events (`TailCallDiagnoser`). You can find this diagnoser as well as the (`InliningDiagnoser`) in a separate package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) Please see [this post](https://georgeplotnikov.github.io/articles/tale-tail-call-dotnet) for all the details. - Hardware Counter Diagnoser. You can find this diagnoser in a separate package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/). Please see Adam Sitnik's [blog post](https://adamsitnik.com/Hardware-Counters-Diagnoser/) for all the details. - Disassembly Diagnoser. It allows you to disassemble the benchmarked code to asm, IL and C#/F#. Please see Adam Sitnik's [blog post](https://adamsitnik.com/Disassembly-Diagnoser/) for all the details. - ETW Profiler (`EtwProfiler`). It allows you to not only benchmark, but also profile the code. It's using TraceEvent, which internally uses ETW and exports all the information to a trace file. The trace file contains all of the stack traces captured by the profiler, PDBs to resolve symbols for both native and managed code and captured GC, JIT and CLR events. Please use one of the free tools: PerfView or Windows Performance Analyzer to analyze and visualize the data from trace file. You can find this diagnoser in a separate package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) Please see Adam Sitnik's [blog post](https://adamsitnik.com/ETW-Profiler/) for all the details. - Concurrency Visualizer Profiler (`ConcurrencyVisualizerProfiler`) It uses `EtwProfiler` to profile the code using ETW and create not only `.etl` file but also a CVTrace file which can be opened by Concurrency Visualizer plugin from Visual Studio. Please see Adam Sitnik's [blog post](https://adamsitnik.com/ConcurrencyVisualizer-Profiler/) for all the details. - Native Memory Profiler (`NativeMemoryProfiler`) It uses `EtwProfiler` to profile the code using ETW and adds the extra columns `Allocated native memory` and `Native memory leak`. Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) for all the details. - Event Pipe Profiler (`EventPipeProfiler`). It is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. - Threading Diagnoser (`ThreadingDiagnoser`) - .NET Core 3.0+ diagnoser that reports some Threading statistics. - Exception Diagnoser (`ExceptionDiagnoser`) - a diagnoser that reports the frequency of exceptions thrown during the operation. ## Usage Below is a sample output from the `GC and Memory Allocation` diagnoser, note the extra columns on the right-hand side ("Gen 0", "Gen 1", "Gen 2" and "Allocated"): ``` Method | Mean | StdErr | Median | Gen 0 | Allocated | ----------------- |------------ |----------- |------------ |------- |---------- | 'new byte[10kB]' | 884.4896 ns | 46.3528 ns | 776.4237 ns | 0.1183 | 10 kB | ``` A config example: ```cs private class Config : ManualConfig { public Config() { AddDiagnoser(MemoryDiagnoser.Default); AddDiagnoser(new InliningDiagnoser()); AddDiagnoser(new EtwProfiler()); AddDiagnoser(ThreadingDiagnoser.Default); AddDiagnoser(ExceptionDiagnoser.Default); } } ``` You can also use one of the following attributes (apply it on a class that contains Benchmarks): ```cs [MemoryDiagnoser] [InliningDiagnoser] [TailCallDiagnoser] [EtwProfiler] [ConcurrencyVisualizerProfiler] [NativeMemoryProfiler] [ThreadingDiagnoser] [ExceptionDiagnoser] ``` In BenchmarkDotNet, 1kB = 1024B, 1MB = 1024kB, and so on. The column Gen X means number of GC collections per 1000 operations for that generation. ## Restrictions * In order to not affect main results we perform a separate run if any diagnoser is used. That's why it might take more time to execute benchmarks. * MemoryDiagnoser: * In order to get the number of allocated bytes in cross platform way we are using `GC.GetAllocatedBytesForCurrentThread` which recently got [exposed](https://github.com/dotnet/corefx/pull/12489) for netcoreapp1.1. That's why BenchmarkDotNet does not support netcoreapp1.0 from version 0.10.1. * MemoryDiagnoser is `99.5%` accurate about allocated memory when using default settings or Job.ShortRun (or any longer job than it). * Threading Diagnoser: * Works only for .NET Core 3.0+ * HardwareCounters: * Windows 8+ only (we plan to add Unix support in the future) * No Hyper-V (Virtualization) support * Requires running as Admin (ETW Kernel Session) * No `InProcessToolchain` support ([#394](https://github.com/dotnet/BenchmarkDotNet/issues/394)) * EtwProfiler, ConcurrencyVisualizerProfiler and NativeMemoryProfiler: * Windows only * Requires running as Admin (ETW Kernel Session) * No `InProcessToolchain` support ([#394](https://github.com/dotnet/BenchmarkDotNet/issues/394)) * Disassembly Diagnoser: * .NET Core disassembler works only on Windows * Mono disassembler does not support recursive disassembling and produces output without IL and C#. * Indirect calls are not tracked. * To be able to compare different platforms, you need to target AnyCPU `AnyCPU` * To get the corresponding C#/F# code from disassembler you need to configure your project in following way: ```xml pdbonly true ``` --- [!include[IntroHardwareCounters](../samples/IntroHardwareCounters.md)] [!include[IntroDisassemblyRyuJit](../samples/IntroDisassemblyRyuJit.md)] [!include[IntroDisassembly](../samples/IntroDisassembly.md)] [!include[IntroDisassemblyAllJits](../samples/IntroDisassemblyAllJits.md)] [!include[IntroDisassemblyDry](../samples/IntroDisassemblyDry.md)] [!include[IntroTailcall](../samples/IntroTailcall.md)] [!include[IntroJitStatsDiagnoser](../samples/IntroJitStatsDiagnoser.md)] [!include[IntroNativeMemory](../samples/IntroNativeMemory.md)] [!include[IntroThreadingDiagnoser](../samples/IntroThreadingDiagnoser.md)] [!include[IntroExceptionDiagnoser](../samples/IntroExceptionDiagnoser.md)] ================================================ FILE: docs/articles/configs/exporters.md ================================================ --- uid: docs.exporters name: Exporters --- # Exporters An *exporter* allows you to export results of your benchmark in different formats. By default, files with results will be located in `.\BenchmarkDotNet.Artifacts\results` directory, but this can be changed via the `ArtifactsPath` property in the `IConfig`. Default exporters are: csv, html and markdown. --- [!include[IntroExport](../samples/IntroExport.md)] [!include[IntroExportJson](../samples/IntroExportJson.md)] [!include[IntroExportXml](../samples/IntroExportXml.md)] ## Plots [You can install R](https://www.r-project.org/) to automatically get nice plots of your benchmark results. First, make sure `Rscript.exe` or `Rscript` is in your path, or define an R_HOME environment variable pointing to the R installation directory. _eg: If `Rscript` is located in `/path/to/R/R-1.2.3/bin/Rscript`, then `R_HOME` must point to `/path/to/R/R-1.2.3/`, it **should not** point to `/path/to/R/R-1.2.3/bin`_ Use `RPlotExporter.Default` and `CsvMeasurementsExporter.Default` in your config, and the `BuildPlots.R` script in your bin directory will take care of the rest. Examples: ``` -barplot.png -boxplot.png --density.png --facetTimeline.png --facetTimelineSmooth.png ---timelineSmooth.png ---timelineSmooth.png ``` A config example in C#: ```cs public class Config : ManualConfig { public Config() { Add(CsvMeasurementsExporter.Default); Add(RPlotExporter.Default); } } ``` A config example in F#: ```fs module MyBenchmark open BenchmarkDotNet.Attributes open BenchmarkDotNet.Configs open BenchmarkDotNet.Exporters open BenchmarkDotNet.Exporters.Csv open MyProjectUnderTest type MyConfig() as this = inherit ManualConfig() do this.Add(CsvMeasurementsExporter.Default) this.Add(RPlotExporter.Default) [< MemoryDiagnoser; Config(typeof); RPlotExporter >] type MyPerformanceTests() = let someTestData = getTestDataAsList () [] member __.SomeTestCase() = someTestData |> myFunctionUnderTest ``` ## CSV The CSV file format is often used to graph the output or to analyze the results programmatically. The CSV exporter may be configured to produce sanitized output, where cell values are numerals and their units are predefined. The CSV exporter and other compatible exporters may consume an instance of `ISummaryStyle` that defines how the output should look like: | Property | Remarks | Default | | ------------------- | -------------------------------------------------- | ------- | | PrintUnitsInHeader | If true, units will be displayed in the header row | false | | PrintUnitsInContent | If true, units will be appended to the value | true | | TimeUnit | If null, unit will be automatically selected | null | | SizeUnit | If null, unit will be automatically selected | null | Example of CSV exporter configured to always use microseconds, kilobytes, and to render units only in column headers: ```cs var exporter = new CsvExporter( CsvSeparator.CurrentCulture, new SummaryStyle( cultureInfo: System.Globalization.CultureInfo.CurrentCulture, printUnitsInHeader: true, printUnitsInContent: false, timeUnit: Perfolizer.Horology.TimeUnit.Microsecond, sizeUnit: SizeUnit.KB )); var config = ManualConfig.CreateMinimumViable().AddExporter(exporter); ``` Excerpt from the resulting CSV file: ``` Method,...,Mean [us],Error [us],StdDev [us],Min [us],Max [us],Allocated [KB] Benchmark,...,"37,647.6","32,717.9","21,640.9","11,209.2","59,492.6",1.58 ``` ================================================ FILE: docs/articles/configs/filters.md ================================================ --- uid: docs.filters name: Filters --- # Filters Sometimes you don't want to run all of your benchmarks. In this case, you can *filter* some of them with the help of *filters*. Predefined filters: | Filter Type | Filters benchmarks by | Console argument | Console example | |---------------------|-----------------------------|--------------------|----------------------------------| | GlobFilter | Provided glob pattern | `filter` | --filter *Serializer*.ToStream | | AttributesFilter | Provided attribute names | `attribute` | --attribute STAThread | | AllCategoriesFilter | All Provided category names | `categories` | --allCategories Priority1 CoreFX | | AnyCategoriesFilter | Any provided category names | `anycategories` | --anyCategories Json Xml | | SimpleFilter | Provided lambda predicate | - | | | NameFilter | Provided lambda predicate | - | | | UnionFilter | Logical AND | - | | | DisjunctionFilter | Logical OR | - | | --- [!include[IntroFilters](../samples/IntroFilters.md)] [!include[IntroCategories](../samples/IntroCategories.md)] [!include[IntroJoin](../samples/IntroJoin.md)] ================================================ FILE: docs/articles/configs/jobs.md ================================================ --- uid: docs.jobs name: Jobs --- # Jobs Basically, a *job* describes how to run your benchmark. Practically, it's a set of characteristics which can be specified. You can specify one or several jobs for your benchmarks. ## Characteristics There are several categories of characteristics which you can specify. Let's consider each category in detail. ### Id It's a single string characteristic. It allows to name your job. This name will be used in logs and a part of a folder name with generated files for this job. `Id` doesn't affect benchmark results, but it can be useful for diagnostics. If you don't specify `Id`, random value will be chosen based on other characteristics ### Environment `Environment` specifies an environment of the job. You can specify the following characteristics: * `Platform`: `x86` or `x64` * `Runtime`: * `Clr`: Full .NET Framework (available only on Windows) * `Core`: CoreCLR (x-plat) * `Mono`: Mono (x-plat) * `Jit`: * `LegacyJit` (available only for `Runtime.Clr`) * `RyuJit` (available only for `Runtime.Clr` and `Runtime.Core`) * `Llvm` (available only for `Runtime.Mono`) * `Affinity`: [Affinity](https://msdn.microsoft.com/library/system.diagnostics.process.processoraffinity.aspx) of a benchmark process * `GcMode`: settings of Garbage Collector * `Server`: `true` (Server mode) or `false` (Workstation mode) * `Concurrent`: `true` (Concurrent mode) or `false` (NonConcurrent mode) * `CpuGroups`: Specifies whether garbage collection supports multiple CPU groups * `Force`: Specifies whether the BenchmarkDotNet's benchmark runner forces full garbage collection after each benchmark invocation * `AllowVeryLargeObjects`: On 64-bit platforms, enables arrays that are greater than 2 gigabytes (GB) in total size * `LargeAddressAware`: Specifies that benchmark can handle addresses larger than 2 gigabytes. See also: @BenchmarkDotNet.Samples.IntroLargeAddressAware and [`LARGEADDRESSAWARE`](https://learn.microsoft.com/cpp/build/reference/largeaddressaware-handle-large-addresses) * `false`: Benchmark uses the defaults (64-bit: enabled; 32-bit: disabled). * `true`: Explicitly specify that benchmark can handle addresses larger than 2 gigabytes. * `EnvironmentVariables`: customized environment variables for target benchmark. See also: @BenchmarkDotNet.Samples.IntroEnvVars BenchmarkDotNet will use host process environment characteristics for non specified values. ### Run In this category, you can specify how to benchmark each method. * `RunStrategy`: * `Throughput`: default strategy which allows to get good precision level * `ColdStart`: should be used only for measuring cold start of the application or testing purpose * `Monitoring`: A mode without overhead evaluating, with several target iterations * `LaunchCount`: how many times we should launch process with target benchmark * `WarmupCount`: how many warmup iterations should be performed * `IterationCount`: how many target iterations should be performed (if specified, `BenchmarkDotNet.Jobs.RunMode.MinIterationCount` and `BenchmarkDotNet.Jobs.RunMode.MaxIterationCount` will be ignored) * `IterationTime`: desired time of a single iteration * `UnrollFactor`: how many times the benchmark method will be invoked per one iteration of a generated loop * `InvocationCount`: count of invocation in a single iteration (if specified, `IterationTime` will be ignored), must be a multiple of `UnrollFactor` * `MinIterationCount`: Minimum count of target iterations that should be performed, the default value is 15 * `MaxIterationCount`: Maximum count of target iterations that should be performed, the default value is 100 * `MinWarmupIterationCount`: Minimum count of warmup iterations that should be performed, the default value is 6 * `MaxWarmupIterationCount`: Maximum count of warmup iterations that should be performed, the default value is 50 Usually, you shouldn't specify such characteristics like `LaunchCount`, `WarmupCount`, `IterationCount`, or `IterationTime` because BenchmarkDotNet has a smart algorithm to choose these values automatically based on received measurements. You can specify it for testing purposes or when you are damn sure that you know the right characteristics for your benchmark (when you set `IterationCount` = `20` you should understand why `20` is a good value for your case). ### Accuracy If you want to change the accuracy level, you should use the following characteristics instead of manually adjusting values of `WarmupCount`, `IterationCount`, and so on. * `MaxRelativeError`, `MaxAbsoluteError`: Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error). *In these two characteristics*, the error means half of 99.9% confidence interval. `MaxAbsoluteError` is an absolute `TimeInterval`; doesn't have a default value. `MaxRelativeError` defines max acceptable (`() / Mean`). * `MinIterationTime`: Minimum time of a single iteration. Unlike `Run.IterationTime`, this characteristic specifies only the lower limit. In case of need, BenchmarkDotNet can increase this value. * `MinInvokeCount`: Minimum about of target method invocation. Default value if `4` but you can decrease this value for cases when single invocations takes a lot of time. * `EvaluateOverhead`: if your benchmark method takes nanoseconds, BenchmarkDotNet overhead can significantly affect measurements. If this characteristic is enabled, the overhead will be evaluated and subtracted from the result measurements. Default value is `true`. * `WithOutlierMode`: sometimes you could have outliers in your measurements. Usually these are unexpected outliers which arose because of other processes activities. By default (`OutlierMode.RemoveUpper`), all upper outliers (which is larger than Q3) will be removed from the result measurements. However, some of benchmarks have *expected* outliers. In these situation, you expect that some of invocation can produce outliers measurements (e.g. in case of network activities, cache operations, and so on). If you want to see result statistics with these outliers, you should use `OutlierMode.DontRemove`. If you can also choose `OutlierMode.RemoveLower` (outliers which are smaller than Q1 will be removed) or `OutlierMode.RemoveAll` (all outliers will be removed). See also: @BenchmarkDotNet.Mathematics.OutlierMode * `AnalyzeLaunchVariance`: this characteristic makes sense only if `Run.LaunchCount` is default. If this mode is enabled and, BenchmarkDotNet will try to perform several launches and detect if there is a variance between launches. If this mode is disable, only one launch will be performed. ### Infrastructure Usually, you shouldn't specify any characteristics from this section, it can be used for advanced cases only. * `Toolchain`: a toolchain which generates source code for target benchmark methods, builds it, and executes it. BenchmarkDotNet has own toolchains for .NET, .NET Core, Mono and CoreRT projects. If you want, you can define own toolchain. * `Clock`: a clock which will be used for measurements. BenchmarkDotNet automatically choose the best available clock source, but you can specify own clock source. * `EngineFactory`: a provider for measurement engine which performs all the measurement magic. If you don't trust BenchmarkDotNet, you can define own engine and implement all the measurement stages manually. ## Usage There are several ways to specify a job. ### Object style You can create own jobs directly from the source code via a custom config: ```cs [Config(typeof(Config))] public class MyBenchmarks { private class Config : ManualConfig { public Config() { AddJob( new Job("MySuperJob", RunMode.Dry, EnvironmentMode.RyuJitX64) { Environment = { Runtime = CoreRuntime.Core90 }, Run = { LaunchCount = 5, IterationTime = TimeInterval.Millisecond * 200 }, Accuracy = { MaxRelativeError = 0.01 } }); // The same, using the .With() factory methods: AddJob( Job.Dry .WithPlatform(Platform.X64) .WithJit(Jit.RyuJit) .WithRuntime(CoreRuntime.Core90) .WithLaunchCount(5) .WithIterationTime(TimeInterval.Millisecond * 200) .WithMaxRelativeError(0.01) .WithId("MySuperJob")); } } // Benchmarks } ``` Basically, it's a good idea to start with predefined values (e.g. `EnvironmentMode.RyuJitX64` and `RunMode.Dry` passed as constructor args) and modify rest of the properties using property setters or with help of object initializer syntax. Note that the job cannot be modified after it's added into config. Trying to set a value on property of the frozen job will throw an `InvalidOperationException`. Use the `Job.Frozen` property to determine if the code properties can be altered. If you do want to create a new job based on frozen one (all predefined job values are frozen) you can use the `.With()` extension method ```cs var newJob = Job.Dry.WithPlatform(Platform.X64); ``` or pass the frozen value as a constructor argument ```c# var newJob = new Job(Job.Dry) { Environment = { Platform = Platform.X64 } }; ``` or use the `.Apply()` method on unfrozen job ```c# var newJob = new Job() { Environment = { Platform = Platform.X64 } }.Apply(Job.Dry); ``` in any case the Id property will not be transfered and you must pass it explicitly (using the .ctor id argument or the `.WithId()` extension method). ### Attribute style You can also add new jobs via attributes. Examples: ```cs [DryJob] [MonoJob] [SimpleJob(RuntimeMoniker.Net90)] [SimpleJob(RuntimeMoniker.NetCoreApp31)] [LegacyJitX86Job, LegacyJitX64Job, RyuJitX64Job] [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 5, iterationCount: 5, id: "FastAndDirtyJob")] public class MyBenchmarkClass ``` Note that each of the attributes identifies a separate job, the sample above will result in 8 different jobs, not a single merged job. ### Attribute style for merging jobs Sometimes you want to apply some changes to other jobs, without adding a new job to a config (which results in one extra benchmark run). To do that you can use following predefined job mutator attributes: * `[EvaluateOverhead]` * `[GcConcurrent]` * `[GcForce]` * `[GcServer]` * `[InnerIterationCount]` * `[InvocationCount]` * `[IterationCount]` * `[IterationTime]` * `[MaxAbsoluteError]` * `[MaxIterationCount]` * `[MaxRelativeError]` * `[MinInvokeCount]` * `[MinIterationCount]` * `[MinIterationTime]` * `[Outliers]` * `[ProcessCount]` * `[RunOncePerIteration]` * `[WarmupCount]` * `[MinWarmupCount]` * `[MaxWarmupCount]` So following example: ```cs [ClrJob, CoreJob] [GcServer(true)] public class MyBenchmarkClass ``` Is going to be merged to a config with two jobs: * CoreJob with `GcServer=true` * ClrJob with `GcServer=true` #### Custom attributes You can also create your own custom attributes with your favourite set of jobs. Example: ```cs [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)] public class MySuperJobAttribute : Attribute, IConfigSource { protected MySuperJobAttribute() { var job = new Job("MySuperJob", RunMode.Core); job.Env.Platform = Platform.X64; Config = ManualConfig.CreateEmpty().AddJob(job); } public IConfig Config { get; } } [MySuperJob] public class MyBenchmarks ``` --- [!include[IntroGcMode](../samples/IntroGcMode.md)] ================================================ FILE: docs/articles/configs/loggers.md ================================================ --- uid: docs.loggers name: Loggers --- # Loggers A **logger** allows you to log results of your benchmark. By default, you can see log on console and in a file (`.log`). ================================================ FILE: docs/articles/configs/orderers.md ================================================ --- uid: docs.orderers name: Orderers --- # Orderers Orderers allows customizing the order of benchmark results in the summary table. The following built-in order policies are available. - - - You can also use a custom orderer by implementing the interface. --- [!include[IntroOrderAttr](../samples/IntroOrderAttr.md)] [!include[IntroOrderManual](../samples/IntroOrderManual.md)] ================================================ FILE: docs/articles/configs/powerplans.md ================================================ --- uid: docs.powerplans name: Power Plans --- # Power Plans BenchmarkDotNet forces Windows OS to execute on the High-Performance power plan. You can disable this feature by modify PowerPlanMode property. You can see it in the @BenchmarkDotNet.Samples.IntroPowerPlan. Please note. During an execution, BenchmarkDotNet saves the current power plan and applies it according to the PowerPlanMode property. When all of the benchmarks finish, a previous power plan comes back. However, if someone killed process or energy was plugged off, we could stay with the High-Performance power plan. In this situation, we should return it manually in Windows Control Panel or by powercfg command. ### Links * Power policy settings: https://learn.microsoft.com/windows/win32/power/power-policy-settings * Powercfg command: https://learn.microsoft.com/windows-hardware/design/device-experiences/powercfg-command-line-options * @BenchmarkDotNet.Samples.IntroPowerPlan --- ================================================ FILE: docs/articles/configs/toc.yml ================================================ - name: Configs href: configs.md - name: Jobs href: jobs.md - name: Columns href: columns.md - name: Exporters href: exporters.md - name: Loggers href: loggers.md - name: Diagnosers href: diagnosers.md - name: Toolchains href: toolchains.md - name: Analysers href: analysers.md - name: Validators href: validators.md - name: Filters href: filters.md - name: Orderers href: orderers.md - name: ConfigOptions href: configoptions.md ================================================ FILE: docs/articles/configs/toolchains.md ================================================ --- uid: docs.toolchains name: Toolchains --- # Toolchains To achieve process-level isolation, BenchmarkDotNet generates, builds and executes a new console app per every benchmark. A **toolchain** contains generator, builder, and executor. When you run your benchmarks without specifying the toolchain in an explicit way, the default one is used: * Roslyn for Full .NET Framework and Mono * dotnet cli for .NET Core and NativeAOT ## Multiple frameworks support If you want to test multiple frameworks, your project file **MUST target all of them** and you **MUST install the corresponding SDKs**: ```xml netcoreapp3.1;net8.0;net48 ``` If you run your benchmarks without specifying any custom settings, BenchmarkDotNet is going to run the benchmarks **using the same framework as the host process**: ```cmd dotnet run -c Release -f netcoreapp3.1 # is going to run the benchmarks using .NET Core 3.1 dotnet run -c Release -f net8.0 # is going to run the benchmarks using .NET 8.0 dotnet run -c Release -f net48 # is going to run the benchmarks using .NET 4.8 mono $pathToExe # is going to run the benchmarks using Mono from your PATH ``` To run the benchmarks for multiple runtimes with a single command, you need to specify the target framework moniker names via `--runtimes|-r` console argument: ```cmd dotnet run -c Release -f net8.0 --runtimes net8.0 netcoreapp3.1 # is going to run the benchmarks using .NET 8.0 and .NET Core 3.1 dotnet run -c Release -f net8.0 --runtimes net8.0 net48 # is going to run the benchmarks using .NET 8.0 and .NET 4.8 ``` What is going to happen if you provide multiple Full .NET Framework monikers? Let's say: ```cmd dotnet run -c Release -f net461 net472 net48 ``` Full .NET Framework always runs every .NET executable using the latest .NET Framework available on a given machine. If you try to run the benchmarks for a few .NET TFMs, they are all going to be executed using the latest .NET Framework from your machine. The only difference is that they are all going to have different features enabled depending on target version they were compiled for. You can read more about this [here](https://docs.microsoft.com/en-us/dotnet/framework/migration-guide/version-compatibility) and [here](https://docs.microsoft.com/en-us/dotnet/framework/migration-guide/application-compatibility). This is **.NET Framework behavior which can not be controlled by BenchmarkDotNet or any other tool**. **Note:** Console arguments support works only if you pass the `args` to `BenchmarkSwitcher`: ```cs class Program { static void Main(string[] args) => BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args); // crucial to make it work } ``` You can achieve the same thing using `[SimpleJobAttribute]`: ```cs using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { [SimpleJob(RuntimeMoniker.Net48)] [SimpleJob(RuntimeMoniker.Mono)] [SimpleJob(RuntimeMoniker.NetCoreApp31)] [SimpleJob(RuntimeMoniker.Net80)] public class TheClassWithBenchmarks ``` Or using a custom config: ```cs using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Samples { class Program { static void Main(string[] args) { var config = DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80)) .AddJob(Job.Default.WithRuntime(ClrRuntime.Net48)) .AddJob(Job.Default.WithRuntime(MonoRuntime.Default)); BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args, config); } } } ``` The recommended way of running the benchmarks for multiple runtimes is to use the `--runtimes` console line argument. By using the console line argument you don't need to edit the source code anytime you want to change the list of runtimes. Moreover, if you share the source code of the benchmark other people can run it even if they don't have the exact same framework version installed. ## Custom .NET Core Runtime We can run your benchmarks for custom `` if you want. All you need to do is to create custom toolchain by calling `CsProjCoreToolchain.From` method, which accepts `NetCoreAppSettings`. ```cs public class MyConfig : ManualConfig { public MyConfig() { Add(Job.Default.With( CsProjCoreToolchain.From( new NetCoreAppSettings( targetFrameworkMoniker: "net8.0-windows", runtimeFrameworkVersion: "8.0.101", name: ".NET 8.0 Windows")))); } } ``` ## Custom .NET Runtime It's possible to benchmark a private build of .NET Runtime. All you need to do is to define a job with the right version of `ClrRuntime`. ```cs BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args, DefaultConfig.Instance.AddJob( Job.ShortRun.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0")))); ``` This sends the provided version as a `COMPLUS_Version` env var to the benchmarked process. ## Custom dotnet cli path We internally use dotnet cli to build and run .NET Core executables. Sometimes it might be mandatory to use non-default dotnet cli path. An example scenario could be a comparison of RyuJit 32bit vs 64 bit. It required due this [limitation](https://github.com/dotnet/cli/issues/7532) of dotnet cli ```cs public class CustomPathsConfig : ManualConfig { public CustomPathsConfig() { var dotnetCli32bit = NetCoreAppSettings .NetCoreApp31 .WithCustomDotNetCliPath(@"C:\Program Files (x86)\dotnet\dotnet.exe", "32 bit cli"); var dotnetCli64bit = NetCoreAppSettings .NetCoreApp31 .WithCustomDotNetCliPath(@"C:\Program Files\dotnet\dotnet.exe", "64 bit cli"); AddJob(Job.RyuJitX86.WithToolchain(CsProjCoreToolchain.From(dotnetCli32bit)).WithId("32 bit cli")); AddJob(Job.RyuJitX64.WithToolchain(CsProjCoreToolchain.From(dotnetCli64bit)).WithId("64 bit cli")); } } ``` ``` ini BenchmarkDotNet=v0.10.9.20170910-develop, OS=Windows 10 Redstone 1 (10.0.14393) Processor=Intel Core i7-6600U CPU 2.60GHz (Skylake), ProcessorCount=4 Frequency=2742185 Hz, Resolution=364.6727 ns, Timer=TSC .NET Core SDK=2.1.0-preview1-007074 [Host] : .NET Core 2.0.0 (Framework 4.6.00001.0), 64bit RyuJIT 32 bit cli : .NET Core 2.0.0 (Framework 4.6.00001.0), 32bit RyuJIT 64 bit cli : .NET Core 2.0.0 (Framework 4.6.00001.0), 64bit RyuJIT Jit=RyuJit ``` This feature is now also exposed with the `--cli` console argument. Example: `dotnet run -c Release -- --cli "C:\Projects\machinelearning\Tools\dotnetcli\dotnet.exe"` ## CoreRun To use CoreRun for running the benchmarks you need to use `--coreRun `command line argument. You can combine it with `--cli` described above. This is most probably the easiest and most reliable way of running benchmarks against local CoreFX/CoreCLR builds. Example: `dotnet run -c Release -- --coreRun "C:\Projects\corefx\bin\testhost\netcoreapp-Windows_NT-Release-x64\shared\Microsoft.NETCore.App\9.9.9\CoreRun.exe"` --- [!include[IntroInProcess](../samples/IntroInProcess.md)] [!include[IntroInProcessWrongEnv](../samples/IntroInProcessWrongEnv.md)] ## NativeAOT BenchmarkDotNet supports [NativeAOT](https://github.com/dotnet/runtime/tree/main/src/coreclr/nativeaot)! However, you might want to know how it works to get a better understanding of the results that you get. As every AOT solution, NativeAOT has some [limitations](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/limitations.md) like limited reflection support or lack of dynamic assembly loading. Because of that, the host process (what you run from command line) is never an AOT process, but just a regular .NET process. This process (called Host process) uses reflection to read benchmarks metadata (find all `[Benchmark]` methods etc), generates a new project that references the benchmarks and compiles it using ILCompiler. Such compilation produces a native executable, which is later started by the Host process. This process (called Benchmark or Child process) performs the actual benchmarking and reports the results back to the Host process. By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the NativeAOT benchmark according to [this instructions](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/compiling.md). This is why you need to: - install [pre-requisites](https://docs.microsoft.com/en-us/dotnet/core/deploying/native-aot/#prerequisites) required by NativeAOT compiler - target .NET to be able to run NativeAOT benchmarks (example: `net7.0` in the .csproj file) - run the app as a .NET process (example: `dotnet run -c Release -f net7.0`). - specify the NativeAOT runtime in an explicit way, either by using console line arguments `--runtimes nativeaot7.0` (the recommended approach), or by using`[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(NativeAotRuntime.Net70)`: ```cmd dotnet run -c Release -f net7.0 --runtimes nativeaot7.0 ``` or: ```cs var config = DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(NativeAotRuntime.Net70)); // compiles the benchmarks as net7.0 and uses the latest NativeAOT to build a native app BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args, config); ``` or: ```cs [SimpleJob(RuntimeMoniker.NativeAot70)] // compiles the benchmarks as net7.0 and uses the latest NativeAOT to build a native app public class TheTypeWithBenchmarks { [Benchmark] // the benchmarks go here } ``` ### Customization If you want to benchmark some particular version of NativeAOT (or from a different NuGet feed) you have to specify it in an explicit way: ```cs var config = DefaultConfig.Instance .AddJob(Job.ShortRun .WithToolchain(NativeAotToolchain.CreateBuilder() .UseNuGet( microsoftDotNetILCompilerVersion: "7.0.0-*", // the version goes here nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json") // this address might change over time .DisplayName("NativeAOT NuGet") .TargetFrameworkMoniker("net7.0") .ToToolchain())); ``` The builder allows to configure more settings: - specify packages restore path by using `PackagesRestorePath($path)` - rooting all application assemblies by using `RootAllApplicationAssemblies($bool)`. This is disabled by default. - generating stack trace metadata by using `IlcGenerateStackTraceData($bool)`. This option is enabled by default. - set optimization preference by using `IlcOptimizationPreference($value)`. The default is `Speed`, you can configure it to `Size` or nothing - set instruction set for the target OS, architecture and hardware by using `IlcInstructionSet($value)`. By default BDN recognizes most of the instruction sets on your machine and enables them. BenchmarkDotNet supports [rd.xml](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/rd-xml-format.md) files. To get given file respected by BenchmarkDotNet you need to place it in the same folder as the project that defines benchmarks and name it `rd.xml` or in case of multiple files give them `.rd.xml` extension. The alternative to `rd.xml` files is annotating types with [DynamicallyAccessedMembers](https://devblogs.microsoft.com/dotnet/app-trimming-in-net-5/) attribute. If given benchmark is not supported by NativeAOT, you need to apply `[AotFilter]` attribute for it. Example: ```cs [Benchmark] [AotFilter("Not supported by design.")] public object CreateInstanceNames() => System.Activator.CreateInstance(_assemblyName, _typeName); ``` ### Generated files By default BenchmarkDotNet removes the generates files after finishing the run. To keep them on the disk you need to pass `--keepFiles true` command line argument or apply `[KeepBenchmarkFiles]` attribute to the class which defines benchmark(s). Then, read the folder from the tool output. In the example below it's `D:\projects\performance\artifacts\bin\MicroBenchmarks\Release\net7.0\Job-KRLVKQ`: ```log // ***** Building 1 exe(s) in Parallel: Start ***** // start dotnet restore -r win-x64 /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in D:\projects\performance\artifacts\bin\MicroBenchmarks\Release\net7.0\Job-KRLVKQ // command took 2.74s and exited with 0 // start dotnet build -c Release -r win-x64 --no-restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in D:\projects\performance\artifacts\bin\MicroBenchmarks\Release\net7.0\Job-KRLVKQ // command took 3.82s and exited with 0 ``` If you go to `D:\projects\performance\artifacts\bin\MicroBenchmarks\Release\net7.0\Job-KRLVKQ`, you can see the generated project file (named `BenchmarkDotNet.Autogenerated.csproj`), code (file name ends with `.notcs`) and find the native executable (in the `bin\**\native` subfolder). Example: ```cmd cd D:\projects\performance\artifacts\bin\MicroBenchmarks\Release\net7.0\Job-KRLVKQ cat .\BenchmarkDotNet.Autogenerated.csproj ``` ```log false false Exe net7.0 win-x64 Job-KRLVKQ Job-KRLVKQ true x64 False false false true false Speed linklink True True false false false true ``` ### Compiling source to native code using the ILCompiler you built If you are a NativeAOT contributor and you want to benchmark your local build of NativeAOT you have to provide necessary info (path to shipping packages). You can do that from command line: ```cmd dotnet run -c Release -f net7.0 --runtimes nativeaot7.0 --ilcPackages D:\projects\runtime\artifacts\packages\Release\Shipping\ ``` or explicitly in the code: ```cs var config = DefaultConfig.Instance .AddJob(Job.ShortRun .WithToolchain(NativeAotToolchain.CreateBuilder() .UseLocalBuild(@"C:\Projects\runtime\artifacts\packages\Release\Shipping\") .DisplayName("NativeAOT local build") .TargetFrameworkMoniker("net7.0") .ToToolchain())); ``` BenchmarkDotNet is going to follow [these instructrions](https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/nativeaot.md#building) to get it working for you. **Note**: BenchmarkDotNet is going to run `dotnet restore` on the auto-generated project and restore the packages to a temporary folder. It might take some time, but the next time you rebuild dotnet/runtime repo and run the same command BenchmarkDotNet is going to use the new ILCompiler package. ## Wasm BenchmarkDotNet supports Web Assembly on Unix! However, currently you need to build the **dotnet runtime** yourself to be able to run the benchmarks. For up-to-date docs, you should visit [dotnet/runtime repository](https://github.com/dotnet/runtime/blob/main/docs/workflow/testing/libraries/testing-wasm.md). The docs below are specific to Ubuntu 18.04 at the moment of writing this document (16/07/2020). Firs of all, you need to install.... **npm** 10+: ```cmd curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash - sudo apt install nodejs ``` After this, you need to install [jsvu](https://github.com/GoogleChromeLabs/jsvu): ```cmd npm install jsvu -g ``` Add it to PATH: ```cmd export PATH="${HOME}/.jsvu:${PATH}" ``` And use it to install V8, JavaScriptCore and SpiderMonkey: ```cmd jsvu --os=linux64 --engines=javascriptcore,spidermonkey,v8 ``` Now you need to install [Emscripten](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions): ```cmd git clone https://github.com/emscripten-core/emsdk.git cd emsdk ./emsdk install latest ./emsdk activate latest source ./emsdk_env.sh ``` The last thing before cloning dotnet/runtime repository is creation of `EMSDK_PATH` env var used by Mono build scripts: ```cmd export EMSDK_PATH=$EMSDK ``` Now you need to clone dotnet/runtime repository: ```cmd git clone https://github.com/dotnet/runtime cd runtime ``` Install [all Mono prerequisites](https://github.com/dotnet/runtime/blob/main/docs/workflow/testing/libraries/testing-wasm.md): ```cmd sudo apt-get install cmake llvm-9 clang-9 autoconf automake libtool build-essential python curl git lldb-6.0 liblldb-6.0-dev libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev libssl-dev libnuma-dev libkrb5-dev zlib1g-dev ``` And FINALLY build Mono Runtime with Web Assembly support: ```cmd ./build.sh --arch wasm --os Browser -c release ``` And that you have .NET 5 feed added to your `nuget.config` file: ```xml ``` Now you should be able to run the Wasm benchmarks! [!include[IntroWasm](../samples/IntroWasm.md)] ## MonoAotLLVM BenchmarkDotNet supports doing Mono AOT runs with both the Mono-Mini compiler and the Mono-LLVM compiler (which uses llvm on the back end). Using this tool chain requires the following flags: ``` --runtimes monoaotllvm --aotcompilerpath --customruntimepack ``` and optionally (defaults to mini) ``` --aotcompilermode ``` As of this writing, the mono aot compiler is not available as a seperate download or nuget package. Therefore, it is required to build the compiler in the [dotnet/runtime repository]. The compiler binary (mono-sgen) is built as part of the `mono` subset, so it can be built (along with the runtime pack) like so (in the root of [dotnet/runtime]). `./build.sh -subset mono+libs -c Release` The compiler binary should be generated here (modify for your platform): ``` /artifacts/obj/mono/OSX.x64.Release/mono/mini/mono-sgen ``` And the runtime pack should be generated here: ``` artifacts/bin/microsoft.netcore.app.runtime.osx-x64/Release/ ``` ================================================ FILE: docs/articles/configs/validators.md ================================================ --- uid: docs.validators name: Validators --- # Validators A **validator** can validate your benchmarks before they are executed and produce validation errors. If any of the validation errors is critical, then none of the benchmarks will get executed. Available validators are: * `BaselineValidator.FailOnError` - it checks if more than 1 Benchmark per class has `Baseline = true` applied. This validator is mandatory. * `JitOptimizationsValidator.(Dont)FailOnError` - it checks whether any of the referenced assemblies is non-optimized. `DontFailOnError` version is enabled by default. * `ExecutionValidator.(Dont)FailOnError` - it checks if it is possible to run your benchmarks by executing each of them once. Optional. * `ReturnValueValidator.(Dont)FailOnError` - it checks if non-void benchmarks return equal values. Optional. ================================================ FILE: docs/articles/contributing/building.md ================================================ # Building There are two recommended options to build BenchmarkDotNet from source: ## Visual Studio - [Visual Studio](https://www.visualstudio.com/downloads/) (Community, Professional, Enterprise) with .NET 4.6.2 SDK and F# support. - [.NET 10 SDK](https://dotnet.microsoft.com/download). Once all the necessary tools are in place, building is trivial. Simply open solution file **BenchmarkDotNet.slnx** that lives at the base of the repository and run Build action. ## Command-line [Cake (C# Make)](https://cakebuild.net/) is a cross platform build automation system with a C# DSL to do things like compiling code, copy files/folders, running unit tests, compress files and build NuGet packages. The build currently depends on the following prerequisites: - Windows: - PowerShell version 5 or higher - MSBuild version 15.1 or higher - .NET Framework 4.6 or higher - Linux: - Install [Mono version 5 or higher](https://www.mono-project.com/download/stable/#download-lin) - Install [fsharp package](https://fsharp.org/use/linux/) - Install packages required to .NET Core SDK - `gettext` - `libcurl4-openssl-dev` - `libicu-dev` - `libssl-dev` - `libunwind8` - macOS - Install [Mono version 5 or higher](https://www.mono-project.com/download/stable/#download-mac) - Install [fsharp package](https://fsharp.org/use/mac/) - Install the latest version of [OpenSSL](https://www.openssl.org/source/). In order to run various build tasks from terminal, use `build.cmd` file in the repository root. `build.cmd` is a cross-platform script that can be used the same way on Windows, Linux, and macOS. When executed without arguments, it prints help information with list of all available build tasks. ================================================ FILE: docs/articles/contributing/debugging.md ================================================ # Debugging There should be two debug profiles available in VS drop down ![](https://cloud.githubusercontent.com/assets/6011991/15627671/89f2405a-24eb-11e6-8bd1-c9d45613e0f6.png "Debug profiles") ================================================ FILE: docs/articles/contributing/disassembler.md ================================================ # Contributing to Disassembler The disassembler might looks scarry, but once you know how it works and how to debug it, it's very easy to develop it. ### How it works We have 3 disassemblers: - Mono - x64 for Windows and Linux - x86 for Windows The MonoDisassembler is very simple: it spawns Mono with the right arguments to get the asm, Mono prints the output to the console and we just parse it. Single class does the job: `MonoDisassembler`. When it comes to Windows disassemblers it's not so easy. To obtain the disassm we are using ClrMD. ClrMD can attach only to the process of same bitness (architecture). This is why we have two disassemblers: x64 and x86. The code is the same (single class, linked in two projects) but compiled for two different architectures. We keep both disassemblers in the resources of the BenchmarkDotNet.dll. When we need the disassembler, we search for it in the resources, copy it to the disk and run (it's an exe). On Linux it's simpler (only x64 is supported) and we don't spawn a new process (everything is done in-proc). ### How to debug the disassembler You need to create a new console app project which executes the code that you would like to disassemble. In this app, you need to run the desired code (to get it jitted) and just don't exit before attaching the disassembler and getting the disassembly. Disassembler requires some arguments to run: id of the process to attach, full type name of the type which contains desired method, name of desired method and few other (see the example below). Personally I use following code to run the console app and print arguments that are required to attach to it: ```cs namespace Sample { class Program { static void Main(string[] args) { var result = Benchmark(); // execute the benchmark do method gets jitted Console.WriteLine( $"{Process.GetCurrentProcess().Id} " + // process Id $"\"{typeof(Program).FullName}\" " + // full type name $"{nameof(Benchmark)} " + // benchmarked method name $"{bool.FalseString} " + // print Source "2 " + // recursive depth $"{Path.GetTempFileName()}.xml"); // result xml file path while(true) { Console.WriteLine("Press Ctrl+C to kill the process"); Console.ReadLine(); // block the exe, attach with Disassembler now } GC.KeepAlive(result); } public static IntPtr Benchmark() { return new IntPtr(42).Multiply(4); } } public static class IntPtrHelper { [MethodImpl(MethodImplOptions.AggressiveInlining)] public unsafe static IntPtr Multiply(this IntPtr a, int factor) { return (sizeof(IntPtr) == sizeof(int)) ? new IntPtr((int)a * factor) : new IntPtr((long)a * factor); } } } ``` **Important**: Please remember that every new classic .NET project in VS compiles as 32 bit. If you want to check the asm produced for x64 you need to go to the properties of the console app (Alt+Enter) and uncheck "Prefer 32 bit" in the "Build" tab. Once you configure your app, you should run it. It will give you an output similar to this: `13672 Sample.Program Benchmark True 7 C:\Users\adsitnik\AppData\Local\Temp\tmpDCB9.tmp.xml` Now you go to BenchmarkDotNet solution, select desired Disassembler project in the Solution Explorer and Set it as Startup project. After this you go to the project's properties and in the Debug tab copy-paste the arguments for the disassembler. Now when you start debugging, your IDE will spawn new process of the disassembler with the right arguments to attach to the desired exe. You should be able to debug it like any other app. Please keep in mind that you should always use the disassembler for the correct processor architecture. If you fail to debug it, you are most probably using the wrong one. ================================================ FILE: docs/articles/contributing/documentation.md ================================================ # Documentation BenchmarkDotNet uses [DocFX](https://dotnet.github.io/docfx/) as a documentation generation tool. ## Hints * If you want to provide a link to API, you can use [cross references](https://dotnet.github.io/docfx/tutorial/links_and_cross_references.html#different-syntax-of-cross-reference) by [UID](https://dotnet.github.io/docfx/tutorial/links_and_cross_references.html#define-uid). For example, `[SimpleJobAttribute](xref:BenchmarkDotNet.Attributes.SimpleJobAttribute)` and `@BenchmarkDotNet.Attributes.SimpleJobAttribute` will be transformed to [SimpleJobAttribute](xref:BenchmarkDotNet.Attributes.SimpleJobAttribute). ## Notes DocFX uses the [following syntax](https://dotnet.github.io/docfx/spec/docfx_flavored_markdown.html?tabs=tabid-1%2Ctabid-a#note-warningtipimportant) inside block quote for different types of notes: ```markdown > [!NOTE] > note content > [!TIP] > tip content > [!WARNING] > warning content > [!IMPORTANT] > important content > [!Caution] > caution content ``` It will be transformed to: > [!NOTE] > note content > [!TIP] > tip content > [!WARNING] > warning content > [!IMPORTANT] > important content > [!Caution] > caution content ## Building documentation locally You can build documentation locally with the help of the `docs-build` build task: ``` build.cmd docs-build ``` ## See also * [DocFX User Manual](https://dotnet.github.io/docfx/tutorial/docfx.exe_user_manual.html) * [DocFX Tutorials: Links and Cross References](https://dotnet.github.io/docfx/tutorial/links_and_cross_references.html) * [DocFX Flavored Markdown](https://dotnet.github.io/docfx/spec/docfx_flavored_markdown.html?tabs=tabid-1%2Ctabid-a#file-inclusion) ================================================ FILE: docs/articles/contributing/miscellaneous.md ================================================ #Miscellaneous topics ## F# # We have full F# support, all you have to do is to run `dotnet restore` to download the compilers etc. ## Chat room [![Join the chat at https://gitter.im/dotnet/BenchmarkDotNet](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dotnet/BenchmarkDotNet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ## How can I help? [Here is a list of up-for-grabs issues](https://github.com/dotnet/BenchmarkDotNet/issues?q=is%3Aissue+is%3Aopen+label%3Aup-for-grabs) ================================================ FILE: docs/articles/contributing/running-tests.md ================================================ # Running Tests To run all tests just run the following command in the repo root: ```cmd dotnet test -c Release BenchmarkDotNet.slnx ``` Most of the tests projects target `net462` and `net8.0`. If the change that you want to test is not specific to any particular runtime, you can run the tests for one of them. ```cmd dotnet test -c Release -f net8.0 BenchmarkDotNet.slnx ``` You should be able to run all of tests from your IDE as well. ## Verify Tests For some unit tests (e.g. for exporter tests) BenchmarkDotNet uses [Verify](https://github.com/VerifyTests/Verify). * The expected value for each test is stored in a `*.verified.txt` file located near the test source file in the repository. Verify generates verified file's names automatically according test name and its parameters. This files must be added under the source control. * It also creates a `*.received` file for each failed test. You can use diff tools for convenient file comparison. By default you can find test run results on the test runner console as usual. You can comment out the line ```result.DisableDiff()``` in ```VerifySettingsFactory.Create``` method and then Verify will open KDiff for each failed test. This way you can easily understand what's the difference between verified and received values and choose the correct one. ================================================ FILE: docs/articles/contributing/toc.yml ================================================ - name: Building href: building.md - name: Debugging href: debugging.md - name: Running tests href: running-tests.md - name: Miscellaneous topics href: miscellaneous.md - name: Disassembler href: disassembler.md - name: Documentation href: documentation.md ================================================ FILE: docs/articles/faq.md ================================================ --- uid: docs.faq name: FAQ --- # FAQ (Frequently asked questions) * **Q** Why can't I install BenchmarkDotNet in Visual Studio 2010/2012/2013? **A** BenchmarkDotNet requires NuGet 3.x+ and can't be installed in old versions of Visual Studio which use NuGet 2.x. Consider to use Visual Studio 2015/2017 or [Rider](https://www.jetbrains.com/rider/). See also: [BenchmarkDotNet#237](https://github.com/dotnet/BenchmarkDotNet/issues/237), [roslyn#12780](https://github.com/dotnet/roslyn/issues/12780). * **Q** Why can't I install BenchmarkDotNet in a new .NET Core Console App in Visual Studio 2017? **A** BenchmarkDotNet supports only netcoreapp2.0+. Some old Visual Studio 2017 can create a new application which targets netcoreapp1.0. You should upgrade it up to 2.0. If you want to target netcoreapp1.0 in your main assembly, it's recommended to create a separated project for benchmarks. * **Q** I created a new .NET Core Console App in Visual Studio 2017. Now I want to run my code on CoreCLR, full .NET Framework, and Mono. How can I do it? **A** Use the following lines in your `.csproj` file: ```xml netcoreapp2.0;net46 AnyCPU ``` And mark your benchmark class with the following attributes: ```cs [CoreJob, ClrJob, MonoJob] ``` * **Q** My source code targets old versions of .NET Framework or .NET Core, but BenchmarkDotNet requires `net461` and `netcoreapp2.0`. How can I run benchmarks in this case? **A** It's a good practice to introduce an additional console application (e.g. `MyAwesomeLibrary.Benchmarks`) which will depend on your code and BenchmarkDotNet. Due to the fact that users usually run benchmarks in a develop environment and don't distribute benchmarks for users, it shouldn't be a problem. * **Q** I wrote a small benchmark, but BenchmarkDotNet requires a lot of time for time measurements. How can I reduce this time? **A** By default, BenchmarkDotNet automatically chooses a number of iterations which allows achieving the best precision. If you don't need such level of precision and just want to have a quick way to get approximated results, you can specify all parameters manually. For example, you can use the `SimpleJob` or `ShortRunJob` attributes: ```cs [SimpleJob(launchCount: 1, warmupCount: 3, iterationCount: 5, invocationCount:100, id: "QuickJob")] [ShortRunJob] ``` * **Q** My benchmark unexpectedly stopped and I saw the information about error code. What can I do? **A** BenchmarkDotNet generates, builds and runs new process for every benchmark. This behavior is sometimes interpreted by anti-virus as dangerous, and the process is killed. Use `EnvironmentAnalyser` to detect antivirus software and configure your benchmark to use [`InProcessToolchain`](xref:BenchmarkDotNet.Samples.IntroInProcess). * **Q** Can I run benchmark on the virtual machine? **A** Yes, of course. However, it can affect results because of the shared, physical machine, virtualization process and incorrect `Stopwatch.Frequency`. If you are unsure whether an application is running on virtual environment, use `EnvironmentAnalyser` to detect VM hypervisor. * **Q** I have failed to run my benchmarks, I am getting following errors about non-optimized dll. What can I do? ``` Assembly BenchmarkDotNet.Samples which defines benchmarks references non-optimized BenchmarkDotNet If you own this dependency, please, build it in RELEASE. If you don't, you can create custom config with DontFailOnError to disable our custom policy and allow this b Assembly BenchmarkDotNet.Samples which defines benchmarks is non-optimized Benchmark was built without optimization enabled (most probably a DEBUG configuration). Please, build it in RELEASE. ``` **A** You should always run your benchmarks in RELEASE mode with optimizations enabled (default setting for RELEASE). However if you have to use non-optimized 3rd party assembly you have to create custom config to disable our default policy. ```cs public class AllowNonOptimized : ManualConfig { public AllowNonOptimized() { Add(JitOptimizationsValidator.DontFailOnError); // ALLOW NON-OPTIMIZED DLLS Add(DefaultConfig.Instance.GetLoggers().ToArray()); // manual config has no loggers by default Add(DefaultConfig.Instance.GetExporters().ToArray()); // manual config has no exporters by default Add(DefaultConfig.Instance.GetColumnProviders().ToArray()); // manual config has no columns by default } } ``` * **Q** I'm running benchmarks on Linux and I see: `// ! Failed to set up priority Highest for thread ... Make sure you have the right permissions.` What can I do? **A** This message can appear when using in-process toolchains (for example via `[InProcess]`) because BenchmarkDotNet tries to set `ThreadPriority.Highest`. On Linux, raising scheduling priority requires additional permissions. Instead of running the whole benchmark as root (which can create root-owned build artifacts), you can grant the benchmark executable the `CAP_SYS_NICE` capability: ```bash sudo setcap cap_sys_nice=eip /path/to/YourBenchmarksExecutable ``` If you don't need high thread priority, you can ignore the message or avoid in-process toolchains. * **Q** I have failed to run my benchmarks from LINQPad. How can I fix this problem? ``` Assembly LINQPadQuery which defines benchmarks references non-optimized LINQPad Assembly LINQPadQuery which defines benchmarks is non-optimized Benchmark was built without optimization enabled (most probably a DEBUG configuration). Please, build it in RELEASE. ``` **A** You need to make sure that you are using **AnyCPU** 5.22.05+ build of LINQPad with optimizations enabled. To enable the optimizations you need to go to Preferences -> Query and select `compile with /optimize+` * **Q** I'm trying to use `RPlotExporter` but there are no any images in the `results` folder **A** Try to specify `R_LIBS_USER` (e.g. `R_LIBS_USER=/usr/local/lib/R/` on Linux/macOS, see also: [#692](https://github.com/dotnet/BenchmarkDotNet/issues/692)) * **Q** My benchmark failed with OutOfMemoryException. How can I fix this problem? **A** BenchmarkDotNet continues to run additional iterations until desired accuracy level is achieved. It's possible only if the benchmark method doesn't have any side-effects. If your benchmark allocates memory and keeps it alive, you are creating a memory leak. You should redesign your benchmark and remove the side-effects. You can use `OperationsPerInvoke`, `IterationSetup` and `IterationCleanup` to do that. An example: ```cs public class OOM { private StringBuilder buffer = new StringBuilder(); [Benchmark] public void HasSideEffects() { // This method is growing the buffer to infinity // because it's executed millions of times buffer.Append('a'); } [Benchmark(OperationsPerInvoke = 16)] public void HasNoSideEffects() { buffer.Clear(); for (int i = 0; i < 1024; i++) buffer.Append('a'); } } ``` ================================================ FILE: docs/articles/features/baselines.md ================================================ --- uid: docs.baselines name: Benchmark and Job Baselines --- # Benchmark and Job Baselines In order to scale your results, you can mark a benchmark method or a job as a baseline. Let's learn this feature by examples. --- [!include[IntroBenchmarkBaseline](../samples/IntroBenchmarkBaseline.md)] [!include[IntroRatioSD](../samples/IntroRatioSD.md)] [!include[IntroCategoryBaseline](../samples/IntroCategoryBaseline.md)] [!include[IntroJobBaseline](../samples/IntroJobBaseline.md)] ================================================ FILE: docs/articles/features/disassembler.md ================================================ --- uid: docs.disassembler name: Disassembler --- # Disassembler Can be enabled by using `[DisassemblyDiagnoser]` or command line args: `-d` or `--disasm`. The configuration options available from code level are: * `maxDepth`: Includes called methods to given level. 1 by default, indexed from 1. To print just the benchmark set it to 0. This option is also available from the console arguments level `--disasmDepth`. * `printSource`: C#|F#|VB source code will be printed. False by default. * `printInstructionAddresses`: Print instruction addresses. False by default. * `exportGithubMarkdown`: Exports to GitHub markdown. True by default. * `exportHtml`: Exports to HTML with clickable links. False by default. * `exportCombinedDisassemblyReport`: Exports all benchmarks to a single HTML report. Makes it easy to compare different runtimes or methods (each becomes a column in HTML table). * `exportDiff`: Exports a diff of the assembly code to the Github markdown format. False by default. ### Requirements Disassembly Diagnoser requires following settings in your `.csproj` file: ```xml AnyCPU pdbonly true ``` To get the source code we need to locate and read the `.pdb` files. This is why we need `DebugType` and `DebugSymbols` settings. To compare different platforms the project which defines benchmarks has to target `AnyCPU`. > [!NOTE] > By default, BenchmarkDotNet excludes disassembler's native dependencies that aren't used on current target platform. > Use the following settings when running the benchmark binary on a different platform than it was built on, or multiple platforms. > > ```xml > all > ``` Or specify `` for the platform that it will be run on. ### Disassembly Diagnoser for Mono on Windows If you want to get a disassembly listing for Mono on Windows, you need `as` and `x86_64-w64-mingw32-objdump.exe` tools. If you don't have it, you will get a warning like follows: ``` It's impossible to get Mono disasm because you don't have some required tools: 'as' is not recognized as an internal or external command 'x86_64-w64-mingw32-objdump.exe' is not recognized as an internal or external command ``` The easiest way to get these tools: 1. Download and install [Cygwin](https://www.cygwin.com/) 2. On the "Select Packages" screen, search for `binutils` 3. Install `binutils` and `mingw64-x86_64-binutils` 4. Add `cygwin64\bin\` (or `cygwin\bin\`) in `%PATH%` ![](../../images/cygwin-binutils.png) --- [!include[IntroDisassembly](../samples/IntroDisassembly.md)] [!include[IntroDisassemblyRyuJit](../samples/IntroDisassemblyRyuJit.md)] [!include[IntroDisassemblyAllJits](../samples/IntroDisassemblyAllJits.md)] [!include[IntroDisassemblyDry](../samples/IntroDisassemblyDry.md)] ================================================ FILE: docs/articles/features/etwprofiler.md ================================================ --- #cspell:ignore etwprofiler uid: docs.etwprofiler name: EtwProfiler --- # EtwProfiler `EtwProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a trace file which can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). ![](https://adamsitnik.com/images/etwprofiler/flamegraph.png) ## How it works `EtwProfiler` uses `TraceEvent` library which internally uses Event Tracing for Windows (ETW) to capture stack traces and important .NET Runtime events. Before the process with benchmarked code is started, EtwProfiler starts User and Kernel ETW sessions. Every session writes data to it's own file and captures different data. User session listens for the .NET Runtime events (GC, JIT etc) while the Kernel session gets CPU stacks and Hardware Counter events. After this, the process with benchmarked code is started. During the benchmark execution all the data is captured and written to a trace file. Moreover, BenchmarkDotNet Engine emits it's own events to be able to differentiate jitting, warmup, pilot and actual workload when analyzing the trace file. When the benchmarking is over, both sessions are closed and the two trace files are merged into one. ## Limitations What we have today comes with following limitations: * EtwProfiler works only on Windows (one day we might implement similar thing for Unix using EventPipe) * Requires to run as Admin (to create ETW Kernel Session) * No `InProcessToolchain` support * To get the best possible managed code symbols you should configure your project in following way: ```xml pdbonly true ``` > [!NOTE] > On certain machines [Intel TDT and Windows Defender](https://www.microsoft.com/en-us/security/blog/2021/04/26/defending-against-cryptojacking-with-microsoft-defender-for-endpoint-and-intel-tdt/) can cause CPU samples to be captured with no value. > You can correct this problem by disabling the feature using `powershell.exe Set-MpPreference -DisableTDTFeature $true`. > *WARNING:* Disabling security features will make your machine less secure; do so at your own risk. ## How to use it? You need to install `BenchmarkDotNet.Diagnostics.Windows` package. It can be enabled in few ways, some of them: * Use the new attribute (apply it on a class that contains Benchmarks): ```cs using BenchmarkDotNet.Diagnostics.Windows.Configs; [EtwProfiler] public class TheClassThatContainsBenchmarks { /* benchmarks go here */ } ``` * Extend the `DefaultConfig.Instance` with new instance of `EtwProfiler`: ```cs class Program { static void Main(string[] args) => BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args, DefaultConfig.Instance .AddDiagnoser(new EtwProfiler())); // HERE } ``` * Passing `-p ETW` or `--profiler ETW` command line argument to `BenchmarkSwitcher` ## Configuration To configure the new diagnoser you need to create an instance of `EtwProfilerConfig` class and pass it to the `EtwProfiler` constructor. The parameters that `EtwProfilerConfig` ctor takes are: * `performExtraBenchmarksRun` - if set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. True by default. * `bufferSizeInMb` - ETW session buffer size, in MB. 256 by default. * `intervalSelectors` - interval per hardware counter, if not provided then default values will be used. * `kernelKeywords` - kernel session keywords, ImageLoad (for native stack frames) and Profile (for CPU Stacks) are the defaults. * `providers` - providers that should be enabled, if not provided then default values will be used. ## Using PerfView to work with trace files PerfView is a free .NET profiler from Microsoft. If you don't know how to use it you should watch [these instructional videos](https://channel9.msdn.com/Series/PerfView-Tutorial) first. If you are familiar with PerfView, then the only thing you need to know is that BenchmarkDotNet performs Jitting by running the code once, Pilot Experiment to determine how many times benchmark should be executed per iteration, non-trivial Warmup and Actual Workload. This is why when you open your trace file in PerfView you will see your benchmark in a few different places of the StackTrace. ![](https://adamsitnik.com/images/etwprofiler/flamegraph_not_filtered.png) The simplest way to filter the data to the actual benchmarks runs is to open the `CallTree` tab, put "EngineActualStage" in the Find box, press enter and when PerfView selects `EngineActualStage` in the `CallTree` press `Alt+R` to Set Time Range. ![](https://adamsitnik.com/images/etwprofiler/perfview.gif) If you want to filter the trace to single iteration, then you must go to the Events panel and search for the `WorkloadActual/Start` and `WorkloadActual/Stop` events. 1. Open Events window 2. Put "WorkloadActual" in the Filter box and hit enter. 3. Press control or shift and choose the Start and Stop events from the left panel. Hit enter. 4. Choose iteration that you want to investigate (events are sorted by time). 5. Select two or more cells from the "Time MSec" column. 6. Right click, choose "Open Cpu Stacks". 7. Choose the process with benchmarks, right-click, choose "Drill Into" ![](https://adamsitnik.com/images/etwprofiler/perfview_events.gif) ================================================ FILE: docs/articles/features/event-pipe-profiler.md ================================================ --- uid: docs.event-pipe-profiler name: EventPipeProfiler --- # EventPipeProfiler `EventPipeProfiler` is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. Collected data are exported to trace files (`.speedscope.json` and `.nettrace`) which can be analyzed using [SpeedScope](https://www.speedscope.app/), [PerfView](https://github.com/Microsoft/perfview), and [Visual Studio Profiler](https://learn.microsoft.com/visualstudio/profiling/profiling-feature-tour). This new profiler is available from the 0.12.1 version. ![](https://wojciechnagorski.github.io/images/EventPipeProfiler/SpeedScopeAdvance.png) # Configuration `EventPipeProfiler` can be enabled in three ways: 1. Using parameter `-p EP` or `--profiler EP` from the console line. 2. Marking the benchmarked class with `[EventPipeProfiler(...)]` attribute. You can find an example below. 3. Using a custom configuration. You can find an example below. [!include[IntroEventPipeProfiler](../samples/IntroEventPipeProfiler.md)] [!include[IntroEventPipeProfilerAdvanced](../samples/IntroEventPipeProfilerAdvanced.md)] ================================================ FILE: docs/articles/features/parameterization.md ================================================ --- uid: docs.parameterization name: Benchmark Parameterization --- # Parameterization --- [!include[IntroParams](../samples/IntroParams.md)] [!include[IntroParamsSource](../samples/IntroParamsSource.md)] [!include[IntroParamsAllValues](../samples/IntroParamsAllValues.md)] [!include[IntroParamsPriority](../samples/IntroParamsPriority.md)] [!include[IntroArguments](../samples/IntroArguments.md)] [!include[IntroArgumentsSource](../samples/IntroArgumentsSource.md)] [!include[IntroArrayParam](../samples/IntroArrayParam.md)] [!include[IntroArguments](../samples/IntroArgumentsPriority.md)] ================================================ FILE: docs/articles/features/setup-and-cleanup.md ================================================ --- uid: docs.setup-and-cleanup name: Setup And Cleanup --- # Setup And Cleanup Sometimes we want to write some logic which should be executed *before* or *after* a benchmark, but we don't want to measure it. For this purpose, BenchmarkDotNet provides a set of attributes: [`[GlobalSetup]`](xref:BenchmarkDotNet.Attributes.GlobalSetupAttribute), [`[GlobalCleanup]`](xref:BenchmarkDotNet.Attributes.GlobalCleanupAttribute), [`[IterationSetup]`](xref:BenchmarkDotNet.Attributes.IterationSetupAttribute), [`[IterationCleanup]`](xref:BenchmarkDotNet.Attributes.IterationCleanupAttribute). --- [!include[IntroSetupCleanupGlobal](../samples/IntroSetupCleanupGlobal.md)] [!include[IntroSetupCleanupIteration](../samples/IntroSetupCleanupIteration.md)] [!include[IntroSetupCleanupTarget](../samples/IntroSetupCleanupTarget.md)] ================================================ FILE: docs/articles/features/statistics.md ================================================ --- uid: docs.statistics name: Statistics --- # Statistics --- [!include[IntroStatisticsColumns](../samples/IntroStatisticsColumns.md)] [!include[IntroPercentiles](../samples/IntroPercentiles.md)] [!include[IntroRankColumn](../samples/IntroRankColumn.md)] [!include[IntroMultimodal](../samples/IntroMultimodal.md)] [!include[IntroOutliers](../samples/IntroOutliers.md)] ================================================ FILE: docs/articles/features/toc.yml ================================================ - name: Parameterization href: parameterization.md - name: Baselines href: baselines.md - name: Setup And Cleanup href: setup-and-cleanup.md - name: Statistics href: statistics.md - name: Disassembler href: disassembler.md - name: EtwProfiler href: etwprofiler.md - name: EventPipeProfiler href: event-pipe-profiler.md - name: VSProfiler href: vsprofiler.md - name: VSTest href: vstest.md ================================================ FILE: docs/articles/features/vsprofiler.md ================================================ --- uid: docs.vsprofiler name: VS Profiler --- # Running with Visual Studio profiler Visual Studio supports [profiler integration with BenchmarkDotNet](https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet) on Windows through its [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package. Once installed, Visual Studio specific diagnosers will capture performance data in runs and automatically open traces if launched through Visual Studio ![](../../images/vs-profiler-demo.png) ## How it works First, install the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package in your benchmarking project. Next add one or more of the Visual Studio diagnosers to your benchmark to capture the relevant profiling information while benchmarking. Lastly, run your benchmarks and a diagsession will be generated. If run from Visual Studio the diagsession will automatically be opened. ## Available Diagnosers * `[CPUUsageDiagnoser]` - Enables the [CPU Usage tool](https://learn.microsoft.com/visualstudio/profiling/cpu-usage). * `[DatabaseDiagnoser]` - Enables the [Database tool](https://learn.microsoft.com/visualstudio/profiling/analyze-database) * `[DotNetCountersDiagnoser]` - Enables the [.NET Counters tool](https://learn.microsoft.com/visualstudio/profiling/dotnet-counters-tool) * `[DotNetObjectAllocDiagnoser]` - Enables the [.NET Object Allocation tool](https://learn.microsoft.com/visualstudio/profiling/dotnet-alloc-tool). When using this tool, you must also specify `[DotNetObjectAllocJobConfiguration]` on the benchmark. If this is missing the run will fail and you will receive an error indicating you need to add it. * `[EventsDiagnoser]` - Enables the [Events tool](https://learn.microsoft.com/visualstudio/profiling/events-viewer) * `[FileIODiagnoser]` - Enables the [File IO tool](https://learn.microsoft.com/visualstudio/profiling/use-file-io) ## How to use it? After installing the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package add the following code as a benchmark: ```cs using System; using System.Security.Cryptography; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Running; using Microsoft.VSDiagnostics; namespace MyBenchmarks { [CPUUsageDiagnoser] public class Md5VsSha256 { private const int N = 10000; private readonly byte[] data; private readonly SHA256 sha256 = SHA256.Create(); private readonly MD5 md5 = MD5.Create(); public Md5VsSha256() { data = new byte[N]; new Random(42).NextBytes(data); } [Benchmark] public byte[] Sha256() => sha256.ComputeHash(data); [Benchmark] public byte[] Md5() => md5.ComputeHash(data); } public class Program { public static void Main(string[] args) { var summary = BenchmarkRunner.Run(typeof(Program).Assembly); } } } ``` In this case we have added the `[CpuUsageDiagnoser]` to capture a CPU sampling trace. From here run the benchmark in Visual Studio (Ctrl+F5), and after the benchmark run the resulting diagsession will be displayed. Double clicking on one of the benchmark rows shown under the Benchmarks tab will filter the time selection to the specific benchmark allowing you to better isolate and investigate. ![](../../images/vs-profiler-demo.png) ================================================ FILE: docs/articles/features/vstest.md ================================================ --- uid: docs.vstest name: Running with VSTest --- # Running with VSTest BenchmarkDotNet supports discovering and executing benchmarks through VSTest. This provides an alternative user experience to running benchmarks with the CLI and may be preferable for those who like their IDE's VSTest integrations that they may have used when running unit tests. Below is an example of running some benchmarks from the BenchmarkDotNet samples project in Visual Studio's Test Explorer. ![](../../images/vs-testexplorer-demo.png) ## About VSTest VSTest is one of the most popular test platforms in use in the .NET ecosystem, with test frameworks such as MSTest, xUnit, and NUnit providing support for it. Many IDEs, including Visual Studio and Rider, provide UIs for running tests through VSTest which some users may find more accessible than running them through the command line. It may seem counterintuitive to run performance tests on a platform that is designed for unit tests that expect a boolean outcome of "Passed" or "Failed". However, VSTest provides good value as a protocol for discovering and executing tests. In addition, we can still make use of this boolean output to indicate if the benchmark had validation errors that caused it to fail to run. ## Caveats and things to know * **The benchmark measurements may be affected by the VSTest host and your IDE!** If you want to have accurate measurements, it is recommended to run benchmarks through the CLI without other processes on the machine impacting performance. This does not mean that the measurements are useless though, it will still be able to provide useful measurements during development when comparing different approaches. * **The test adapter will not display or execute benchmarks if optimizations are disabled.** Please ensure you are compiling in Release mode or with `Optimize` set to true. Using an `InProcess` toolchain will let you run your benchmarks with optimizations disabled and will let you attach the debugger as well. * **The VSTest adapter will not call your application's entry point.** If you use the entry point to customize how your benchmarks are run, you will need to do this through other means such as an assembly-level `IConfigSource`, as shown [here](#setting-a-default-configuration). * **The test adapter will generate an entry point for you automatically.** The generated entry point will pass the command line arguments and the current assembly into `BenchmarkSwitcher`, so you can still use it in your CLI as well as in VSTest. This means you can delete your entry point and only need to define your benchmarks. If you want to use a custom entry point, you can still do so by setting `GenerateProgramFile` to `false` in your project file. ## Getting started * **Step 1.** Install the NuGet packages. You need to install two packages into your benchmark project: * `BenchmarkDotNet.TestAdapter`: Implements the VSTest protocol for BenchmarkDotNet * `Microsoft.NET.Test.Sdk`: Includes all the pieces needed for the VSTest host to run and load the VSTest adapter. * **Step 2.** Make sure that the entry point is configured correctly. As mentioned in the caveats section, `BenchmarkDotNet.TestAdapter` will generate an entry point for you automatically. So, if you have an entry point already, you will either need to delete it or set `GenerateProgramFile` to `false` in your project file to continue using your existing one. Here is an example of a `.csproj` file based on the default Console Application template: ```xml Exe net8.0 enable enable false ``` * **Step 3.** Make sure that your IDE supports VSTest integration. In Visual Studio, everything works out of the box. In Rider/R#, the VSTest integration might need to be activated: * Go to the "Unit Testing" settings page. * Rider: Settings -> Build, Execution, Deployment -> Unit Testing -> VSTest * R#: Extensions -> ReSharper -> Options -> Tools -> Unit Testing -> Test Frameworks -> VSTest * Make sure that the "Enable VSTest adapter support" checkbox is checked. In recent versions of Rider, this may be enabled by default. * **Step 4.** Switch to the `Release` configuration. As mentioned above, the TestAdapter is not able to discover and run benchmarks with optimizations disabled (by design). * **Step 5.** Build the project. In order to discover the benchmarks, the VSTest adapter needs to be able to find the assembly. Once you build the project, you should observe the discovered benchmarks in your IDE's Unit Test Explorer. If you correctly performed all the steps above, you should be able to run your benchmarks in your IDE using embedded unit testing features. If this doesn't work for you, don't hesitate to file [a new GitHub issue](https://github.com/dotnet/BenchmarkDotNet/issues/new). ## Setting a default configuration Previously, it was common for the default configuration to be defined inside the entry point. Since the entry point is not used when running benchmarks through VSTest, the default configuration must be specified using a `Config` attribute that is set on the assembly instead. First, create a class that extends `ManualConfig` or `IConfig` which sets the default configuration you want: ```csharp class MyDefaultConfig : ManualConfig { public MyDefaultConfig() { AddJob(Job.Dry); AddLogger(Loggers.ConsoleLogger.Default); AddValidator(JitOptimizationsValidator.DontFailOnError); } } ``` Then, set an assembly attribute with the following. ```csharp [assembly: Config(typeof(MyDefaultConfig))] ``` By convention, assembly attributes are usually defined inside `AssemblyInfo.cs` in a directory called `Properties`. ## Viewing the results The full output from BenchmarkDotNet that you would have been used to seeing in the past will be sent to the "Tests" output of your IDE. Use this view if you want to see the tabular view that compares multiple benchmarks with each other or if you want to see the results for each individual iteration. One more place where you can view the results is in each individual test's output messages. In Visual Studio, this can be viewed by clicking on the test in the Test Explorer after running it and looking at the Test Detail Summary. Since this only displays statistics for a single benchmark case, it does not show the tabulated view that compares multiple benchmark cases. Instead, it displays a histogram and various other useful statistics. Not all IDEs support displaying these output messages, so you may only be able to view the results using the "Tests" output. ================================================ FILE: docs/articles/guides/choosing-run-strategy.md ================================================ --- #cspell:ignore runstrategy uid: docs.runstrategy name: Choosing RunStrategy --- # Choosing RunStrategy If you run a benchmark, you always (explicitly or implicitly) use a [job](xref:docs.jobs). Each `Job` has the `RunStrategy` parameter which allows switching between different benchmark modes. The default `RunStrategy` is `Throughput`, and it works fine for most cases. However, other strategies are also useful in some specific cases. ## Throughput `Throughput` is the default `RunStrategy`, works perfectly for microbenchmarking. It's automatically choosing the amount of operation in main iterations based on a set of pilot iterations. The amount of iterations will also be chosen automatically based on accuracy job settings. A benchmark method should have a steady state. Of course, you can manually set all the characteristics. An example: ```cs [SimpleJob(launchCount: 3, warmupCount: 10, iterationCount: 30)] public class MyBenchmarkClass ``` --- [!include[IntroColdStart](../samples/IntroColdStart.md)] [!include[IntroMonitoring](../samples/IntroMonitoring.md)] ================================================ FILE: docs/articles/guides/console-args.md ================================================ --- uid: docs.console-args name: Console Arguments --- # How to use console arguments `BenchmarkSwitcher` supports various console arguments, to make it work you need to pass the `args` to switcher: ```cs class Program { static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); } ``` **Note:** the docs that you are currently reading might get outdated, to get the most up-to-date info about supported console arguments run the benchmarks with `--help`. ## Filter The `--filter` or just `-f` allows you to filter the benchmarks by their full name (`namespace.typeName.methodName`) using glob patterns. Examples: 1. Run all benchmarks from System.Memory namespace: `-f 'System.Memory*'` 2. Run all benchmarks: `-f '*'` 3. Run all benchmarks from ClassA and ClassB `-f '*ClassA*' '*ClassB*'` **Note**: If you would like to **join** all the results into a **single summary**, you need to put `--join`. For example: `-f '*ClassA*' '*ClassB*' --join` ## List of benchmarks The `--list` allows you to print all of the available benchmark names. Available options are: * `flat` - prints list of the available benchmarks: `--list flat` ```ini BenchmarkDotNet.Samples.Algo_Md5VsSha256.Md5 BenchmarkDotNet.Samples.Algo_Md5VsSha256.Sha256 BenchmarkDotNet.Samples.IntroArguments.Benchmark BenchmarkDotNet.Samples.IntroArgumentsSource.SingleArgument BenchmarkDotNet.Samples.IntroArgumentsSource.ManyArguments BenchmarkDotNet.Samples.IntroArrayParam.ArrayIndexOf BenchmarkDotNet.Samples.IntroArrayParam.ManualIndexOf BenchmarkDotNet.Samples.IntroBasic.Sleep [...] ``` * `tree` - prints tree of the available benchmarks: `--list tree` ```ini BenchmarkDotNet └─Samples ├─Algo_Md5VsSha256 │ ├─Md5 │ └─Sha256 ├─IntroArguments │ └─Benchmark ├─IntroArgumentsSource │ ├─SingleArgument │ └─ManyArguments ├─IntroArrayParam │ ├─ArrayIndexOf │ └─ManualIndexOf ├─IntroBasic │ ├─Sleep [...] ``` The `--list` option works with the `--filter` option. Examples: * `--list flat --filter *IntroSetupCleanup*` prints: ```ini BenchmarkDotNet.Samples.IntroSetupCleanupGlobal.Logic BenchmarkDotNet.Samples.IntroSetupCleanupIteration.Benchmark BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkA BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkB BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkC BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkD ``` * `--list tree --filter *IntroSetupCleanup*` prints: ```ini BenchmarkDotNet └─Samples ├─IntroSetupCleanupGlobal │ └─Logic ├─IntroSetupCleanupIteration │ └─Benchmark └─IntroSetupCleanupTarget ├─BenchmarkA ├─BenchmarkB ├─BenchmarkC └─BenchmarkD ``` ## Categories You can also filter the benchmarks by categories: * `--anyCategories` - runs all benchmarks that belong to **any** of the provided categories * `--allCategories`- runs all benchmarks that belong to **all** provided categories ## Diagnosers * `-m`, `--memory` - enables MemoryDiagnoser and prints memory statistics * `-t`, `--threading` - enables `ThreadingDiagnoser` and prints threading statistics * `-d`, `--disasm`- enables DisassemblyDiagnoser and exports diassembly of benchmarked code. When you enable this option, you can use: * `--disasmDepth` - Sets the recursive depth for the disassembler. * `--disasmDiff` - Generates diff reports for the disassembler. ## Runtimes The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: * Clr - BDN will either use Roslyn (if you run it as .NET app) or latest installed .NET SDK to build the benchmarks (if you run it as .NET Core app). * Core - if you run it as .NET Core app, BDN will use the same target framework moniker, if you run it as .NET app it's going to use net8.0. * Mono - it's going to use the Mono from `$Path`, you can override it with `--monoPath`. * net46, net461, net462, net47, net471, net472, net48, net481 - to build and run benchmarks against specific .NET Framework version. * netcoreapp3.1, net5.0, net6.0, net7.0, net8.0 - to build and run benchmarks against specific .NET (Core) version. * nativeaot5.0, nativeaot6.0, nativeaot7.0, nativeaot8.0 - to build and run benchmarks using NativeAOT. Can be customized with additional options: `--ilcPackages`, `--ilCompilerVersion`. * mono6.0, mono7.0, mono8.0 - to build and run benchmarks with .Net 6+ using MonoVM. Example: run the benchmarks for .NET 4.7.2 and .NET 8.0: ```log dotnet run -c Release -- --runtimes net472 net8.0 ``` Example: run the benchmarks for .NET Core 3.1 and latest .NET SDK installed on your PC: ```log dotnet run -c Release -f netcoreapp3.1 -- --runtimes clr core ``` But same command executed with `-f net6.0` is going to run the benchmarks for .NET 6.0: ```log dotnet run -c Release -f net6.0 -- --runtimes clr core ``` ## Number of invocations and iterations * `--launchCount` - how many times we should launch process with target benchmark. The default is 1. * `--warmupCount` - how many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic. * `--minWarmupCount` - minimum count of warmup iterations that should be performed. The default is 6. * `--maxWarmupCount` - maximum count of warmup iterations that should be performed. The default is 50. * `--iterationTime` - desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default. * `--iterationCount` - how many target iterations should be performed. By default calculated by the heuristic. * `--minIterationCount` - minimum number of iterations to run. The default is 15. * `--maxIterationCount` - maximum number of iterations to run. The default is 100. * `--invocationCount` - invocation count in a single iteration. By default calculated by the heuristic. * `--unrollFactor` - how many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default * `--runOncePerIteration` - run the benchmark exactly once per iteration. False by default. Example: run single warmup iteration, from 9 to 12 actual workload iterations. ```log dotnet run -c Release -- --warmupCount 1 --minIterationCount 9 --maxIterationCount 12 ``` ## Specifying custom default settings for console argument parser If you want to have a possibility to specify custom default Job settings programmatically and optionally overwrite it with console line arguments, then you should create a global config with single job marked as `.AsDefault` and pass it to `BenchmarkSwitcher` together with the console line arguments. Example: run single warmup iteration by default. ```cs static void Main(string[] args) => BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args, GetGlobalConfig()); static IConfig GetGlobalConfig() => DefaultConfig.Instance .With(Job.Default .WithWarmupCount(1) .AsDefault()); // the KEY to get it working ``` Now, the default settings are: `WarmupCount=1` but you might still overwrite it from console args like in the example below: ```log dotnet run -c Release -- --warmupCount 2 ``` ## Response files support Benchmark.NET supports parsing parameters via response files. for example you can create file `run.rsp` with following content ``` --warmupCount 1 --minIterationCount 9 --maxIterationCount 12 ``` and run it using `dotnet run -c Release -- @run.rsp`. It would be equivalent to running following command line ```log dotnet run -c Release -- --warmupCount 1 --minIterationCount 9 --maxIterationCount 12 ``` ## Statistical Test To perform a Mann–Whitney U Test and display the results in a dedicated column you need to provide the Threshold: * `--statisticalTest`- Threshold for Mann–Whitney U Test. Examples: 5%, 10ms, 100ns, 1s Example: run Mann–Whitney U test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET 6.0 will be baseline because it was first. ```log dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5% ``` ## Example Usages * Use Job.ShortRun for running the benchmarks: `-j short` * Run benchmarks in process: `-i` * Run benchmarks for .NET 4.7.2, .NET 8.0 and Mono. .NET 4.7.2 will be baseline because it was first.: `--runtimes net472 net8.0 Mono` * Run benchmarks for .NET 8.0 and .NET 10.0. .NET 8.0 will be baseline because it was first.: `--runtimes net8.0 net10.0` * Use MemoryDiagnoser to get GC stats: `-m` * Use DisassemblyDiagnoser to get disassembly: `-d` * Use HardwareCountersDiagnoser to get hardware counter info: `--counters CacheMisses+InstructionRetired` * Run all benchmarks exactly once: `-f * -j Dry` * Run all benchmarks from System.Memory namespace: `-f System.Memory*` * Run all benchmarks from ClassA and ClassB using type names: `-f ClassA ClassB` * Run all benchmarks from ClassA and ClassB using patterns: `-f *.ClassA.* *.ClassB.*` * Run all benchmarks called `BenchmarkName` and show the results in single summary: `--filter *.BenchmarkName --join` * Run selected benchmarks once per iteration: `--runOncePerIteration` * Run selected benchmarks 100 times per iteration. Perform single warmup iteration and 5 actual workload iterations: `--invocationCount 100 --iterationCount 5 --warmupCount 1` * Run selected benchmarks 250ms per iteration. Perform from 9 to 15 iterations: `--iterationTime 250 --maxIterationCount 15 --minIterationCount 9` * Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET Core 6.0 will be baseline because it was provided as first.: `--filter * --runtimes net6.0 net8.0 --statisticalTest 5%` * Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2': `--envVars ENV_VAR_KEY_1:value_1 ENV_VAR_KEY_2:value_2` * Hide Mean and Ratio columns (use double quotes for multi-word columns: "Alloc Ratio"): `-h Mean Ratio` ## More * `-j`, `--job` (Default: Default) Dry/Short/Medium/Long or Default * `-r`, `--runtimes` Full target framework moniker for .NET Core and .NET. For Mono just 'Mono'. For NativeAOT please append target runtime version (example: 'nativeaot7.0'). First one will be marked as baseline! * `-e`, `--exporters` GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML * `-m`, `--memory` (Default: false) Prints memory statistics * `-t`, `--threading` (Default: false) Prints threading statistics * `--exceptions` (Default: false) Prints exception statistics * `-d, --disasm` (Default: false) Gets disassembly of benchmarked code * `-p, --profiler` Profiles benchmarked code using selected profiler. Available options: EP/ETW/CV/NativeMemory * `-f, --filter` Glob patterns * `-h, --hide` Hides columns by name * `-i, --inProcess` (Default: false) Run benchmarks in Process * `-a, --artifacts` Valid path to accessible directory * `--outliers` (Default: RemoveUpper) `DontRemove`/`RemoveUpper`/`RemoveLower`/`RemoveAll` * `--affinity` Affinity mask to set for the benchmark process * `--allStats` (Default: false) Displays all statistics (min, max & more) * `--allCategories` Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed * `--anyCategories` Any Categories to run * `--attribute` Run all methods with given attribute (applied to class or method) * `--join` (Default: false) Prints single table with results for all benchmarks * `--keepFiles` (Default: false) Determines if all auto-generated files should be kept or removed after running the benchmarks. * `--noOverwrite` (Default: false) Determines if the exported result files should not be overwritten (be default they are overwritten). * `--counters` Hardware Counters * `--cli` Path to dotnet cli (optional). * `--packages` The directory to restore packages to (optional). * `--coreRun` Path(s) to CoreRun (optional). * `--monoPath` Optional path to Mono which should be used for running benchmarks. * `--clrVersion` Optional version of private CLR build used as the value of `COMPLUS_Version` env var. * `--ilCompilerVersion` Optional version of Microsoft.DotNet.ILCompiler which should be used to run with NativeAOT. Example: "7.0.0-preview.3.22123.2" * `--ilcPackages` Optional path to shipping packages produced by local dotnet/runtime build. Example: 'D:\projects\runtime\artifacts\packages\Release\Shipping\' * `--launchCount` How many times we should launch process with target benchmark. The default is 1. * `--warmupCount` How many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic. * `--minWarmupCount` Minimum count of warmup iterations that should be performed. The default is 6. * `--maxWarmupCount` Maximum count of warmup iterations that should be performed. The default is 50. * `--iterationTime` Desired time of execution of an iteration in milliseconds. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default * `--iterationCount` How many target iterations should be performed. By default calculated by the heuristic. * `--minIterationCount` Minimum number of iterations to run. The default is 15. * `--maxIterationCount` Maximum number of iterations to run. The default is 100. * `--invocationCount` Invocation count in a single iteration. By default calculated by the heuristic. * `--unrollFactor` How many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default * `--strategy` The RunStrategy that should be used. Throughput/ColdStart/Monitoring. * `--platform` The Platform that should be used. If not specified, the host process platform is used (default). AnyCpu/X86/X64/Arm/Arm64/LoongArch64. * `--runOncePerIteration` (Default: false) Run the benchmark exactly once per iteration. * `--info` (Default: false) Print environment information. * `--apples` (Default: false) Runs apples-to-apples comparison for specified Jobs. * `--list` (Default: Disabled) Prints all of the available benchmark names. Flat/Tree * `--disasmDepth` (Default: 1) Sets the recursive depth for the disassembler. * `--disasmFilter` Glob patterns applied to full method signatures by the the disassembler. * `--disasmDiff` (Default: false) Generates diff reports for the disassembler. * `--logBuildOutput` Log Build output. * `--generateBinLog` Generate msbuild `binlog` for builds * `--buildTimeout` Build timeout in seconds. * `--wakeLock` Prevents the system from entering sleep or turning off the display. None/System/Display. * `--stopOnFirstError` (Default: false) Stop on first error. * `--statisticalTest` Threshold for Mann–Whitney U Test. Examples: 5%, 10ms, 100ns, 1s * `--disableLogFile` Disables the `logfile`. * `--maxWidth` Max parameter column width, the default is 20. * `--envVars` Colon separated environment variables (key:value) * `--memoryRandomization` Specifies whether Engine should allocate some random-sized memory between iterations. It makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration. * `--wasmEngine` Full path to a java script engine used to run the benchmarks, used by Wasm toolchain. * `--wasmArgs` (Default: --expose_wasm) Arguments for the javascript engine used by Wasm toolchain. * `--customRuntimePack` Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently. * `--AOTCompilerPath` Path to Mono AOT compiler, used for MonoAotLLVM. * `--AOTCompilerMode` (Default: mini) Mono AOT compiler mode, either 'mini' or 'llvm' * `--wasmDataDir` Wasm data directory * `--wasmCoreCLR` (Default: false) Use CoreCLR runtime pack (Microsoft.NETCore.App.Runtime.browser-wasm) instead of the Mono runtime pack for WASM benchmarks. * `--noForcedGCs` Specifying would not forcefully induce any GCs. * `--noOverheadEvaluation` Specifying would not run the evaluation overhead iterations. * `--resume` (Default: false) Continue the execution if the last run was stopped. * `--help` Display this help screen. * `--version` Display version information. ================================================ FILE: docs/articles/guides/customizing-runtime.md ================================================ --- uid: docs.customizing-runtime name: Customizing Runtime --- # Customizing Runtime Currently, we have only information about customizing Mono in this section. If you want to customize .NET Core, read an article about @docs.toolchains. --- [!include[IntroCustomMono](../samples/IntroCustomMono.md)] [!include[IntroCustomMonoArguments](../samples/IntroCustomMonoArguments.md)] [!include[IntroEnvVars](../samples/IntroEnvVars.md)] [!include[IntroStaThread](../samples/IntroStaThread.md)] ================================================ FILE: docs/articles/guides/dotnet-new-templates.md ================================================ --- uid: docs.dotnet-new-templates name: BenchmarkDotNet templates --- # BenchmarkDotNet templates BenchmarkDotNet provides project templates to setup your benchmarks easily. The template exists for each major .NET language ([C#](https://learn.microsoft.com/dotnet/csharp/), [F#](https://learn.microsoft.com/dotnet/fsharp/) and [VB](https://learn.microsoft.com/dotnet/visual-basic/)) with equivalent features and structure. ## How to install the templates The templates requires the [.NET Core SDK](https://www.microsoft.com/net/download). Once installed, run the following command to install the templates: ```log dotnet new install BenchmarkDotNet.Templates ``` If you want to uninstall all BenchmarkDotNet templates: ```log dotnet new uninstall BenchmarkDotNet.Templates ``` The template is a nuget package distributed over nuget: [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates/). ## Basic usage To create a new C# benchmark library project from the template, run: ```log dotnet new benchmark ``` If you'd like to create F# or VB project, you can specify project language with `-lang` option: ```log dotnet new benchmark -lang F# dotnet new benchmark -lang VB ``` ## Project template specific options The template projects has five additional options - all of them are optional. By default a console app project targeting `net6.0` is created. This lets you run the benchmarks from console (`dotnet run`) or from your favorite IDE. The option `-f` or `--framework` changes the target framework: ```log dotnet new benchmark -f net472 ``` You can specify `--console-app=false` to create a class library project targeting `netstandard2.0` by default: ```log dotnet new benchmark --console-app=false ``` The option `-b` or `--benchmarkName` sets the name of the benchmark class: ```log dotnet new benchmark -b Md5VsSha256 ``` BenchmarkDotNet lets you create a dedicated configuration class (see [Configs](xref:docs.configs)) to customize the execution of your benchmarks. To create a benchmark project with a configuration class, use the option `-c` or `--config`: ```log dotnet new benchmark -c ``` The option `--no-restore` if specified, skips the automatic nuget restore after the project is created: ```log dotnet new benchmark --no-restore ``` Use the `-h` or `--help` option to display all possible arguments with a description and the default values: ```log dotnet new benchmark --help ``` ## How to run the benchmarks Please read [how to run your benchmarks](xref:docs.how-to-run). ## The relationship of BenchmarkDotNet and BenchmarkDotNet.Templates The version of the template nuget package is synced with the [BenchmarkDotNet](https://www.nuget.org/packages/BenchmarkDotNet/) package. For instance, the template version `0.11.5` is referencing [BenchmarkDotnet 0.11.15](https://www.nuget.org/packages/BenchmarkDotNet/0.11.5) - there is no floating version behavior. **Note**: This will maybe change when BenchmarkDotNet reaches `1.x`. ## References For more info about the `dotnet new` CLI, please read [the documentation](https://learn.microsoft.com/dotnet/core/tools/dotnet). ================================================ FILE: docs/articles/guides/getting-started.md ================================================ # Getting started To get started with BenchmarkDotNet, please follow these steps. ## Step 1. Create a project Create a new console application. ## Step 2. Installation Install BenchmarkDotNet via the NuGet package: [BenchmarkDotNet](https://www.nuget.org/packages/BenchmarkDotNet/) ```cmd > dotnet add package BenchmarkDotNet ``` Read more about BenchmarkDotNet NuGet packages: @docs.nuget ## Step 3. Design a benchmark Write a class with methods that you want to measure and mark them with the `Benchmark` attribute. In the following example, we compare [MD5](https://en.wikipedia.org/wiki/MD5) and [SHA256](https://en.wikipedia.org/wiki/SHA-2) cryptographic hash functions: ```cs using System; using System.Security.Cryptography; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Running; namespace MyBenchmarks { public class Md5VsSha256 { private const int N = 10000; private readonly byte[] data; private readonly SHA256 sha256 = SHA256.Create(); private readonly MD5 md5 = MD5.Create(); public Md5VsSha256() { data = new byte[N]; new Random(42).NextBytes(data); } [Benchmark] public byte[] Sha256() => sha256.ComputeHash(data); [Benchmark] public byte[] Md5() => md5.ComputeHash(data); } public class Program { public static void Main(string[] args) { var summary = BenchmarkRunner.Run(); } } } ``` The `BenchmarkRunner.Run()` call runs your benchmarks and prints results to the console. ## Step 4. Run benchmarks Start your console application to run the benchmarks. The application must be built in the Release configuration. ```cmd > dotnet run -c Release ``` ## Step 5. View results View the results. Here is an example of output from the above benchmark: ``` BenchmarkDotNet=v0.13.2, OS=Windows 10 (10.0.19045.2251) Intel Core i7-4770HQ CPU 2.20GHz (Haswell), 1 CPU, 8 logical and 4 physical cores .NET SDK=7.0.100 [Host] : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2 DefaultJob : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2 | Method | Mean | Error | StdDev | |------- |---------:|---------:|---------:| | Sha256 | 51.57 us | 0.311 us | 0.291 us | | Md5 | 21.91 us | 0.138 us | 0.129 us | ``` ## Step 6. Analyze results BenchmarkDotNet will automatically create basic reports in the `.\BenchmarkDotNet.Artifacts\results` folder that can be shared and analyzed. To help analyze performance in further depth, you can configure your benchmark to collect and output more detailed information. Benchmark configuration can be conveniently changed by adding attributes to the class containing your benchmarks. For example: * [Diagnosers](../configs/diagnosers.md) * GC allocations: `[MemoryDiagnoser]` * Code size and disassembly: `[DisassemblyDiagnoser]` * Threading statistics: `[ThreadingDiagnoser]` * [Exporters](../configs/exporters.md) * CSV reports with raw data: `[CsvMeasurementsExporter]` * JSON reports with raw data: `[JsonExporter]` * Plots (if you have installed R): `[RPlotExporter]` For more information, see [Configs](../configs/configs.md). ## Next steps BenchmarkDotNet provides features which aid high-quality performance research. If you want to know more about BenchmarkDotNet features, check out the [Overview](../overview.md) page. If you have any questions, check out the [FAQ](../faq.md) page. If you didn't find an answer for your question on this page, [ask it on Gitter](https://gitter.im/dotnet/BenchmarkDotNet) or [create an issue on GitHub](https://github.com/dotnet/BenchmarkDotNet/issues). ================================================ FILE: docs/articles/guides/good-practices.md ================================================ # Good Practices ## Use the Release build without an attached debugger Never use the Debug build for benchmarking. *Never*. The debug version of the target method can run 10–100 times slower. The release mode means that you should have `true` in your csproj file or use [/optimize](https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/) for `csc`. Also, never use an attached debugger (e.g. Visual Studio or WinDbg) during the benchmarking. The best way is build our benchmark in the Release mode and run it from the command line. ## Try different environments Please, don't extrapolate your results. Or do it very carefully. I remind you again: the results in different environments may vary significantly. If a `Foo1` method is faster than a `Foo2` method for CLR4, .NET Framework 4.5, x64, RyuJIT, Windows, it means that the `Foo1` method is faster than the `Foo2` method for CLR4, .NET Framework 4.5, x64, RyuJIT, Windows and nothing else. And you can not say anything about methods performance for CLR 2 or .NET Framework 4.6 or LegacyJIT-x64 or x86 or Linux+Mono until you try it. ## Avoid dead code elimination You should also use the result of calculation. For example, if you run the following code: ```cs void Foo() { Math.Exp(1); } ``` then JIT can eliminate this code because the result of `Math.Exp` is not used. The better way is use it like this: ```cs double Foo() { return Math.Exp(1); } ``` ## Power settings and other applications * Turn off all of the applications except the benchmark process and the standard OS processes. If you run benchmark and work in the Visual Studio at the same time, it can negatively affect to benchmark results. * If you use laptop for benchmarking, keep it plugged in and use the maximum performance mode. ================================================ FILE: docs/articles/guides/how-it-works.md ================================================ # How it works BenchmarkDotNet follows the following steps to run your benchmarks: 1. `BenchmarkRunner` generates an isolated project per each runtime settings and builds it in Release mode. 2. Next, we take each method/job/params combination and try to measure its performance by launching benchmark process several times (`LaunchCount`). 3. An invocation of the workload method is an *operation*. A bunch of operation is an *iteration*. If you have an `IterationSetup` method, it will be invoked before each iteration, but not between operations. We have the following type of iterations: * `Jitting`: The overhead/workload methods are invoked to ensure they are JIT-compiled (and on tiered runtimes, to promote them when possible). These iterations are not used for measurements. * `Pilot`: The best operation count will be chosen. * `OverheadWarmup`, `OverheadWorkload`: BenchmarkDotNet overhead will be evaluated. * `ActualWarmup`: Warmup of the workload method. * `ActualWorkload`: Actual measurements. * `Result` = `ActualWorkload` - `` 4. After all of the measurements, BenchmarkDotNet creates: * An instance of the `Summary` class that contains all information about benchmark runs. * A set of files that contains summary in human-readable and machine-readable formats. * A set of plots. ## Pseudocode If you don't understand our "count terminology", then you might find following pseudocode useful: ```cs IEnumerable Run(Benchmark benchmark) { var toolchain = benchmark.GetToolchain(); var autoGeneratedProject = toolchain.Generate(benchmark); var exe = toolchain.Build(autoGeneratedProject); foreach (var runIndex in LaunchCount) // LaunchCount = 1 by default yield return ParseResults(Process.Start(exe).Output); // calls ActualRun in a separate process } Result ActualRun(Method method, Job job) { GlobalSetup(); JittingStage(method); // triggers JIT compilation (and tiering if enabled) before Pilot/Warmup int unrollFactor = job.Run.UnrollFactor; // 16 by default long perfectInvocationCount = Pilot(method, unrollFactor); WarmupStage(EMPTY_METHOD, perfectInvocationCount, unrollFactor); // EMPTY_METHOD has same return type and arguments as benchmark var overhead = ActualStage(EMPTY_METHOD, perfectInvocationCount, unrollFactor); WarmupStage(method, perfectInvocationCount, unrollFactor); var result = ActualStage(method, perfectInvocationCount); if (MemoryDiagnoser.IsEnabled) var gcStats = MeasureGcStats(method, perfectInvocationCount, unrollFactor); GlobalCleanup(); return (result - Median(overhead), gcStats); } void JittingStage(Method method) { RunIteration(method, invokeCount: 1, unrollFactor: 1); if (JitInfo.IsTiered) { for (int i = 0; i < JitInfo.MaxTierPromotions; i++) { RunIteration(method, invokeCount: JitInfo.TieredCallCountThreshold, unrollFactor: 1); Thread.Sleep(250); } } } long Pilot(Method method, int unrollFactor) { // invokeCount is the equivalent of InnerIterationCount from xunit-performance long invokeCount = minInvokeCount; while (true) { var measurement = RunIteration(method, invokeCount, unrollFactor); if (heuristic.IsPilotRequirementMet(measurement)) break; invokeCount *= 2; } return invokeCount; } void Warmup(Method method, long invokeCount, int unrollFactor) { while (true) { var measurement = RunIteration(method, invokeCount, unrollFactor); if (heuristic.IsWarmupRequirementMet(measurement)) break; } } IEnuberable Workload(Method method, long invokeCount, int unrollFactor) { while (true) { var measurement = RunIteration(method, invokeCount, unrollFactor); if (measurement.IsNotOutlier) yield return measurement; if (heuristic.IsWorkloadRequirementMet(measurement)) yield break; } } // every iteration invokes the method (invokeCount / unrollFactor) times Measurement RunIteration(Method method, long invokeCount, long unrollFactor) { IterationSetup(); MemoryCleanup(); var clock = Clock.Start(); for (long i = 0; i < invokeCount / unrollFactor; i++) { // we perform manual loop unrolling!! method(); // 1st call method(); // 2nd call method(); // (unrollFactor - 1)'th call method(); // unrollFactor'th call } var clockSpan = clock.GetElapsed(); IterationCleanup(); MemoryCleanup(); return Measurement(clockSpan); } GcStats MeasureGcStats(Method method, long invokeCount, long unrollFacto) { // we enable monitoring after workload actual run, for this single iteration which is executed at the end // so even if we enable AppDomain monitoring in separate process // it does not matter, because we have already obtained the results! EnableMonitoring(); IterationSetup(); var initialGcStats = GcStats.ReadInitial(); // we do NOT start any clock here, because the enabled monitoring might have some overhead // so we just get the gc stats and ignore the timing // it's last thing the process does before it dies, so also enabled monitoring is not an issue for next benchmarks // because each of them is going to be executed in a new process for (long i = 0; i < invokeCount / unrollFactor; i++) { // we perform manual loop unrolling!! method(); // 1st call method(); // 2nd call method(); // (unrollFactor - 1)'th call method(); // unrollFactor'th call } var finalGcStats = GcStats.ReadFinal(); IterationCleanup(); return finalGcStats - initialGcStats; // the result is the difference between the stats collected after and before running the extra iteration } ``` ================================================ FILE: docs/articles/guides/how-to-run.md ================================================ --- uid: docs.how-to-run --- # How to run your benchmarks There are several ways to run your benchmarks. What is important is that **BenchmarkDotNet works only with Console Apps**. It does not support any other kind of application like ASP.NET, Azure WebJobs, etc. ## Types If you have just a few types with benchmarks, you can use `BenchmarkRunner`: ```cs var summary = BenchmarkRunner.Run(); var summary = BenchmarkRunner.Run(typeof(MyBenchmarkClass)); ``` The disadvantage of `BenchmarkRunner` is that it always runs all benchmarks in a given type (or assembly) and to change the type you need to modify the source code. But it's great for a quick start. ## BenchmarkSwitcher If you have more types and you want to choose which benchmark to run (either by using console line arguments or console input) you should use `BenchmarkSwitcher`: ```cs static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); ``` Also you can use the config command style to specify some config from command line (more [](xref:docs.console-args)): ```log dotnet run -c Release -- --job short --runtimes net472 net7.0 --filter *BenchmarkClass1* ``` The most important thing about `BenchmarkSwitcher` is that you need to pass the `args` from `Main` to the `Run` method. If you don't, it won't parse the arguments. ================================================ FILE: docs/articles/guides/nuget.md ================================================ --- uid: docs.nuget name: Installing NuGet packages --- # Installing NuGet packages ## Packages We have the following set of NuGet packages (you can install it directly from `nuget.org`): * `BenchmarkDotNet`: BenchmarkDotNet infrastructure and logic. This is all you need to run benchmarks. * `BenchmarkDotNet.Annotations`: Basic BenchmarkDotNet annotations for your benchmarks. * `BenchmarkDotNet.Diagnostics.Windows`: an additional optional package that provides a set of Windows diagnosers. * `BenchmarkDotNet.Diagnostics.dotTrace`: an additional optional package that provides DotTraceDiagnoser. * `BenchmarkDotNet.Diagnostics.dotMemory`: an additional optional package that provides DotMemoryDiagnoser. * `BenchmarkDotNet.Templates`: Templates for BenchmarkDotNet. You might find other NuGet packages that start with `BenchmarkDotNet` name, but they are internal BDN packages that should not be installed manually. All that matters are the three packages mentioned above. ## Versioning system and feeds We have 3 kinds of versions: *stable*, *nightly*, and *develop*. You can get the current version from the source code via `BenchmarkDotNetInfo.FullVersion` and the full title via `BenchmarkDotNetInfo.FullTitle`. ### Stable These versions are available from the official NuGet feed. ```xml ``` ### Nightly If you want to use a nightly version of the BenchmarkDotNet, add the `https://www.myget.org/F/benchmarkdotnet/api/v3/index.json` feed in the `` section of your `NuGet.config`: ```xml ``` Now you can install the packages from the `bdn-nightly` feed. ### Develop You also can build BenchmarkDotNet from source code: ```sh build.cmd pack ``` ================================================ FILE: docs/articles/guides/toc.yml ================================================ - name: Getting Started href: getting-started.md - name: How to run your benchmarks href: how-to-run.md - name: Good Practices href: good-practices.md - name: Installing NuGet packages href: nuget.md - name: Choosing RunStrategy href: choosing-run-strategy.md - name: Customizing runtime href: customizing-runtime.md - name: How it works href: how-it-works.md - name: Console Arguments href: console-args.md - name: Troubleshooting href: troubleshooting.md - name: BenchmarkDotNet templates href: dotnet-new-templates.md ================================================ FILE: docs/articles/guides/troubleshooting.md ================================================ # Troubleshooting ## BenchmarkDotNet You need to be aware of the fact that to ensure process-level isolation BenchmarkDotNet generates, builds and executes every benchmark in a dedicated process. For .NET and Mono we generate a C# file and compile it using Roslyn. For .NET Core and NativeAOT we generate not only C# file but also a project file which later is restored and build with dotnet cli. If your project has some non-trivial build settings like a `.props` and `.target` files or native dependencies things might not work well out of the box. How do you know that BenchmarkDotNet has failed to build the project? BDN is going to tell you about it. An example: ```log // Validating benchmarks: // ***** BenchmarkRunner: Start ***** // ***** Found 1 benchmark(s) in total ***** // ***** Building 1 exe(s) in Parallel: Start ***** // start dotnet restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\netcoreapp2.1\c6045772-d3c7-4dbe-ab37-4aca6dcb6ec4 // command took 0.51s and exited with 1 // ***** Done, took 00:00:00 (0.66 sec) ***** // Found 1 benchmarks: // IntroBasic.Sleep: DefaultJob // Build Error: Standard output: Standard error: C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\netcoreapp2.1\c6045772-d3c7-4dbe-ab37-4aca6dcb6ec4\BenchmarkDotNet.Autogenerated.csproj(36,1): error MSB4025: The project file could not be loaded. Unexpected end of file while parsing Comment has occurred. Line 36, position 1. // BenchmarkDotNet has failed to build the auto-generated boilerplate code. // It can be found in C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\netcoreapp2.1\c6045772-d3c7-4dbe-ab37-4aca6dcb6ec4 // Please follow the troubleshooting guide: https://benchmarkdotnet.org/articles/guides/troubleshooting.html ``` If the error message is not clear enough, you need to investigate it further. How to troubleshoot the build process: 1. Run the benchmarks with `--logBuildOutput` command line argument. 2. Read the error message. If it does not contain the answer to your problem, please continue to the next step. 3. Go to the build artifacts folder (path printed by BDN). 4. The folder should contain: * a file with source code (ends with `.notcs` to make sure IDE don't include it in other projects by default) * a project file (`.csproj`) * a script file (`.bat` on Windows, `.sh` for other OSes) which should be doing exactly the same thing as BDN does: * dotnet restore * dotnet build (with some parameters like `-c Release`) 5. Run the script, read the error message. From here you continue with the troubleshooting like it was a project in your solution. The recommended order of solving build issues: 1. Change the right settings in your project file which defines benchmarks to get it working. 2. Customize the `Job` settings using available options like `job.WithCustomBuildConfiguration($name)`or `job.With(new Argument[] { new MsBuildArgument("/p:SomeProperty=Value")})`. 3. Implement your own `IToolchain` and generate and build all the right things in your way (you can use existing Builders and Generators and just override some methods to change specific behaviour). 4. Report a bug in BenchmarkDotNet repository. ## Debugging Benchmarks ## In the same process If your benchmark builds but fails to run, you can simply debug it. The first thing you should try is to do it in a single process (host process === runner process). 1. Use `DebugInProcessConfig` ```cs static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new DebugInProcessConfig()); ``` 2. Set the breakpoints in your favorite IDE 3. Start debugging the project with benchmarks ## In a different process Sometimes you won't be able to reproduce the problem in the same process. In this case, you have 3 options: ### Launch a debugger from the benchmark process using Debugger API ```cs [GlobalSetup] public void Setup() { System.Diagnostics.Debugger.Launch(); } ``` ### Attach a debugger from IDE Modify your benchmark to sleep until the Debugger is not attached and use your favorite IDE to attach the debugger to benchmark process. **Do attach to the process which is running the benchmark** (the arguments of the process are going to be `--benchmarkId $someNumber --benchmarkName $theName`), not the host process. ```cs [GlobalSetup] public void Setup() { while(!System.Diagnostics.Debugger.IsAttached) Thread.Sleep(TimeSpan.FromMilliseconds(100)); } ``` ### One of the above, but with a Debug build By default, BDN builds everything in Release. But debugging Release builds even with full symbols might be non-trivial. To enforce BDN to build the benchmark in Debug please use `DebugBuildConfig` and then attach the debugger. ================================================ FILE: docs/articles/license.md ================================================ ### The MIT License Copyright (c) 2013–2025 .NET Foundation and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: docs/articles/overview.md ================================================ --- uid: docs.overview name: Overview --- # Overview ## Install Create new console application and install the [BenchmarkDotNet](https://www.nuget.org/packages/BenchmarkDotNet/) NuGet package. We support: * *Projects:* classic and modern with PackageReferences * *Runtimes:* Full .NET Framework (4.6+), .NET Core (2.0+), Mono, NativeAOT * *OS:* Windows, Linux, MacOS * *Languages:* C#, F#, VB ## Design a benchmark Create a new console application, write a class with methods that you want to measure, and mark them with the `Benchmark` attribute. In the following example, we compare the [MD5](https://en.wikipedia.org/wiki/MD5) and [SHA256](https://en.wikipedia.org/wiki/SHA-2) cryptographic hash functions: ```cs using System; using System.Security.Cryptography; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Running; namespace MyBenchmarks { public class Md5VsSha256 { private const int N = 10000; private readonly byte[] data; private readonly SHA256 sha256 = SHA256.Create(); private readonly MD5 md5 = MD5.Create(); public Md5VsSha256() { data = new byte[N]; new Random(42).NextBytes(data); } [Benchmark] public byte[] Sha256() => sha256.ComputeHash(data); [Benchmark] public byte[] Md5() => md5.ComputeHash(data); } public class Program { public static void Main(string[] args) { var summary = BenchmarkRunner.Run(typeof(Program).Assembly); } } } ``` The `BenchmarkRunner.Run(typeof(Program).Assembly)` call runs all benchmarks in the current assembly and prints results to the console. If you'd like to run a single benchmark type, you can use `BenchmarkRunner.Run()`. BenchmarkDotNet discovers benchmark types via reflection, so benchmarks can be placed in different files as long as they are in the same project/assembly and the benchmark classes are `public`. Note that BenchmarkDotNet will only run benchmarks if the application is built in the Release configuration. This is to prevent unoptimized code from being benchmarked. BenchmarkDotNet will issue an error if you forget to change the configuration. ## Benchmark results ``` BenchmarkDotNet=v0.13.2, OS=Windows 10 (10.0.19045.2251) Intel Core i7-4770HQ CPU 2.20GHz (Haswell), 1 CPU, 8 logical and 4 physical cores .NET SDK=7.0.100 [Host] : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2 DefaultJob : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2 | Method | Mean | Error | StdDev | |------- |---------:|---------:|---------:| | Sha256 | 51.57 us | 0.311 us | 0.291 us | | Md5 | 21.91 us | 0.138 us | 0.129 us | ``` ## Jobs BenchmarkDotNet can benchmark your code in several environments at once. For example, to compare your benchmark's performance in .NET Framework, .NET Core, Mono and NativeAOT, you can add `SimpleJob` attributes to the benchmark class: ```cs [SimpleJob(RuntimeMoniker.Net481)] [SimpleJob(RuntimeMoniker.Net70)] [SimpleJob(RuntimeMoniker.NativeAot70)] [SimpleJob(RuntimeMoniker.Mono)] public class Md5VsSha256 ``` Example of the result: ``` BenchmarkDotNet=v0.13.2, OS=Windows 10 (10.0.19045.2251) Intel Core i7-4770HQ CPU 2.20GHz (Haswell), 1 CPU, 8 logical and 4 physical cores .NET SDK=7.0.100 [Host] : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2 .NET 7.0 : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2 .NET Framework 4.8.1 : .NET Framework 4.8.1 (4.8.9037.0), X64 RyuJIT VectorSize=256 Mono : Mono 6.12.0 (Visual Studio), X64 VectorSize=128 NativeAOT 7.0 : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2 | Method | Job | Runtime | Mean | Error | StdDev | |------- |--------------------- |--------------------- |----------:|---------:|---------:| | Sha256 | .NET 7.0 | .NET 7.0 | 51.90 us | 0.341 us | 0.302 us | | Md5 | .NET 7.0 | .NET 7.0 | 21.96 us | 0.052 us | 0.049 us | | Sha256 | .NET Framework 4.8.1 | .NET Framework 4.8.1 | 206.33 us | 2.069 us | 1.834 us | | Md5 | .NET Framework 4.8.1 | .NET Framework 4.8.1 | 23.28 us | 0.094 us | 0.083 us | | Sha256 | Mono | Mono | 167.70 us | 1.216 us | 1.137 us | | Md5 | Mono | Mono | 42.12 us | 0.145 us | 0.136 us | | Sha256 | NativeAOT 7.0 | NativeAOT 7.0 | 51.45 us | 0.226 us | 0.200 us | | Md5 | NativeAOT 7.0 | NativeAOT 7.0 | 21.88 us | 0.050 us | 0.041 us | ``` There are many predefined job attributes which you can use. For example, you can compare `LegacyJitX86`, `LegacyJitX64`, and `RyuJitX64`: ```cs [LegacyJitX86Job, LegacyJitX64Job, RyuJitX64Job] ``` Or, you can define your own jobs: ```cs [Config(typeof(Config))] public class Md5VsSha256 { private class Config : ManualConfig { public Config() { AddJob(new Job(Job.Dry) { Environment = { Jit = Jit.LegacyJit, Platform = Platform.X64 }, Run = { LaunchCount = 3, WarmupCount = 5, IterationCount = 10 }, Accuracy = { MaxRelativeError = 0.01 } }); } } ``` Read more: [Jobs](configs/jobs.md), [Configs](configs/configs.md) ## Columns You can add columns to the summary table: ```cs [MinColumn, MaxColumn] public class Md5VsSha256 ``` | Method | Median | StdDev | Min | Max | | ------ | ----------- | --------- | ----------- | ----------- | | Sha256 | 131.3200 us | 4.6744 us | 129.8216 us | 147.7630 us | | Md5 | 26.2847 us | 0.4424 us | 25.8442 us | 27.4258 us | You can also define custom columns based on the full benchmark summary. Read more: [Columns](configs/columns.md) ## Exporters You can export the results of your benchmark in different formats: ```cs [MarkdownExporter, AsciiDocExporter, HtmlExporter, CsvExporter, RPlotExporter] public class Md5VsSha256 ``` If you have installed R, `RPlotExporter` will generate a lot of nice plots: ![](../images/v0.12.0/rplot.png) Read more: [Exporters](configs/exporters.md) ## Baseline To view the relative performance of your benchmarks, mark one of your benchmark methods as the `Baseline`: ```cs public class Sleeps { [Benchmark] public void Time50() => Thread.Sleep(50); [Benchmark(Baseline = true)] public void Time100() => Thread.Sleep(100); [Benchmark] public void Time150() => Thread.Sleep(150); } ``` A new column will be added to the summary table: | Method | Median | StdDev | Ratio | | ------- | ----------- | --------- | ------ | | Time100 | 100.2640 ms | 0.1238 ms | 1.00 | | Time150 | 150.2093 ms | 0.1034 ms | 1.50 | | Time50 | 50.2509 ms | 0.1153 ms | 0.50 | Read more: [Baselines](features/baselines.md) ## Params You can mark one or several fields or properties in your class with the `Params` attribute. In this attribute, you can specify a set of values. BenchmarkDotNet will run benchmarks for each combination of params values. ```cs public class IntroParams { [Params(100, 200)] public int A { get; set; } [Params(10, 20)] public int B { get; set; } [Benchmark] public void Benchmark() { Thread.Sleep(A + B + 5); } } ``` | Method | Median | StdDev | A | B | | --------- | ----------- | --------- | ---- | ---- | | Benchmark | 115.3325 ms | 0.0242 ms | 100 | 10 | | Benchmark | 125.3282 ms | 0.0245 ms | 100 | 20 | | Benchmark | 215.3024 ms | 0.0375 ms | 200 | 10 | | Benchmark | 225.2710 ms | 0.0434 ms | 200 | 20 | Read more: [Parameterization](features/parameterization.md) ## Languages You can also write benchmarks in `F#` or `VB`. ```fs type StringKeyComparison () = let mutable arr : string [] = [||] let dict1 = ConcurrentDictionary<_,_>() let dict2 = ConcurrentDictionary<_,_>(StringComparer.Ordinal) [] member val public DictSize = 0 with get, set [] member self.GlobalSetupData() = dict1.Clear(); dict2.Clear() arr <- getStrings self.DictSize arr |> Array.iter (fun x -> dict1.[x] <- true ; dict2.[x] <- true) [] member self.StandardLookup () = lookup arr dict1 [] member self.OrdinalLookup () = lookup arr dict2 ``` ```vb Public Class Sample Public Property A As Integer Public Property B As Integer Public Function Benchmark() As Integer return A + B End Function End Class ``` ## Diagnostics A diagnoser can attach to your benchmarks and collect additional information. Examples of diagnosers built in to BenchmarkDotNet are: - Garbge collection and allocation statistics (`MemoryDiagnoser`). - Lock contention and thread pool statistics (`ThreadingDiagnoser`), which is only available on .NET Core 3.0+. - JIT inlining events (`InliningDiagnoser`). You can find this diagnoser in a separated package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) Below is a sample benchmark using `MemoryDiagnoser`. Note the extra columns on the right-hand side (`Gen 0` and `Allocated`): ``` Method | Mean | StdDev | Gen 0 | Allocated | ---------- |----------- |---------- |------- |---------- | Iterative | 31.0739 ns | 0.1091 ns | - | 0 B | LINQ | 83.0435 ns | 1.0103 ns | 0.0069 | 32 B | ``` Read more: [Diagnosers](configs/diagnosers.md) ## BenchmarkRunner There are several ways to run your benchmarks. ```cs var summary = BenchmarkRunner.Run(); var summary = BenchmarkRunner.Run(typeof(MyBenchmarkClass)); var summaries = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); ``` Read more: [How to run your benchmarks](guides/how-to-run.md) ================================================ FILE: docs/articles/samples/IntroArguments.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroArguments --- ## Sample: IntroArguments As an alternative to using [`[Params]`](xref:BenchmarkDotNet.Attributes.ParamsAttribute), you can specify arguments for your benchmarks. There are several ways to do it (described below). The [`[Arguments]`](xref:BenchmarkDotNet.Attributes.ArgumentsAttribute) allows you to provide a set of values. Every value must be a compile-time constant (it's C# language limitation for attributes in general). You can also combine [`[Arguments]`](xref:BenchmarkDotNet.Attributes.ArgumentsAttribute) with [`[Params]`](xref:BenchmarkDotNet.Attributes.ParamsAttribute). As a result, you will get results for each combination of params values. ### Source code [!code-csharp[IntroArguments.cs](../../../samples/BenchmarkDotNet.Samples/IntroArguments.cs)] ### Output ```markdown | Method | AddExtra5Miliseconds | a | b | Mean | Error | StdDev | |---------- |--------------------- |---- |--- |---------:|----------:|----------:| | Benchmark | False | 100 | 10 | 110.1 ms | 0.0056 ms | 0.0044 ms | | Benchmark | False | 100 | 20 | 120.1 ms | 0.0155 ms | 0.0138 ms | | Benchmark | False | 200 | 10 | 210.2 ms | 0.0187 ms | 0.0175 ms | | Benchmark | False | 200 | 20 | 220.3 ms | 0.1055 ms | 0.0986 ms | | Benchmark | True | 100 | 10 | 115.3 ms | 0.1375 ms | 0.1286 ms | | Benchmark | True | 100 | 20 | 125.3 ms | 0.1212 ms | 0.1134 ms | | Benchmark | True | 200 | 10 | 215.4 ms | 0.0779 ms | 0.0691 ms | | Benchmark | True | 200 | 20 | 225.4 ms | 0.0775 ms | 0.0725 ms | ``` ### Links * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroArguments --- ================================================ FILE: docs/articles/samples/IntroArgumentsPriority.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroArgumentsPriority --- ## Sample: IntroArgumentsPriority Like Params also Argument columns can be sorted in the table result through their `Priority`. The priority should be defined only once for multiple Arguments and will keep their inner order as they are defined in the method. ### Source code [!code-csharp[IntroArgumentsPriority.cs](../../../samples/BenchmarkDotNet.Samples/IntroArgumentsPriority.cs)] ### Output ```markdown | Method | b | A | c | d | Mean | Error | StdDev | |-------------- |--- |---- |-- |-- |---------:|--------:|--------:| | ManyArguments | ? | 100 | 1 | 2 | 103.4 ms | 0.09 ms | 0.08 ms | | Benchmark | 5 | 100 | ? | ? | 105.5 ms | 0.21 ms | 0.19 ms | | Benchmark | 10 | 100 | ? | ? | 110.5 ms | 0.14 ms | 0.14 ms | | Benchmark | 20 | 100 | ? | ? | 120.4 ms | 0.16 ms | 0.15 ms | ``` ### Links * Priority BaseClass [`PriorityAttribute.cs`](xref:BenchmarkDotNet.Attributes.PriorityAttribute) * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroArgumentsPriority --- ================================================ FILE: docs/articles/samples/IntroArgumentsSource.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroArgumentsSource --- ## Sample: IntroArgumentsSource In case you want to use a lot of values, you should use [`[ArgumentsSource]`](xref:BenchmarkDotNet.Attributes.ArgumentsSourceAttribute). You can mark one or several fields or properties in your class by the [`[ArgumentsSource]`](xref:BenchmarkDotNet.Attributes.ArgumentsSourceAttribute) attribute. In this attribute, you have to specify the name of public method/property which is going to provide the values (something that implements `IEnumerable`). The source may be instance or static. If the source is not in the same type as the benchmark, the type containing the source must be specified in the attribute constructor. ### Source code [!code-csharp[IntroArgumentsSource.cs](../../../samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs)] ### Output ```markdown | Method | time | x | y | Mean | Error | StdDev | |--------------- |------------------ |--- |--- |----------------:|---------------:|---------------:| | SingleArgument | 00:00:00.0100000 | ? | ? | 15,780,658.9 ns | 53,493.3 ns | 50,037.7 ns | | SingleArgument | 00:00:00.1000000 | ? | ? | 110,181,308.0 ns | 517,614.4 ns | 484,176.8 ns | | ManyArguments | ? | 1 | 1 | 3.135 ns | 0.0852 ns | 0.1326 ns | | ManyArguments | ? | 2 | 2 | 13.571 ns | 0.2180 ns | 0.1933 ns | | ManyArguments | ? | 4 | 4 | 13.478 ns | 0.2188 ns | 0.1940 ns | | ManyArguments | ? | 10 | 10 | 13.471 ns | 0.2294 ns | 0.2034 ns | ``` > `?` is displayed when a column is not applicable for the given benchmark (e.g., `x`/`y` for `SingleArgument`, `time` for `ManyArguments`). ### Another example If the values are complex types you need to override `ToString` method to change the display names used in the results. ```cs [DryJob] public class WithNonPrimitiveArgumentsSource { [Benchmark] [ArgumentsSource(nameof(NonPrimitive))] public void Simple(SomeClass someClass, SomeStruct someStruct) { for (int i = 0; i < someStruct.RangeEnd; i++) Console.WriteLine($"// array.Values[{i}] = {someClass.Values[i]}"); } public IEnumerable NonPrimitive() { yield return new object[] { new SomeClass(Enumerable.Range(0, 10).ToArray()), new SomeStruct(10) }; yield return new object[] { new SomeClass(Enumerable.Range(0, 15).ToArray()), new SomeStruct(15) }; } public class SomeClass { public SomeClass(int[] initialValues) => Values = initialValues.Select(val => val * 2).ToArray(); public int[] Values { get; } public override string ToString() => $"{Values.Length} items"; } public struct SomeStruct { public SomeStruct(int rangeEnd) => RangeEnd = rangeEnd; public int RangeEnd { get; } public override string ToString() => $"{RangeEnd}"; } } ``` ```markdown | Method | someClass | someStruct | Mean | Error | |------- |---------- |----------- |---------:|------:| | Simple | 10 items | 10 | 887.2 us | NA | | Simple | 15 items | 15 | 963.1 us | NA | ``` ### Links * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroArgumentsSource --- ================================================ FILE: docs/articles/samples/IntroArrayParam.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroArrayParam --- ## Sample: IntroArrayParam > [!WARNING] > The cost of creating the arguments is not included in the benchmark. So if you want to pass an array as an argument, we are going to allocate it before running the benchmark, and the benchmark will not include this operation. ### Source code [!code-csharp[IntroArrayParam.cs](../../../samples/BenchmarkDotNet.Samples/IntroArrayParam.cs)] ### Output ```markdown | Method | array | value | Mean | Error | StdDev | Allocated | |-------------- |----------- |------ |----------:|----------:|----------:|----------:| | ArrayIndexOf | Array[100] | 4 | 15.558 ns | 0.0638 ns | 0.0597 ns | 0 B | | ManualIndexOf | Array[100] | 4 | 5.345 ns | 0.0668 ns | 0.0625 ns | 0 B | | ArrayIndexOf | Array[3] | 4 | 14.334 ns | 0.1758 ns | 0.1558 ns | 0 B | | ManualIndexOf | Array[3] | 4 | 2.758 ns | 0.0905 ns | 0.1208 ns | 0 B | | ArrayIndexOf | Array[100] | 101 | 78.359 ns | 1.8853 ns | 2.0955 ns | 0 B | | ManualIndexOf | Array[100] | 101 | 80.421 ns | 0.6391 ns | 0.5978 ns | 0 B | ``` ### Links * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroArrayParam --- ================================================ FILE: docs/articles/samples/IntroBasic.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroBasic title: "Sample: IntroBasic" --- ## Sample: IntroBasic ### Source code [!code-csharp[IntroBasic.cs](../../../samples/BenchmarkDotNet.Samples/IntroBasic.cs)] ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroBasic --- ================================================ FILE: docs/articles/samples/IntroBenchmarkBaseline.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroBenchmarkBaseline --- ## Sample: IntroBenchmarkBaseline You can mark a method as a baseline with the help of `[Benchmark(Baseline = true)]`. ### Source code [!code-csharp[IntroBenchmarkBaseline.cs](../../../samples/BenchmarkDotNet.Samples/IntroBenchmarkBaseline.cs)] ### Output As a result, you will have additional `Ratio` column in the summary table: ```markdown | Method | Mean | Error | StdDev | Ratio | |-------- |----------:|----------:|----------:|------:| | Time50 | 50.46 ms | 0.0779 ms | 0.0729 ms | 0.50 | | Time100 | 100.39 ms | 0.0762 ms | 0.0713 ms | 1.00 | | Time150 | 150.48 ms | 0.0986 ms | 0.0922 ms | 1.50 | ``` This column contains the mean value of the ratio distribution. For example, in the case of `Time50`, we divide the first measurement of `Time50` into each measurement of `Time100` (it's the baseline), the second measurement of `Time50` into each measurement of `Time100`, and so on. Next, we calculate the mean of all these values and display it in the `Ratio` column. For `Time50`, we have 0.50. The `Ratio` column was formerly known as `Scaled`. The old title was a source of misunderstanding and confusion because many developers interpreted it as the ratio of means (e.g., `50.46`/`100.39` for `Time50`). The ratio of distribution means and the mean of the ratio distribution are pretty close to each other in most cases, but they are not equal. In @BenchmarkDotNet.Samples.IntroRatioStdDev, you can find an example of how this value can be spoiled by outliers. ### Links * @docs.baselines * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroBenchmarkBaseline --- ================================================ FILE: docs/articles/samples/IntroCategories.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroCategories --- ## Sample: IntroCategories Combined together with `[BenchmarkCategory]` attribute, you can group the benchmarks into categories and filter them by categories. ### Source code [!code-csharp[IntroCategories.cs](../../../samples/BenchmarkDotNet.Samples/IntroCategories.cs)] ### Command line examples: ``` --allCategories=A,B --anyCategories=A,B ``` ### Links * @docs.filters * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroCategories --- ================================================ FILE: docs/articles/samples/IntroCategoryBaseline.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroCategoryBaseline --- ## Sample: IntroCategoryBaseline The only way to have several baselines in the same class is to separate them by categories and mark the class with `[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]`. ### Source code [!code-csharp[IntroCategoryBaseline.cs](../../../samples/BenchmarkDotNet.Samples/IntroCategoryBaseline.cs)] ### Output ```markdown | Method | Categories | Mean | Error | StdDev | Ratio | |-------- |----------- |----------:|----------:|----------:|------:| | Time50 | Fast | 50.46 ms | 0.0745 ms | 0.0697 ms | 1.00 | | Time100 | Fast | 100.47 ms | 0.0955 ms | 0.0893 ms | 1.99 | | | | | | | | | Time550 | Slow | 550.48 ms | 0.0525 ms | 0.0492 ms | 1.00 | | Time600 | Slow | 600.45 ms | 0.0396 ms | 0.0331 ms | 1.09 | ``` ### Links * @docs.baselines * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroCategoryBaseline --- ================================================ FILE: docs/articles/samples/IntroCategoryDiscoverer.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroCategoryDiscoverer --- ## Sample: IntroCategoryDiscoverer The category discovery strategy can be overridden using an instance of `ICategoryDiscoverer`. ### Source code [!code-csharp[IntroCategoryDiscoverer.cs](../../../samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs)] ### Output ```markdown | Method | Categories | Mean | Error | |------- |----------- |---------:|------:| | Bar | All,B | 126.5 us | NA | | Foo | All,F | 114.0 us | NA | ``` ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroCategoryDiscoverer --- ================================================ FILE: docs/articles/samples/IntroColdStart.md ================================================ --- #cspell:ignore runstrategy uid: BenchmarkDotNet.Samples.IntroColdStart --- ## Sample: IntroColdStart If you want to measure cold start (without the pilot and warmup stage), the `ColdStart` strategy is your choice. ### Usage ```cs [SimpleJob(RunStrategy.ColdStart, launchCount:50)] public class MyBenchmarkClass ``` ### Source code [!code-csharp[IntroColdStart.cs](../../../samples/BenchmarkDotNet.Samples/IntroColdStart.cs)] ### Output ```markdown Result 1: 1 op, 1002034900.00 ns, 1.0020 s/op Result 2: 1 op, 10219700.00 ns, 10.2197 ms/op Result 3: 1 op, 10406200.00 ns, 10.4062 ms/op Result 4: 1 op, 10473900.00 ns, 10.4739 ms/op Result 5: 1 op, 10449400.00 ns, 10.4494 ms/op ``` ```markdown Method | Mean | Error | StdDev | Min | Max | Median | ------- |---------:|-----------:|---------:|---------:|-----------:|---------:| Foo | 208.7 ms | 1,707.4 ms | 443.5 ms | 10.22 ms | 1,002.0 ms | 10.45 ms | ``` ### Links * @docs.runstrategy * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroColdStart --- ================================================ FILE: docs/articles/samples/IntroComparableComplexParam.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroComparableComplexParam --- ## Sample: IntroComparableComplexParam You can implement `IComparable` (the non generic version) on your complex parameter class if you want custom ordering behavior for your parameter. One use case for this is having a parameter class that overrides `ToString()`, but also providing a custom ordering behavior that isn't the alphabetical order of the result of `ToString()`. ### Source code [!code-csharp[IntroComparableComplexParam.cs](../../../samples/BenchmarkDotNet.Samples/IntroComparableComplexParam.cs)] ### Links * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroComparableComplexParam --- ================================================ FILE: docs/articles/samples/IntroConfigSource.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroConfigSource --- ## Sample: IntroConfigSource You can define own config attribute. ### Source code [!code-csharp[IntroConfigSource.cs](../../../samples/BenchmarkDotNet.Samples/IntroConfigSource.cs)] ### Links * @docs.configs * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroConfigSource --- ================================================ FILE: docs/articles/samples/IntroConfigUnion.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroConfigUnion --- ## Sample: IntroConfigUnion ### Source code [!code-csharp[IntroConfigUnion.cs](../../../samples/BenchmarkDotNet.Samples/IntroConfigUnion.cs)] ### Links * @docs.configs * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroConfigUnion --- ================================================ FILE: docs/articles/samples/IntroCustomMono.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroCustomMono --- ## Sample: IntroCustomMono BenchmarkDotNet allows you to compare different runtimes, including Mono. If you apply `[MonoJob]` attribute to your class we use your default mono runtime. If you want to compare different versions of Mono you need to provide use the custom paths. You can do this today by using the overloaded ctor of MonoJob attribute or by specifying the runtime in a fluent way. The mono runtime can also operate as an ahead-of-time compiler. Using mono's AOT mode requires providing the AOT compilation arguments, as well as the path to mono's corlib. (See IntroCustomMonoObjectStyleAot in the below example). ### Source code [!code-csharp[IntroCustomMono.cs](../../../samples/BenchmarkDotNet.Samples/IntroCustomMono.cs)] ### Links * @docs.customizing-runtime * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroCustomMono --- ================================================ FILE: docs/articles/samples/IntroCustomMonoArguments.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroCustomMonoArguments --- ## Sample: IntroCustomMonoArguments ### Source code [!code-csharp[IntroCustomMonoArguments.cs](../../../samples/BenchmarkDotNet.Samples/IntroCustomMonoArguments.cs)] ### Output ```markdown | Method | Job | Arguments | Mean | StdDev | |------- |------------------ |------------------- |-----------:|----------:| | Sample | Inlining disabled | --optimize=-inline | 19.4252 ns | 0.4525 ns | | Sample | Inlining enabled | --optimize=inline | 0.0000 ns | 0.0000 ns | ``` ### Links * @docs.customizing-runtime * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroCustomMonoArguments --- ================================================ FILE: docs/articles/samples/IntroDeferredExecution.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroDeferredExecution --- ## Sample: IntroDeferredExecution In LINQ, execution of a query is usually [deferred](https://learn.microsoft.com/dotnet/standard/linq/deferred-execution-example) until the moment when you actually request the data. If your benchmark just returns `IEnumerable` or `IQueryable` it's not measuring the execution of the query, just the creation. This is why we decided to warn you about this issue whenever it happens: ```log Benchmark IntroDeferredExecution.Wrong returns a deferred execution result (IEnumerable). You need to either change the method declaration to return a materialized result or consume it on your own. You can use .Consume() extension method to do that. ``` Don't worry! We are also providing you with a `Consume` extension method which can execute given `IEnumerable` or `IQueryable` and consume its results. All you need to do is to create a [`Consumer`](xref:BenchmarkDotNet.Engines.Consumer) instance, preferably store it in a field (to exclude the cost of creating Consumer from the benchmark itself) and pass it to `Consume` extension method. **Do not call `.ToArray()` because it's an expensive operation and it might dominate given benchmark!** ### Source code [!code-csharp[IntroDeferredExecution.cs](../../../samples/BenchmarkDotNet.Samples/IntroDeferredExecution.cs)] ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDeferredExecution --- ================================================ FILE: docs/articles/samples/IntroDisassembly.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroDisassembly --- ## Sample: IntroDisassembly ### Source code [!code-csharp[IntroDisassembly.cs](../../../samples/BenchmarkDotNet.Samples/IntroDisassembly.cs)] ### Output ```x86asm ; .NET Framework 4.7.2 (CLR 4.0.30319.42000), 32bit LegacyJIT-v4.7.3110.0 05452718 BenchmarkDotNet.Samples.IntroDisassembly.Sum() IL_0000: ldc.r8 0 IL_0009: stloc.0 0545271c d9ee fldz IL_000a: ldc.i4.0 IL_000b: stloc.1 IL_000c: br.s IL_0017 0545271e 33c0 xor eax,eax IL_000e: ldloc.0 IL_000f: ldloc.1 IL_0010: conv.r8 IL_0011: add IL_0012: stloc.0 05452720 8945fc mov dword ptr [ebp-4],eax 05452723 db45fc fild dword ptr [ebp-4] 05452726 dec1 faddp st(1),st IL_0013: ldloc.1 IL_0014: ldc.i4.1 IL_0015: add IL_0016: stloc.1 05452728 40 inc eax IL_0017: ldloc.1 IL_0018: ldc.i4.s 64 IL_001a: blt.s IL_000e 05452729 83f840 cmp eax,40h 0545272c 7cf2 jl 05452720 IL_001c: ldloc.0 IL_001d: ret 0545272e 8be5 mov esp,ebp ``` ```x86asm ; .NET Core 2.1.0 (CoreCLR 4.6.26515.07, CoreFX 4.6.26515.06), 64bit RyuJIT 00007ffa`6c621320 BenchmarkDotNet.Samples.IntroDisassembly.Sum() IL_0000: ldc.r8 0 IL_0009: stloc.0 00007ffa`6c621323 c4e17857c0 vxorps xmm0,xmm0,xmm0 IL_000a: ldc.i4.0 IL_000b: stloc.1 IL_000c: br.s IL_0017 00007ffa`6c621328 33c0 xor eax,eax IL_000e: ldloc.0 IL_000f: ldloc.1 IL_0010: conv.r8 IL_0011: add IL_0012: stloc.0 00007ffa`6c62132a c4e17057c9 vxorps xmm1,xmm1,xmm1 00007ffa`6c62132f c4e1732ac8 vcvtsi2sd xmm1,xmm1,eax 00007ffa`6c621334 c4e17b58c1 vaddsd xmm0,xmm0,xmm1 IL_0013: ldloc.1 IL_0014: ldc.i4.1 IL_0015: add IL_0016: stloc.1 00007ffa`6c621339 ffc0 inc eax IL_0017: ldloc.1 IL_0018: ldc.i4.s 64 IL_001a: blt.s IL_000e 00007ffa`6c62133b 83f840 cmp eax,40h 00007ffa`6c62133e 7cea jl 00007ffa`6c62132a IL_001c: ldloc.0 IL_001d: ret 00007ffa`6c621340 c3 ret ``` ```x86asm Mono 5.12.0 (Visual Studio), 64bit Sum sub $0x18,%rsp mov %rsi,(%rsp) xorpd %xmm0,%xmm0 movsd %xmm0,0x8(%rsp) xor %esi,%esi jmp 2e xchg %ax,%ax movsd 0x8(%rsp),%xmm0 cvtsi2sd %esi,%xmm1 addsd %xmm1,%xmm0 movsd %xmm0,0x8(%rsp) inc %esi cmp $0x40,%esi jl 18 movsd 0x8(%rsp),%xmm0 mov (%rsp),%rsi add $0x18,%rsp retq ``` ### Links * @docs.diagnosers * @docs.disassembler * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDisassembly --- ================================================ FILE: docs/articles/samples/IntroDisassemblyAllJits.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroDisassemblyAllJits --- ## Sample: IntroDisassemblyAllJits You can use a single config to compare the generated assembly code for ALL JITs. But to allow benchmarking any target platform architecture the project which defines benchmarks has to target **AnyCPU**. ```xml AnyCPU ``` ### Source code [!code-csharp[IntroDisassemblyAllJits.cs](../../../samples/BenchmarkDotNet.Samples/IntroDisassemblyAllJits.cs)] ### Output The disassembly result can be obtained [here](https://adamsitnik.com/files/disasm/Jit_Devirtualization-disassembly-report.html). The file was too big to embed it in this doc page. ### Links * @docs.diagnosers * @docs.disassembler * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDisassemblyAllJits --- ================================================ FILE: docs/articles/samples/IntroDisassemblyDry.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroDisassemblyDry --- ## Sample: IntroDisassemblyDry **Getting only the Disassembly without running the benchmarks for a long time.** Sometimes you might be interested only in the disassembly, not the results of the benchmarks. In that case you can use **Job.Dry** which runs the benchmark only **once**. ### Source code [!code-csharp[IntroDisassemblyDry.cs](../../../samples/BenchmarkDotNet.Samples/IntroDisassemblyDry.cs)] ### Links * @docs.diagnosers * @docs.disassembler * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDisassemblyDry --- ================================================ FILE: docs/articles/samples/IntroDisassemblyRyuJit.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroDisassemblyRyuJit --- ## Sample: IntroDisassemblyRyuJit ### Source code [!code-csharp[IntroDisassemblyRyuJit.cs](../../../samples/BenchmarkDotNet.Samples/IntroDisassemblyRyuJit.cs)] ### Output ![](../../images/disasm-demo.png) ### Links * @docs.diagnosers * @docs.disassembler * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDisassemblyRyuJit --- ================================================ FILE: docs/articles/samples/IntroDotMemoryDiagnoser.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser --- ## Sample: IntroDotMemoryDiagnoser If you want to get a memory allocation profile of your benchmarks, just add the `[DotMemoryDiagnoser]` attribute, as shown below. As a result, BenchmarkDotNet performs bonus benchmark runs using attached [dotMemory Command-Line Profiler](https://www.jetbrains.com/help/dotmemory/Working_with_dotMemory_Command-Line_Profiler.html). The obtained dotMemory workspaces are saved to the `artifacts` folder. These dotMemory workspaces can be opened using the [standalone dotMemory](https://www.jetbrains.com/dotmemory/), or [dotMemory in Rider](https://www.jetbrains.com/help/rider/Memory_profiling_of_.NET_code.html). ### Source code [!code-csharp[IntroDotMemoryDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs)] ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser --- ================================================ FILE: docs/articles/samples/IntroDotTraceDiagnoser.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroDotTraceDiagnoser --- ## Sample: IntroDotTraceDiagnoser If you want to get a performance profile of your benchmarks, just add the `[DotTraceDiagnoser]` attribute, as shown below. As a result, BenchmarkDotNet performs bonus benchmark runs using attached [dotTrace Command-Line Profiler](https://www.jetbrains.com/help/profiler/Performance_Profiling__Profiling_Using_the_Command_Line.html). The obtained snapshots are saved to the `artifacts` folder. These snapshots can be opened using the [standalone dotTrace](https://www.jetbrains.com/profiler/), or [dotTrace in Rider](https://www.jetbrains.com/help/rider/Performance_Profiling.html). ### Source code [!code-csharp[IntroDotTraceDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs)] ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDotTraceDiagnoser --- ================================================ FILE: docs/articles/samples/IntroEnvVars.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroEnvVars --- ## Sample: IntroEnvVars You can configure custom environment variables for the process that is running your benchmarks. One reason for doing this might be checking out how different [compilation](https://learn.microsoft.com/dotnet/core/runtime-config/compilation), [threading](https://learn.microsoft.com/dotnet/core/runtime-config/threading), [garbage collector](https://learn.microsoft.com/dotnet/core/runtime-config/garbage-collector) settings affect the performance of .NET Core. ### Source code [!code-csharp[IntroEnvVars.cs](../../../samples/BenchmarkDotNet.Samples/IntroEnvVars.cs)] ### Links * @docs.customizing-runtime * @docs.configs * @docs.jobs * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroEnvVars --- ================================================ FILE: docs/articles/samples/IntroEventPipeProfiler.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroEventPipeProfiler --- ## Sample: EventPipeProfiler The `EventPipeProfiler` can be enabled using the `[EventPipeProfiler(...)]` attribute. This attribute takes the following profiles: - `CpuSampling` - Useful for tracking CPU usage and general .NET runtime information. This is the default option. - `GcVerbose` - Tracks GC collections and samples object allocations. - `GcCollect` - Tracks GC collections only at very low overhead. - `Jit` - Logging when Just in time (JIT) compilation occurs. Logging of the internal workings of the Just In Time compiler. This is fairly verbose. It details decisions about interesting optimization (like inlining and tail call) ### Source code [!code-csharp[EventPipeProfiler.cs](../../../samples/BenchmarkDotNet.Samples/IntroEventPipeProfiler.cs)] ### Output The output should contain information about the exported trace file which can be analyzed using [SpeedScope](https://www.speedscope.app/). ```markdown // * Diagnostic Output - EventPipeProfiler * Exported 1 trace file(s). Example: C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\BenchmarkDotNet.Artifacts\BenchmarkDotNet.Samples.IntroEventPipeProfiler.Sleep-20200406-090113.speedscope.json ``` ================================================ FILE: docs/articles/samples/IntroEventPipeProfilerAdvanced.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroEventPipeProfilerAdvanced --- ## Sample: EventPipeProfilerAdvanced The most advanced and powerful way to use `EventPipeProfiler` is a custom configuration. As you can see the below configuration adds `EventPipeProfiler` that constructor can take the profile and/or a list of providers. Both `EventPipeProfiler` and `dotnet trace` use the `Microsoft.Diagnostics.NETCore.Client` package internally. So before you start using the custom configuration of this profiler, it is worth reading the documentation [here](https://github.com/dotnet/diagnostics/blob/main/documentation/dotnet-trace-instructions.md) and [here](https://learn.microsoft.com/dotnet/core/diagnostics/dotnet-trace#dotnet-trace-collect) where you can find more information about how to configure provider list. ### Source code [!code-csharp[EventPipeProfilerAdvanced.cs](../../../samples/BenchmarkDotNet.Samples/IntroEventPipeProfilerAdvanced.cs)] ### Output The output should contain information about the exported trace file which can be analyzed using [SpeedScope](https://www.speedscope.app/). ```markdown // * Diagnostic Output - EventPipeProfiler * Exported 1 trace file(s). Example: C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\BenchmarkDotNet.Artifacts\BenchmarkDotNet.Samples.IntroEventPipeProfilerAdvanced.RentAndReturn_Shared-20200406-090136.speedscope.json ``` ================================================ FILE: docs/articles/samples/IntroExceptionDiagnoser.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroExceptionDiagnoser --- ## Sample: IntroExceptionDiagnoser The `ExceptionDiagnoser` uses [AppDomain.FirstChanceException](https://learn.microsoft.com/en-us/dotnet/api/system.appdomain.firstchanceexception) API to report: * Exception frequency: The number of exceptions thrown during the operations divided by the number of operations. ### Source code [!code-csharp[IntroExceptionDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs)] ### Output | Method | Mean | Error | StdDev | Exception frequency | |----------------------- |---------:|----------:|----------:|--------------------:| | ThrowExceptionRandomly | 4.936 us | 0.1542 us | 0.4499 us | 0.1381 | ### Links * @docs.diagnosers * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroExceptionDiagnoser --- ================================================ FILE: docs/articles/samples/IntroExport.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroExport --- ## Sample: IntroExport BenchmarkDotNet has a lot of predefined exporters. ### Source code [!code-csharp[IntroExport.cs](../../../samples/BenchmarkDotNet.Samples/IntroExport.cs)] ### Links * @docs.exporters * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroExport --- ================================================ FILE: docs/articles/samples/IntroExportJson.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroExportJson --- ## Sample: IntroExportJson BenchmarkDotNet has a set of json exporters. You can customize the following properties of these exporters: * `fileNameSuffix`: a string which be placed in the end of target file name. * `indentJson`=`false`/`true`: should we format json or not. * `excludeMeasurements`=`false`/`true`: should we exclude detailed information about measurements or not (the final summary with statistics will be in the json file anyway). ### Source code [!code-csharp[IntroExportJson.cs](../../../samples/BenchmarkDotNet.Samples/IntroExportJson.cs)] ### Output Example of `IntroJsonExport-report-brief.json`: ```js { "Title":"IntroJsonExport", "HostEnvironmentInfo":{ "BenchmarkDotNetCaption":"BenchmarkDotNet-Dev.Core", "BenchmarkDotNetVersion":"0.9.9.0", "OsVersion":"Microsoft Windows NT 6.2.9200.0", "ProcessorName":{ "IsValueCreated":true, "Value":"Intel(R) Core(TM) i7-4702MQ CPU 2.20GHz" }, "ProcessorCount":8, "ClrVersion":"MS.NET 4.0.30319.42000", "Architecture":"64-bit", "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"RELEASE", "JitModules":"clrjit-v4.6.1586.0", "DotNetCliVersion":"1.0.0-preview2-003121", "ChronometerFrequency":2143474, "HardwareTimerKind":"Tsc" }, "Benchmarks":[ { "ShortInfo":"IntroJsonExport_Sleep10", "Namespace":"BenchmarkDotNet.Samples.Intro", "Type":"IntroJsonExport", "Method":"Sleep10", "MethodTitle":"Sleep10", "Parameters":"", "Properties":{ "Mode":"Throughput", "Platform":"Host", "Jit":"Host", "Runtime":"Host", "GcMode":"Host", "WarmupCount":"Auto", "IterationCount":"Auto", "LaunchCount":"Auto", "IterationTime":"Auto", "Affinity":"Auto" }, "Statistics":{ "N":20, "Min":10265993.7209375, "LowerFence":10255329.082734371, "Q1":10337369.528437499, "Median":10360382.6953125, "Mean":10362283.085796878, "Q3":10392063.158906251, "UpperFence":10474103.60460938, "Max":10436008.3209375, "InterquartileRange":54693.630468752235, "Outliers":[ ], "StandardError":10219.304338928456, "Variance":2088683623.4328396, "StandardDeviation":45702.118369205156, "Skewness":-0.1242777170069375, "Kurtosis":2.31980277935226, "ConfidenceInterval":{ "Mean":10362283.085796878, "Error":10219.304338928456, "Level":6, "Margin":20029.836504299772, "Lower":10342253.249292579, "Upper":10382312.922301177 }, "Percentiles":{ "P0":10265993.7209375, "P25":10338555.905625, "P50":10360382.6953125, "P67":10373496.555659376, "P80":10400703.4841875, "P85":10417280.326718749, "P90":10424125.595812501, "P95":10435620.51609375, "P100":10436008.3209375 } } },{ "ShortInfo":"IntroJsonExport_Sleep20", "Namespace":"BenchmarkDotNet.Samples.Intro", "Type":"IntroJsonExport", "Method":"Sleep20", "MethodTitle":"Sleep20", "Parameters":"", "Properties":{ "Mode":"Throughput", "Platform":"Host", "Jit":"Host", "Runtime":"Host", "GcMode":"Host", "WarmupCount":"Auto", "IterationCount":"Auto", "LaunchCount":"Auto", "IterationTime":"Auto", "Affinity":"Auto" }, "Statistics":{ "N":20, "Min":20258672.37, "LowerFence":20206333.269843742, "Q1":20325342.761249997, "Median":20362636.192500003, "Mean":20360791.931687497, "Q3":20404682.4221875, "UpperFence":20523691.913593754, "Max":20422396.073125, "InterquartileRange":79339.66093750298, "Outliers":[ ], "StandardError":10728.817907277158, "Variance":2302150673.7502208, "StandardDeviation":47980.732317777525, "Skewness":-0.50826238372439869, "Kurtosis":2.11050327966268, "ConfidenceInterval":{ "Mean":20360791.931687497, "Error":10728.817907277158, "Level":6, "Margin":21028.48309826323, "Lower":20339763.448589232, "Upper":20381820.414785761 }, "Percentiles":{ "P0":20258672.37, "P25":20327638.975312497, "P50":20362636.192500003, "P67":20391669.3762875, "P80":20406370.68625, "P85":20412542.034406248, "P90":20414412.5376875, "P95":20416606.697718751, "P100":20422396.073125 } } } ] } ``` ### Links * @docs.exporters * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroExportJson --- ================================================ FILE: docs/articles/samples/IntroExportXml.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroExportXml --- ## Sample: IntroExportXml BenchmarkDotNet has a set of XML exporters. You can customize the following properties of these exporters: * `fileNameSuffix`: a string which be placed in the end of target file name. * `indentXml`=`false`/`true`: should we format xml or not. * `excludeMeasurements`=`false`/`true`: should we exclude detailed information about measurements or not (the final summary with statistics will be in the XML file anyway). ### Source code [!code-csharp[IntroExportXml.cs](../../../samples/BenchmarkDotNet.Samples/IntroExportXml.cs)] ### Output Example of `IntroXmlExport-report-brief.xml`: ```xml IntroXmlExport BenchmarkDotNet 0.10.9.20170805-develop Windows 10 Redstone 2 (10.0.15063) Intel Core i7-3770K CPU 3.50GHz (Ivy Bridge) 8 .NET Framework 4.7 (CLR 4.0.30319.42000) 64bit False True RELEASE clrjit-v4.7.2101.1 1.0.4 3410220 Tsc IntroXmlExport.Sleep10: DefaultJob BenchmarkDotNet.Samples.Intro IntroXmlExport Sleep10 Sleep10 15 10989865.8785938 10989836.0967969 10990942.6053125 10991249.5870313 10991270.0524583 10991680.2776563 10992786.7861719 10992115.5501563 737.672343749553 148.484545262958 330714.902729213 575.07817097262 -0.67759778074187 3.14296703520386 15 10991270.0524583 148.484545262958 L999 614.793505974065 10990655.2589524 10991884.8459643 10989865.8785938 10991027.3689063 10991249.5870313 10991489.490875 10991696.7722187 10991754.5031875 10991933.1939688 10992067.441125 10992115.5501563 ``` ### Links * @docs.exporters * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroExportXml --- ================================================ FILE: docs/articles/samples/IntroFilters.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroFilters --- ## Sample: IntroFilters You can either use one of the predefined `Filter` types or create a custom type which implements `IFilter` interface. ### Source code [!code-csharp[IntroFilters.cs](../../../samples/BenchmarkDotNet.Samples/IntroFilters.cs)] ### Links * @docs.filters * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroFilters --- ================================================ FILE: docs/articles/samples/IntroFluentConfigBuilder.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroFluentConfigBuilder --- ## Sample: IntroFluentConfigBuilder There is no need to create new Config type, you can simply use fluent interface. ### Source code [!code-csharp[IntroFluentConfigBuilder.cs](../../../samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs)] ### Links * @docs.configs * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroFluentConfigBuilder --- ================================================ FILE: docs/articles/samples/IntroGcMode.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroGcMode --- ## Sample: IntroGcMode ### Source code [!code-csharp[IntroGcMode.cs](../../../samples/BenchmarkDotNet.Samples/IntroGcMode.cs)] ### Output ### Links * @docs.jobs * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroGcMode --- ================================================ FILE: docs/articles/samples/IntroGenericTypeArguments.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroGenericTypeArguments --- ## Sample: IntroGenericTypeArguments ### Source code [!code-csharp[IntroGenericTypeArguments.cs](../../../samples/BenchmarkDotNet.Samples/IntroGenericTypeArguments.cs)] ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroGenericTypeArguments --- ================================================ FILE: docs/articles/samples/IntroHardwareCounters.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroHardwareCounters --- ## Sample: IntroHardwareCounters This diagnoser is not enabled in explicit way as the other diagnosers. You need to specify `[HardwareCounters]` and we choose the right diagnoser in the runtime. ### Source code [!code-csharp[IntroHardwareCounters.cs](../../../samples/BenchmarkDotNet.Samples/IntroHardwareCounters.cs)] ### Output | Method | Mean | Mispredict rate | BranchInstructions/Op | BranchMispredictions/Op | |------------------- |------------ |---------------- |---------------------- |------------------------ | | SortedBranch | 21.4539 us | 0,04% | 70121 | 24 | | UnsortedBranch | 136.1139 us | 23,70% | 68788 | 16301 | | SortedBranchless | 28.6705 us | 0,06% | 35711 | 22 | | UnsortedBranchless | 28.9336 us | 0,05% | 35578 | 17 | ### Links * @docs.diagnosers * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroHardwareCounters --- ================================================ FILE: docs/articles/samples/IntroInProcess.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroInProcess --- ## Sample: IntroInProcess InProcessEmitToolchain is our toolchain which does not generate any new executable. It emits IL on the fly and runs it from within the process itself. It can be useful if want to run the benchmarks very fast or if you want to run them for framework which we don't support. An example could be a local build of CoreCLR. ### Usage ```cs [InProcessAttribute] public class TypeWithBenchmarks { } ``` ### Source code [!code-csharp[IntroInProcess.cs](../../../samples/BenchmarkDotNet.Samples/IntroInProcess.cs)] ### Output ### Links * @docs.toolchains * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroInProcess --- ================================================ FILE: docs/articles/samples/IntroInProcessWrongEnv.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroInProcessWrongEnv --- ## Sample: IntroInProcessWrongEnv ### Source code [!code-csharp[IntroInProcessWrongEnv.cs](../../../samples/BenchmarkDotNet.Samples/IntroInProcessWrongEnv.cs)] ### Output ### Links * @docs.toolchains * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroInProcessWrongEnv --- ================================================ FILE: docs/articles/samples/IntroInliningDiagnoser.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroInliningDiagnoser --- ## Sample: IntroInliningDiagnoser This sample shows how to add InliningDiagnoser with events from only one namespace (BenchmarkDotNet.Samples). ### Source code [!code-csharp[IntroInliningDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroInliningDiagnoser.cs)] ================================================ FILE: docs/articles/samples/IntroJitStatsDiagnoser.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroJitStatsDiagnoser --- ## Sample: IntroJitStatsDiagnoser This diagnoser shows various stats from the JIT compiler that were collected during entire benchmark run (warmup phase and BenchmarkDotNet-generated boilerplate code are included): * Amount of JITted methods. * Amount of [tiered methods](https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-core-3-0#tiered-compilation). * How much memory JIT allocated during the benchmark. ### Restrictions * Windows only ### Source code [!code-csharp[IntroJitStatsDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroJitStatsDiagnoser.cs)] ### Output | Method | Mean | Error | StdDev | Methods JITted | Methods Tiered | JIT allocated memory | |------- |---------:|---------:|---------:|---------------:|---------------:|---------------------:| | Sleep | 15.50 ms | 0.052 ms | 0.048 ms | 1,102 | 214 | 221,736 B | ### Links * @docs.diagnosers * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroJitStatsDiagnoser --- ================================================ FILE: docs/articles/samples/IntroJobBaseline.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroJobBaseline --- ## Sample: IntroJobBaseline If you want to compare several runtime configuration, you can mark one of your jobs with `baseline = true`. ### Source code [!code-csharp[IntroJobBaseline.cs](../../../samples/BenchmarkDotNet.Samples/IntroJobBaseline.cs)] ### Output ```ini BenchmarkDotNet=v0.10.12, OS=Windows 10 Redstone 3 [1709, Fall Creators Update] (10.0.16299.192) Processor=Intel Core i7-6700HQ CPU 2.60GHz (Skylake), ProcessorCount=8 Frequency=2531249 Hz, Resolution=395.0619 ns, Timer=TSC .NET Core SDK=2.0.3 [Host] : .NET Core 2.0.3 (Framework 4.6.25815.02), 64bit RyuJIT Job-MXFYPZ : .NET Framework 4.7 (CLR 4.0.30319.42000), 64bit RyuJIT-v4.7.2600.0 Core : .NET Core 2.0.3 (Framework 4.6.25815.02), 64bit RyuJIT Mono : Mono 5.4.0 (Visual Studio), 64bit ``` ```markdown Method | Runtime | Mean | Error | StdDev | Ratio | RatioSD | ---------- |-------- |---------:|----------:|----------:|------:|--------:| SplitJoin | Clr | 19.42 us | 0.2447 us | 0.1910 us | 1.00 | 0.00 | SplitJoin | Core | 13.00 us | 0.2183 us | 0.1935 us | 0.67 | 0.01 | SplitJoin | Mono | 39.14 us | 0.7763 us | 1.3596 us | 2.02 | 0.07 | ``` ### Links * @docs.baselines * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroJobBaseline --- ================================================ FILE: docs/articles/samples/IntroJoin.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroJoin --- ## Sample: IntroJoin If you are using `BenchmarkSwitcher` and want to run all the benchmarks with a category from all types and join them into one summary table, use the `--join` option (or `BenchmarkSwitcher.RunAllJoined`): ### Source code [!code-csharp[IntroJoin.cs](../../../samples/BenchmarkDotNet.Samples/IntroJoin.cs)] ### Command line ``` --join --allCategories=IntroJoinA ``` ### Output ```markdown | Type | Method | Mean | Error | |----------- |------- |---------:|------:| | IntroJoin1 | A | 10.99 ms | NA | | IntroJoin2 | A | 12.50 ms | NA | ``` ### Links * @docs.filters * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroJoin --- ================================================ FILE: docs/articles/samples/IntroLargeAddressAware.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroLargeAddressAware title: "Sample: IntroLargeAddressAware" --- ## Sample: IntroLargeAddressAware ### Source code [!code-csharp[IntroLargeAddressAware.cs](../../../samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs)] ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroLargeAddressAware --- ================================================ FILE: docs/articles/samples/IntroMaui.md ================================================ --- uid: BenchmarkDotNet.Samples.Maui --- ## Sample: IntroMaui To use BenchmarkDotNet with [.NET MAUI](https://dotnet.microsoft.com/apps/maui), you will need to build a small UI for running benchmarks and displaying the results. .NET MAUI allows you to run your benchmarks on Android, iOS, Mac Catalyst, and Windows from a single codebase. Other notes: * Use `Release` builds when running actual benchmarks. * Consider disabling trimming appropriately for your benchmarks. ### Source code [!code-csharp[MainPage.xaml.cs](../../../samples/BenchmarkDotNet.Samples.Maui/MainPage.xaml.cs)] ### Output #### Windows ![MAUI Output - Windows](../../images/maui-screenshot-windows.png) #### Android ![MAUI Output - Android](../../images/maui-screenshot-android.png) #### iOS ![MAUI Output - iOS](../../images/maui-screenshot-ios.png) #### Mac Catalyst ![MAUI Output - mac](../../images/maui-screenshot-mac.png) ### Links * [.NET MAUI Documentation](https://learn.microsoft.com/dotnet/maui/) * [Trimming in .NET MAUI](https://learn.microsoft.com/dotnet/maui/deployment/trimming) * The permanent link to this sample: @BenchmarkDotNet.Samples.Maui --- ================================================ FILE: docs/articles/samples/IntroMonitoring.md ================================================ --- #cspell:ignore runstrategy uid: BenchmarkDotNet.Samples.IntroMonitoring --- ## Sample: IntroMonitoring If a benchmark method takes at least 100ms, you can also use the `Monitoring` strategy. In this case, the pilot stage will be omitted, by default you get 1 iteration = 1 operation (or you can manually set amount of operation in an iteration). Also you can use `[IterationSetup]` and `[IterationCleanup]` in this case: it shouldn't affect time measurements (but it can affect results of MemoryDiagnoser). It's a perfect mode for benchmarks which doesn't have a steady state and the performance distribution is tricky: `Monitoring` will help you to collect a set of measurements and get statistics. ### Usage ```cs [SimpleJob(RunStrategy.Monitoring, launchCount: 10, warmupCount: 0, iterationCount: 100)] public class MyBenchmarkClass ``` ### Source code [!code-csharp[IntroMonitoring.cs](../../../samples/BenchmarkDotNet.Samples/IntroMonitoring.cs)] ### Output ```markdown Result 1: 1 op, 61552600.00 ns, 61.5526 ms/op Result 2: 1 op, 10141700.00 ns, 10.1417 ms/op Result 3: 1 op, 10482900.00 ns, 10.4829 ms/op Result 4: 1 op, 50410900.00 ns, 50.4109 ms/op Result 5: 1 op, 10421400.00 ns, 10.4214 ms/op Result 6: 1 op, 20556100.00 ns, 20.5561 ms/op Result 7: 1 op, 70473200.00 ns, 70.4732 ms/op Result 8: 1 op, 50581700.00 ns, 50.5817 ms/op Result 9: 1 op, 10559000.00 ns, 10.5590 ms/op Result 10: 1 op, 70496300.00 ns, 70.4963 ms/op ``` | Method | Mean | Error | StdDev | Min | Q1 | Q3 | Max | |------- |---------:|---------:|---------:|---------:|---------:|---------:|---------:| | Foo | 36.57 ms | 40.03 ms | 26.47 ms | 10.14 ms | 10.48 ms | 61.55 ms | 70.50 ms | ### Links * @docs.runstrategy * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroMonitoring --- ================================================ FILE: docs/articles/samples/IntroMultimodal.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroMultimodal --- ## Sample: IntroMultimodal ### Source code [!code-csharp[IntroMultimodal.cs](../../../samples/BenchmarkDotNet.Samples/IntroMultimodal.cs)] ### Output ```markdown Method | Mean | Error | StdDev | Median | MValue | ------------ |---------:|-----------:|------------:|---------:|-------:| Unimodal | 100.5 ms | 0.0713 ms | 0.0667 ms | 100.5 ms | 2.000 | Bimodal | 144.5 ms | 16.9165 ms | 49.8787 ms | 100.6 ms | 3.571 | Trimodal | 182.5 ms | 27.4285 ms | 80.8734 ms | 200.5 ms | 4.651 | Quadrimodal | 226.6 ms | 37.2269 ms | 109.7644 ms | 200.7 ms | 5.882 | // * Warnings * MultimodalDistribution IntroMultimodal.Bimodal: MainJob -> It seems that the distribution is bimodal (mValue = 3.57) IntroMultimodal.Trimodal: MainJob -> It seems that the distribution is multimodal (mValue = 4.65) IntroMultimodal.Quadrimodal: MainJob -> It seems that the distribution is multimodal (mValue = 5.88) ``` ### Links * @docs.statistics * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroMultimodal --- ================================================ FILE: docs/articles/samples/IntroNativeMemory.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroNativeMemory --- ## Sample: IntroNativeMemory The `NativeMemoryProfiler` uses `EtwProfiler` to profile the code using ETW and adds the extra columns `Allocated native memory` and `Native memory leak` to the benchmark results table. ### Source code [!code-csharp[IntroNativeMemory.cs](../../../samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs)] ### Output | Method | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated | Allocated native memory | Native memory leak | |---------------------- |-------------:|--------------:|-------------:|------:|------:|------:|----------:|------------------------:|-------------------:| | BitmapWithLeaks | 73,456.43 ns | 22,498.10 ns | 1,233.197 ns | - | - | - | 177 B | 13183 B | 11615 B | | Bitmap | 91,590.08 ns | 101,468.12 ns | 5,561.810 ns | - | - | - | 180 B | 12624 B | - | | AllocHGlobal | 79.91 ns | 43.93 ns | 2.408 ns | - | - | - | - | 80 B | - | | AllocHGlobalWithLeaks | 103.50 ns | 153.21 ns | 8.398 ns | - | - | - | - | 80 B | 80 B | ### Profiling memory leaks The BenchmarkDotNet repeats benchmarking function many times. Sometimes it can cause a memory overflow. In this case, the BenchmarkDotNet shows the message: ```ini OutOfMemoryException! BenchmarkDotNet continues to run additional iterations until desired accuracy level is achieved. It's possible only if the benchmark method doesn't have any side-effects. If your benchmark allocates memory and keeps it alive, you are creating a memory leak. You should redesign your benchmark and remove the side-effects. You can use `OperationsPerInvoke`, `IterationSetup` and `IterationCleanup` to do that. ``` In this case, you should try to reduce the number of invocation, by adding `[ShortRunJob]` attribute or using `Job.Short` for custom configuration. ### Links * @docs.diagnosers * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroNativeMemory --- ================================================ FILE: docs/articles/samples/IntroNuGet.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroNuGet --- ## Sample: IntroNuGet You can set specific versions of NuGet dependencies for each job using MsBuild properties in your csproj. It allows comparing different versions of the same package (if there are no breaking changes in API). ### Source code [!code-csharp[IntroNuGet.cs](../../../samples/BenchmarkDotNet.Samples/IntroNuGet.cs)] ### Output | Method | Job | Arguments | Mean | Error | StdDev | |-------------------------- |------- |-------------------- |---------:|----------:|----------:| | ToImmutableArrayBenchmark | v9.0.0 | /p:SciVersion=9.0.0 | 1.173 μs | 0.0057 μs | 0.0086 μs | | ToImmutableArrayBenchmark | v9.0.3 | /p:SciVersion=9.0.3 | 1.173 μs | 0.0038 μs | 0.0058 μs | | ToImmutableArrayBenchmark | v9.0.5 | /p:SciVersion=9.0.5 | 1.172 μs | 0.0107 μs | 0.0157 μs | ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroNuGet --- ================================================ FILE: docs/articles/samples/IntroOrderAttr.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroOrderAttr --- ## Sample: IntroOrderAttr ### Source code [!code-csharp[IntroOrderAttr.cs](../../../samples/BenchmarkDotNet.Samples/IntroOrderAttr.cs)] ### Links * @docs.orderers * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroOrderAttr --- ================================================ FILE: docs/articles/samples/IntroOrderManual.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroOrderManual --- ## Sample: IntroOrderManual ### Source code [!code-csharp[IntroOrderManual.cs](../../../samples/BenchmarkDotNet.Samples/IntroOrderManual.cs)] ### Links * @docs.orderers * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroOrderManual --- ================================================ FILE: docs/articles/samples/IntroOutliers.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroOutliers --- ## Sample: IntroOutliers ### Source code [!code-csharp[IntroOutliers.cs](../../../samples/BenchmarkDotNet.Samples/IntroOutliers.cs)] ### Output ```markdown Method | Job | OutlierMode | Mean | Error | StdDev | ------- |-------------------- |------------ |---------:|------------:|------------:| Foo | DontRemoveOutliers | DontRemove | 150.5 ms | 239.1911 ms | 158.2101 ms | Foo | RemoveUpperOutliers | RemoveUpper | 100.5 ms | 0.1931 ms | 0.1149 ms | // * Hints * Outliers IntroOutliers.Foo: DontRemoveOutliers -> 1 outlier was detected IntroOutliers.Foo: RemoveUpperOutliers -> 1 outlier was removed ``` ### Links * @docs.statistics * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroOutliers --- ================================================ FILE: docs/articles/samples/IntroParams.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroParams --- ## Sample: IntroParams You can mark one or several fields or properties in your class by the [`[Params]`](xref:BenchmarkDotNet.Attributes.ParamsAttribute) attribute. In this attribute, you can specify set of values. Every value must be a compile-time constant. As a result, you will get results for each combination of params values. ### Source code [!code-csharp[IntroParams.cs](../../../samples/BenchmarkDotNet.Samples/IntroParams.cs)] ### Output ```markdown | Method | A | B | Mean | Error | StdDev | |---------- |---- |--- |---------:|--------:|--------:| | Benchmark | 100 | 10 | 115.3 ms | 0.13 ms | 0.12 ms | | Benchmark | 100 | 20 | 125.4 ms | 0.14 ms | 0.12 ms | | Benchmark | 200 | 10 | 215.5 ms | 0.19 ms | 0.18 ms | | Benchmark | 200 | 20 | 225.4 ms | 0.17 ms | 0.16 ms | ``` ### Links * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroParams --- ================================================ FILE: docs/articles/samples/IntroParamsAllValues.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroParamsAllValues --- ## Sample: IntroParamsAllValues If you want to use all possible values of an `enum` or another type with a small number of values, you can use the [`[ParamsAllValues]`](xref:BenchmarkDotNet.Attributes.ParamsAllValuesAttribute) attribute, instead of listing all the values by hand. The types supported by the attribute are: * `bool` * any `enum` that is not marked with `[Flags]` * `Nullable`, where `T` is an enum or boolean ### Source code [!code-csharp[IntroParamsAllValues.cs](../../../samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs)] ### Output ```markdown Method | E | B | Mean | Error | ---------- |------ |------ |---------:|------:| Benchmark | One | ? | 101.4 ms | NA | Benchmark | One | False | 111.1 ms | NA | Benchmark | One | True | 122.0 ms | NA | Benchmark | Two | ? | 201.3 ms | NA | Benchmark | Two | False | 212.1 ms | NA | Benchmark | Two | True | 221.3 ms | NA | Benchmark | Three | ? | 301.4 ms | NA | Benchmark | Three | False | 311.5 ms | NA | Benchmark | Three | True | 320.8 ms | NA | ``` ### Links * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroParamsAllValues --- ================================================ FILE: docs/articles/samples/IntroParamsPriority.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroParamsPriority --- ## Sample: IntroParamsPriority In order to sort columns of parameters in the results table you can use the Property `Priority` inside the params attribute. The priority range is `[Int32.MinValue;Int32.MaxValue]`, lower priorities will appear earlier in the column order. The default priority is set to `0`. ### Source code [!code-csharp[IntroParamsPriority.cs](../../../samples/BenchmarkDotNet.Samples/IntroParamsPriority.cs)] ### Output ```markdown | Method | B | A | Mean | Error | StdDev | |---------- |--- |---- |---------:|--------:|--------:| | Benchmark | 10 | 100 | 115.4 ms | 0.12 ms | 0.11 ms | ``` ### Links * Priority BaseClass [`PriorityAttribute.cs`](xref:BenchmarkDotNet.Attributes.PriorityAttribute) * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroParamsPriority --- ================================================ FILE: docs/articles/samples/IntroParamsSource.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroParamsSource --- ## Sample: IntroParamsSource In case you want to use a lot of values, you should use [`[ParamsSource]`](xref:BenchmarkDotNet.Attributes.ParamsSourceAttribute) You can mark one or several fields or properties in your class by the [`[Params]`](xref:BenchmarkDotNet.Attributes.ParamsAttribute) attribute. In this attribute, you have to specify the name of public method/property which is going to provide the values (something that implements `IEnumerable`). The source may be instance or static. If the source is not in the same type as the benchmark, the type containing the source must be specified in the attribute constructor. ### Source code [!code-csharp[IntroParamsSource.cs](../../../samples/BenchmarkDotNet.Samples/IntroParamsSource.cs)] ### Output ```markdown | Method | B | A | Mean | Error | StdDev | |---------- |--- |---- |---------:|--------:|--------:| | Benchmark | 10 | 100 | 115.5 ms | 0.17 ms | 0.16 ms | | Benchmark | 10 | 200 | 215.6 ms | 0.15 ms | 0.14 ms | | Benchmark | 20 | 100 | 125.5 ms | 0.19 ms | 0.18 ms | | Benchmark | 20 | 200 | 225.5 ms | 0.23 ms | 0.22 ms | ``` ### Remarks **A remark about IParam.** You don't need to use `IParam` anymore since `0.11.0`. Just use complex types as you wish and override `ToString` method to change the display names used in the results. ### Links * @docs.parameterization * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroParamsSource --- ================================================ FILE: docs/articles/samples/IntroPercentiles.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroPercentiles --- ## Sample: IntroPercentiles The percentile represents a higher boundary for specified percentage of the measurements. For example, 95th percentile = 500ms means that 95% of all samples are not slower than 500ms. This metric is not very useful in microbenchmarks, as the values from consequent runs have a very narrow distribution. However, real-world scenarios often have so-called long tail distribution (due to IO delays, locks, memory access latency and so on), so the average execution time cannot be trusted. The percentiles allow to include the tail of distribution into the comparison. However, it requires some preparations steps. At first, you should have enough runs to count percentiles from. The `IterationCount` in the config should be set to 10-20 runs at least. Second, the count of iterations for each run should not be very high, or the peak timings will be averaged. The `IterationTime = 25` works fine for most cases; for long-running benchmarks the `Mode = Mode.SingleRun` will be the best choice. However, feel free to experiment with the config values. Third, if you want to be sure that measurements are repeatable, set the `LaunchCount` to 3 or higher. And last, don't forget to include the columns into the config. They are not included by default (as said above, these are not too useful for most of the benchmarks). There're predefined `StatisticColumn.P0`..`StatisticColumn.P100` for absolute timing percentiles. ### Example Run the IntroPercentiles sample. It contains three benchmark methods. * First delays for 20 ms constantly. * The second has random delays for 10..30 ms. * And the third delays for 10ms 85 times of 100 and delays for 40ms 15 times of 100. Here's the output from the benchmark (some columns removed for brevity): | Method | Median | StdDev | Ratio | P0 | P50 | P80 | P85 | P95 | P100 | |--------------- |----------- |----------- |------ |----------- |----------- |----------- |----------- |----------- |----------- | | ConstantDelays | 20.3813 ms | 0.2051 ms | 1.00 | 20.0272 ms | 20.3813 ms | 20.4895 ms | 20.4954 ms | 20.5869 ms | 21.1471 ms | | RandomDelays | 19.8055 ms | 5.7556 ms | 0.97 | 10.0793 ms | 19.8055 ms | 25.4173 ms | 26.5187 ms | 29.0313 ms | 29.4550 ms | | RareDelays | 10.3385 ms | 11.4828 ms | 0.51 | 10.0157 ms | 10.3385 ms | 10.5211 ms | 40.0560 ms | 40.3992 ms | 40.4674 ms | Also, it's very easy to screw the results with incorrect setup. For example, the same code being run with ```cs new Job { IterationCount = 5, IterationTime = 500 } ``` completely hides the peak values: | Method | Median | StdDev | Ratio | P0 | P50 | P80 | P85 | P95 | P100 | |--------------- |----------- |---------- |------ |----------- |----------- |----------- |----------- |----------- |----------- | | ConstantDelays | 20.2692 ms | 0.0308 ms | 1.00 | 20.1986 ms | 20.2692 ms | 20.2843 ms | 20.2968 ms | 20.3097 ms | 20.3122 ms | | RandomDelays | 18.9965 ms | 0.8601 ms | 0.94 | 18.1339 ms | 18.9965 ms | 19.8126 ms | 19.8278 ms | 20.4485 ms | 20.9466 ms | | RareDelays | 14.0912 ms | 2.8619 ms | 0.70 | 10.2606 ms | 14.0912 ms | 15.7653 ms | 17.3862 ms | 18.6728 ms | 18.6940 ms | ### Source code [!code-csharp[IntroPercentiles.cs](../../../samples/BenchmarkDotNet.Samples/IntroPercentiles.cs)] ### Links * @docs.statistics * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroPercentiles --- ================================================ FILE: docs/articles/samples/IntroPowerPlan.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroPowerPlan --- ## Sample: IntroPowerPlan This sample shows how we can manipulate with power plans. In BenchmarkDotNet we could change power plan in two ways. The first one is to set one from the list: * PowerSaver, guid: a1841308-3541-4fab-bc81-f71556f20b4a * Balanced, guid: 381b4222-f694-41f0-9685-ff5bb260df2e * High-Performance, guid: 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c (the default one) * Ultimate Performance, guid: e9a42b02-d5df-448d-aa00-03f14749eb61 * UserPowerPlan (a current power plan set in computer) The second one rely on guid string. We could easily found currently set GUIDs with cmd command ```cmd powercfg /list ``` If we set power plans in two ways at the same time, the second one will be used. ### Source code [!code-csharp[IntroPowerPlan.cs](../../../samples/BenchmarkDotNet.Samples/IntroPowerPlan.cs)] ### Links * @docs.powerplans * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroPowerPlan --- ================================================ FILE: docs/articles/samples/IntroRankColumn.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroRankColumn --- ## Sample: IntroRankColumn ### Source code [!code-csharp[IntroRankColumn.cs](../../../samples/BenchmarkDotNet.Samples/IntroRankColumn.cs)] ### Output ```markdown Method | Factor | Mean | Error | StdDev | Rank | Rank | Rank | ------- |------- |---------:|---------:|----------:|-----:|-----:|-----:| Foo | 1 | 100.8 ms | 2.250 ms | 0.1272 ms | 1 | I | * | Foo | 2 | 200.8 ms | 4.674 ms | 0.2641 ms | 2 | II | ** | Bar | 1 | 200.9 ms | 2.012 ms | 0.1137 ms | 2 | II | ** | Bar | 2 | 400.7 ms | 4.509 ms | 0.2548 ms | 3 | III | *** | ``` ### Links * @docs.statistics * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroRankColumn --- ================================================ FILE: docs/articles/samples/IntroRatioSD.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroRatioSD --- ## Sample: IntroRatioSD The ratio of two benchmarks is not a single number, it's a distribution. In most simple cases, the range of the ratio distribution is narrow, and BenchmarkDotNet displays a single column `Ratio` with the mean value. However, it also adds the `RatioSD` column (the standard deviation of the ratio distribution) in complex situations. In the below example, the baseline benchmark is spoiled by a single outlier ### Source code [!code-csharp[IntroRatioSD.cs](../../../samples/BenchmarkDotNet.Samples/IntroRatioSD.cs)] ### Output Here are statistics details for the baseline benchmark: ```ini Mean = 600.6054 ms, StdErr = 500.0012 ms (83.25%); N = 10, StdDev = 1,581.1428 ms Min = 100.2728 ms, Q1 = 100.3127 ms, Median = 100.4478 ms, Q3 = 100.5011 ms, Max = 5,100.6163 ms IQR = 0.1884 ms, LowerFence = 100.0301 ms, UpperFence = 100.7837 ms ConfidenceInterval = [-1,789.8568 ms; 2,991.0677 ms] (CI 99.9%), Margin = 2,390.4622 ms (398.01% of Mean) Skewness = 2.28, Kurtosis = 6.57, MValue = 2 -------------------- Histogram -------------------- [-541.891 ms ; 743.427 ms) | @@@@@@@@@ [ 743.427 ms ; 2027.754 ms) | [2027.754 ms ; 3312.082 ms) | [3312.082 ms ; 4458.453 ms) | [4458.453 ms ; 5742.780 ms) | @ --------------------------------------------------- ``` As you can see, a single outlier significantly affected the metrics. Because of this, BenchmarkDotNet adds the `Median` and the `RatioSD` columns in the summary table: ```markdown Method | Mean | Error | StdDev | Median | Ratio | RatioSD | ------- |----------:|--------------:|--------------:|----------:|------:|--------:| Base | 600.61 ms | 2,390.4622 ms | 1,581.1428 ms | 100.45 ms | 1.00 | 0.00 | Slow | 200.50 ms | 0.4473 ms | 0.2959 ms | 200.42 ms | 1.80 | 0.62 | Fast | 50.54 ms | 0.3435 ms | 0.2272 ms | 50.48 ms | 0.45 | 0.16 | ``` Let's look at the `Base` and `Slow` benchmarks. The `Mean` values are `600` and `200` milliseconds; the "Scaled Mean" value is 0.3. The `Median` values are `100` and `200` milliseconds; the "Scaled Median" value is 2. Both values are misleading. BenchmarkDotNet evaluates the ratio distribution and displays the mean (1.80) and the standard deviation (0.62). ### Links * @docs.baselines * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroRatioSD --- ================================================ FILE: docs/articles/samples/IntroRatioStyle.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroRatioStyle --- ## Sample: IntroRatioStyle Using `RatioStyle`, we can override the style of the "Ratio" column in `SummaryStyle`. Here are the possible values: * `Value`: default value that shows the ration value between the current benchmark and the baseline benchmark (e.g., `0.15` or `1.15`) * `Percentage`: express the ration in percentage (e.g., `-85%` or `+15%`) * `Trend`: shows how much the current benchmark is faster or slower than the base benchmark (e.g., `6.63x faster` or `1.15x slower`) ### Source code [!code-csharp[IntroRatioStyle.cs](../../../samples/BenchmarkDotNet.Samples/IntroRatioStyle.cs)] ### Output With the given `RatioStyle.Trend`, we have the following table: ```markdown | Method | Mean | Error | StdDev | Ratio | RatioSD | |--------- |-----------:|--------:|--------:|-------------:|--------:| | Baseline | 1,000.6 ms | 2.48 ms | 0.14 ms | baseline | | | Bar | 150.9 ms | 1.30 ms | 0.07 ms | 6.63x faster | 0.00x | | Foo | 1,150.4 ms | 5.17 ms | 0.28 ms | 1.15x slower | 0.00x | ``` With the default `RatioStyle.Value`, we get the following table: ```markdown | Method | Mean | Error | StdDev | Ratio | |--------- |-----------:|--------:|--------:|------:| | Baseline | 1,000.3 ms | 2.71 ms | 0.15 ms | 1.00 | | Bar | 150.6 ms | 1.67 ms | 0.09 ms | 0.15 | | Foo | 1,150.6 ms | 7.41 ms | 0.41 ms | 1.15 | ``` If we use `RatioStyle.Percentage`, we get the following table: ```markdown | Method | Mean | Error | StdDev | Ratio | RatioSD | |--------- |-----------:|--------:|--------:|---------:|--------:| | Baseline | 1,000.3 ms | 4.69 ms | 0.26 ms | baseline | | | Bar | 150.7 ms | 1.42 ms | 0.08 ms | -85% | 0.1% | | Foo | 1,150.3 ms | 6.13 ms | 0.34 ms | +15% | 0.0% | ``` ### Links * @docs.baselines * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroRatioStyle --- ================================================ FILE: docs/articles/samples/IntroSetupCleanupGlobal.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroSetupCleanupGlobal --- ## Sample: IntroSetupCleanupGlobal A method which is marked by the [`[GlobalSetup]`](xref:BenchmarkDotNet.Attributes.GlobalSetupAttribute) attribute will be executed only once per a benchmarked method after initialization of benchmark parameters and before all the benchmark method invocations. A method which is marked by the [`[GlobalCleanup]`](xref:BenchmarkDotNet.Attributes.GlobalCleanupAttribute) attribute will be executed only once per a benchmarked method after all the benchmark method invocations. If you are using some unmanaged resources (e.g., which were created in the `GlobalSetup` method), they can be disposed in the `GlobalCleanup` method. ### Source code [!code-csharp[IntroSetupCleanupGlobal.cs](../../../samples/BenchmarkDotNet.Samples/IntroSetupCleanupGlobal.cs)] ### Links * @docs.setup-and-cleanup * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroSetupCleanupGlobal --- ================================================ FILE: docs/articles/samples/IntroSetupCleanupIteration.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroSetupCleanupIteration --- ## Sample: IntroSetupCleanupIteration A method which is marked by the [`[IterationSetup]`](xref:BenchmarkDotNet.Attributes.IterationSetupAttribute) attribute will be executed exactly once *before each benchmark invocation*, forcing `UnrollFactor=1` and `InvocationCount=1` (we have changed that in 0.11.0). It's not recommended to use this attribute in microbenchmarks because it can spoil the results. However, if you are writing a macrobenchmark (e.g. a benchmark which takes at least 100ms) and you want to prepare some data before each invocation, [`[IterationSetup]`](xref:BenchmarkDotNet.Attributes.IterationSetupAttribute) can be useful. A method which is marked by the [`[IterationCleanup]`](xref:BenchmarkDotNet.Attributes.IterationCleanupAttribute) attribute will be executed exactly once *after each invocation*. This attribute has the same set of constraint with `[IterationSetup]`: it's not recommended to use [`[IterationCleanup]`](xref:BenchmarkDotNet.Attributes.IterationCleanupAttribute) in microbenchmarks. ### Source code [!code-csharp[IntroSetupCleanupIteration.cs](../../../samples/BenchmarkDotNet.Samples/IntroSetupCleanupIteration.cs)] ### The order of method calls ```cs // GlobalSetup // IterationSetup (1) // IterationSetup Jitting // IterationCleanup (1) // IterationCleanup Jitting // IterationSetup (2) // MainWarmup1 // Benchmark // MainWarmup1 // IterationCleanup (2) // MainWarmup1 // IterationSetup (3) // MainWarmup2 // Benchmark // MainWarmup2 // IterationCleanup (3) // MainWarmup2 // IterationSetup (4) // MainTarget1 // Benchmark // MainTarget1 // IterationCleanup (4) // MainTarget1 // IterationSetup (5) // MainTarget2 // Benchmark // MainTarget2 // IterationCleanup (5) // MainTarget2 // IterationSetup (6) // MainTarget3 // Benchmark // MainTarget3 // IterationCleanup (6) // MainTarget3 // GlobalCleanup ``` ### Links * @docs.setup-and-cleanup * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroSetupCleanupIteration --- ================================================ FILE: docs/articles/samples/IntroSetupCleanupTarget.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroSetupCleanupTarget --- ## Sample: IntroSetupCleanupTarget Sometimes it's useful to run setup or cleanups for specific benchmarks. All four setup and cleanup attributes have a Target property that allow the setup/cleanup method to be run for one or more specific benchmark methods. ### Source code [!code-csharp[IntroSetupCleanupTarget.cs](../../../samples/BenchmarkDotNet.Samples/IntroSetupCleanupTarget.cs)] ### The order of method calls ```cs // GlobalSetup A // Benchmark A // GlobalSetup B // Benchmark B // GlobalSetup B // Benchmark C // Benchmark D ``` ### Links * @docs.setup-and-cleanup * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroSetupCleanupTarget --- ================================================ FILE: docs/articles/samples/IntroStaThread.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroStaThread --- ## Sample: IntroStaThread If the code you want to benchmark requires `[System.STAThread]` then you need to apply this attribute to the benchmarked method. BenchmarkDotNet will generate an executable with `[STAThread]` applied to its `Main` method. Using this feature on .NET Core requires .NET Core 2.1 or newer. Older versions will not work due to [this](https://github.com/dotnet/runtime/issues/8834) bug. ### Source code [!code-csharp[IntroStaThread.cs](../../../samples/BenchmarkDotNet.Samples/IntroStaThread.cs)] ### Links * @docs.customizing-runtime * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroStaThread --- ================================================ FILE: docs/articles/samples/IntroStatisticalTesting.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroStatisticalTesting --- ## Sample: IntroStatisticalTesting ### Source code [!code-csharp[IntroStatisticalTesting.cs](../../../samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs)] ### Output ```markdown | Method | Mean | Error | StdDev | Ratio | Welch(1us)/p-values | Welch(3%)/p-values | MannWhitney(1us)/p-values | MannWhitney(3%)/p-values | |--------- |----------:|----------:|----------:|------:|---------------------- |---------------------- |-------------------------- |------------------------- | | Sleep50 | 53.13 ms | 0.5901 ms | 0.1532 ms | 0.51 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0040 | Faster: 1.0000/0.0040 | | Sleep97 | 100.07 ms | 0.9093 ms | 0.2361 ms | 0.97 | Faster: 1.0000/0.0000 | Same: 1.0000/0.1290 | Faster: 1.0000/0.0040 | Same: 1.0000/0.1111 | | Sleep99 | 102.23 ms | 2.4462 ms | 0.6353 ms | 0.99 | Faster: 0.9928/0.0072 | Same: 1.0000/0.9994 | Faster: 0.9960/0.0079 | Same: 1.0000/1.0000 | | Sleep100 | 103.34 ms | 0.8180 ms | 0.2124 ms | 1.00 | Base: 0.5029/0.5029 | Base: 1.0000/1.0000 | Base: 0.7262/0.7262 | Base: 1.0000/1.0000 | | Sleep101 | 103.73 ms | 2.1591 ms | 0.5607 ms | 1.00 | Same: 0.1041/0.8969 | Same: 0.9999/1.0000 | Same: 0.1111/0.9246 | Same: 1.0000/1.0000 | | Sleep103 | 106.21 ms | 1.2511 ms | 0.3249 ms | 1.03 | Slower: 0.0000/1.0000 | Same: 0.9447/1.0000 | Slower: 0.0040/1.0000 | Same: 0.9246/1.0000 | | Sleep150 | 153.16 ms | 3.4929 ms | 0.9071 ms | 1.48 | Slower: 0.0000/1.0000 | Slower: 0.0000/1.0000 | Slower: 0.0040/1.0000 | Slower: 0.0040/1.0000 | // * Legends * Mean : Arithmetic mean of all measurements Error : Half of 99.9% confidence interval StdDev : Standard deviation of all measurements Ratio : Mean of the ratio distribution ([Current]/[Baseline]) Welch(1us)/p-values : Welch-based TOST equivalence test with 1 us threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' Welch(3%)/p-values : Welch-based TOST equivalence test with 3% threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' MannWhitney(1us)/p-values : MannWhitney-based TOST equivalence test with 1 us threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' MannWhitney(3%)/p-values : MannWhitney-based TOST equivalence test with 3% threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' 1 ms : 1 Millisecond (0.001 sec) ``` ### Links * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroStatisticalTesting --- ================================================ FILE: docs/articles/samples/IntroStatisticsColumns.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroStatisticsColumns --- ## Sample: IntroStatisticsColumns ### Source code [!code-csharp[IntroStatisticsColumns.cs](../../../samples/BenchmarkDotNet.Samples/IntroStatisticsColumns.cs)] ### Output | Method | Mean | Error | StdDev | Skewness | Kurtosis | Ratio | RatioSD | |------- |---------:|----------:|----------:|---------:|---------:|------:|--------:| | Md5A | 15.91 us | 0.0807 us | 0.1209 us | 0.4067 | 1.646 | 1.00 | 0.00 | | Md5B | 15.89 us | 0.0709 us | 0.1062 us | 0.5893 | 2.141 | 1.00 | 0.01 | | Sha256 | 36.62 us | 0.6390 us | 0.9564 us | 1.1363 | 4.014 | 2.30 | 0.06 | ### Links * @docs.statistics * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroStatisticsColumns --- ================================================ FILE: docs/articles/samples/IntroStopOnFirstError.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroStopOnFirstError --- ## Sample: IntroStopOnFirstError BenchmarkDotNet can be configured to stop on first error. You just have to add `StopOnFirstError` attribute to your class or use `--stopOnFirstError` command line argument. ### Source code [!code-csharp[IntroStopOnFirstError.cs](../../../samples/BenchmarkDotNet.Samples/IntroStopOnFirstError.cs)] ### Links * @docs.configs * @docs.console-args * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroStopOnFirstError --- ================================================ FILE: docs/articles/samples/IntroSummaryStyle.md ================================================ --- uid: BenchmarkDotNet.SummaryStyle --- ## SummaryStyle in BenchmarkDotNet `SummaryStyle` is a class in BenchmarkDotNet that allows customization of the summary reports of benchmark results. It offers several properties to fine-tune how the results are displayed. ### Usage You can customize the summary report by specifying various properties of `SummaryStyle`. These properties include formatting options like whether to print units in the header or content, setting the maximum width for parameter columns, and choosing units for size and time measurements. ### Source Code [!code-csharp[IntroSummaryStyle.cs](../../../samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs)] ### Properties - `PrintUnitsInHeader`: Boolean to indicate if units should be printed in the header. - `PrintUnitsInContent`: Boolean to control unit printing in the content. - `PrintZeroValuesInContent`: Determines if zero values should be printed. - `MaxParameterColumnWidth`: Integer defining the max width for parameter columns. - `SizeUnit`: Optional `SizeUnit` to specify the unit for size measurements. - `TimeUnit`: Optional `TimeUnit` for time measurement units. - `CultureInfo`: `CultureInfo` to define culture-specific formatting. ### Example Output Using SummaryStyle options: ```markdown | Method | N | Mean [ns] | Error [ns] | StdDev [ns] | |------- |---- |--------------:|-----------:|------------:| | Sleep | 10 | 15,644,973.1 | 32,808.7 | 30,689.3 | | Sleep | 100 | 109,440,686.7 | 236,673.8 | 221,384.8 | ``` Default: ```markdown | Method | N | Mean | Error | StdDev | |------- |---- |----------:|---------:|---------:| | Sleep | 10 | 15.65 ms | 0.039 ms | 0.034 ms | | Sleep | 100 | 109.20 ms | 0.442 ms | 0.392 ms | ``` ### Links * @docs.SummaryStyle * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroSummaryStyle ================================================ FILE: docs/articles/samples/IntroTagColumn.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroTagColumn --- ## Sample: IntroTagColumn In the following example, we introduce two new columns which contains a tag based on a benchmark method name. ### Source code [!code-csharp[IntroTagColumn.cs](../../../samples/BenchmarkDotNet.Samples/IntroTagColumn.cs)] ### Output ```markdown | Method | Mean | Kind | Number | | ------ | ---------- | ---- | ------ | | Bar34 | 10.3636 ms | Bar | 34 | | Bar3 | 10.4662 ms | Bar | 3 | | Foo12 | 10.1377 ms | Foo | 12 | | Foo1 | 10.2814 ms | Foo | 1 | ``` ### Links * @docs.columns * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroTagColumn --- ================================================ FILE: docs/articles/samples/IntroTailcall.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroTailcall --- ## Sample: IntroTailcall You need to use the `TailcallDiagnoser` attribute to configure it. The available options are: * logFailuresOnly: Track only the methods that failed to get tail called. True by default. * filterByNamespace : Track only the methods from declaring type's namespace. Set to false if you want to see all Jit tail events. True by default. ### Restrictions * Windows only * x64 ### Source code [!code-csharp[IntroTailcall.cs](../../../samples/BenchmarkDotNet.Samples/IntroTailcall.cs)] ### Output ```markdown // * Diagnostic Output - TailCallDiagnoser * -------------------- -------------------- Jit_TailCalling.Calc: LegacyJitX64(Jit=LegacyJit, Platform=X64, Runtime=Clr) -------------------- -------------------- Jit_TailCalling.Calc: LegacyJitX86(Jit=LegacyJit, Platform=X86, Runtime=Clr) -------------------- -------------------- Jit_TailCalling.Calc: RyuJitX64(Jit=RyuJit, Platform=X64) -------------------- Caller: . - Callee: BenchmarkDotNet.Samples.JIT.Jit_TailCalling.FactorialWithTailing - int64 (int32,int32) Tail prefix: False Tail call type: RecursiveLoop ------------------- ``` ### Links * @docs.diagnosers * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroTailcall --- ================================================ FILE: docs/articles/samples/IntroTemplate.txt ================================================ --- uid: BenchmarkDotNet.Samples.SAMPLENAME --- ## Sample: SAMPLENAME ### Source code [!code-csharp[SAMPLENAME.cs](../../../samples/BenchmarkDotNet.Samples/SAMPLENAME.cs)] ### Output ### Links * TODO * The permanent link to this sample: @BenchmarkDotNet.Samples.SAMPLENAME --- ================================================ FILE: docs/articles/samples/IntroThreadingDiagnoser.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroThreadingDiagnoser --- ## Sample: IntroThreadingDiagnoser The `ThreadingDiagnoser` uses new APIs [exposed](https://github.com/dotnet/corefx/issues/35500) in .NET Core 3.0 to report: * Completed Work Items: The number of work items that have been processed in ThreadPool (per single operation) * Lock Contentions: The number of times there **was contention** upon trying to take a Monitor's lock (per single operation) ### Source code [!code-csharp[IntroThreadingDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroThreadingDiagnoser.cs)] ### Output | Method | Mean | StdDev | Median | Completed Work Items | Lock Contentions | |-------------------- |--------------:|-----------:|--------------:|---------------------:|-----------------:| | CompleteOneWorkItem | 8,073.5519 ns | 69.7261 ns | 8,111.6074 ns | 1.0000 | - | ### Links * @docs.diagnosers * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroThreadingDiagnoser --- ================================================ FILE: docs/articles/samples/IntroUnicode.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroUnicode --- ## Sample: IntroUnicode Some of the BenchmarkDotNet exporters use Unicode symbols that are not ASCII-compatible (e.g., `μ` or `±`). Unfortunately, some terminals are not supported such symbols. That's why BenchmarkDotNet prints only ASCII characters by default (`μ` will be replaced by `u`). If you want to display Unicode symbols in your terminal, you should use `[UnicodeConsoleLoggerAttribute]` (see usage examples below). > [!WARNING] > This feature works only with terminal(s)|text editor(s) that support Unicode. > On Windows, you may have some troubles with Unicode symbols > if system default code page configured as non-English > (in Control Panel + Regional and Language Options, Language for Non-Unicode Programs). ### Source code [!code-csharp[IntroUnicode.cs](../../../samples/BenchmarkDotNet.Samples/IntroUnicode.cs)] ### Output ```markdown Mean = 1.0265 μs, StdErr = 0.0005 μs (0.05%); N = 15, StdDev = 0.0018 μs Min = 1.0239 μs, Q1 = 1.0248 μs, Median = 1.0264 μs, Q3 = 1.0280 μs, Max = 1.0296 μs IQR = 0.0033 μs, LowerFence = 1.0199 μs, UpperFence = 1.0329 μs ConfidenceInterval = [1.0245 μs; 1.0285 μs] (CI 99.9%), Margin = 0.0020 μs (0.19% of Mean) Skewness = 0.12, Kurtosis = 1.56, MValue = 2 -------------------- Histogram -------------------- [1.023 μs ; 1.030 μs) | @@@@@@@@@@@@@@@ --------------------------------------------------- ``` ```markdown Method | Mean | Error | StdDev | ------- |---------:|----------:|----------:| Foo | 1.027 μs | 0.0020 μs | 0.0018 μs | ``` ### Links * @BenchmarkDotNet.Attributes.UnicodeConsoleLoggerAttribute * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroUnicode --- ================================================ FILE: docs/articles/samples/IntroVisualStudioProfiler.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroVisualStudioProfiler --- ## Sample: Visual Studio Profiler Using the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package you can capture performance profiles of your benchmarks that can be opened in Visual Studio. ### Source code [!code-csharp[IntroVisualStudioDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs)] ### Output The output will contain a path to the collected diagsession and automatically open in Visual Studio when launched from it. ```markdown // * Diagnostic Output - VSDiagnosticsDiagnoser * Collection result moved to 'C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\BenchmarkDotNet.Artifacts\BenchmarkDotNet_IntroVisualStudioProfiler_20241205_192056.diagsession'. Session : {d54ebddb-2d6d-404f-b1da-10acbc89635f} Stopped Exported diagsession file: C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\BenchmarkDotNet.Artifacts\BenchmarkDotNet_IntroVisualStudioProfiler_20241205_192056.diagsession. Opening diagsession in VisualStudio: 15296 ``` ================================================ FILE: docs/articles/samples/IntroWakeLock.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroWakeLock --- ## Sample: IntroWakeLock Running benchmarks may sometimes take enough time such that the system enters sleep or turns off the display. Using a WakeLock prevents the Windows system doing so. ### Source code [!code-csharp[IntroWakeLock.cs](../../../samples/BenchmarkDotNet.Samples/IntroWakeLock.cs)] ### Command line ``` --wakeLock None ``` ``` --wakeLock System ``` ``` --wakeLock Display ``` ### Links * @BenchmarkDotNet.Attributes.WakeLockAttribute * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroWakeLock --- ================================================ FILE: docs/articles/samples/IntroWasm.md ================================================ --- uid: BenchmarkDotNet.Samples.IntroWasm --- ## Sample: IntroWasm `WasmToolchain` builds benchmarks as WebAssembly and runs them under a JavaScript engine (V8 by default). It is supported on Windows, Linux, and macOS. If you hit `NETSDK1147` (missing workload), install the required workload (for example: `dotnet workload install wasm-tools`). ### Source code [!code-csharp[IntroWasm.cs](../../../samples/BenchmarkDotNet.Samples/IntroWasm.cs)] ### Links * @docs.toolchains * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroWasm --- ================================================ FILE: docs/articles/samples/toc.yml ================================================ - name: IntroArguments href: IntroArguments.md - name: IntroArgumentsSource href: IntroArgumentsSource.md - name: IntroArrayParam href: IntroArrayParam.md - name: IntroBasic href: IntroBasic.md - name: IntroBenchmarkBaseline href: IntroBenchmarkBaseline.md - name: IntroCategories href: IntroCategories.md - name: IntroCategoryBaseline href: IntroCategoryBaseline.md - name: IntroCategoryDiscoverer href: IntroCategoryDiscoverer.md - name: IntroColdStart href: IntroColdStart.md - name: IntroComparableComplexParam href: IntroComparableComplexParam.md - name: IntroConfigSource href: IntroConfigSource.md - name: IntroConfigUnion href: IntroConfigUnion.md - name: IntroCustomMono href: IntroCustomMono.md - name: IntroCustomMonoArguments href: IntroCustomMonoArguments.md - name: IntroDeferredExecution href: IntroDeferredExecution.md - name: IntroDisassembly href: IntroDisassembly.md - name: IntroDisassemblyAllJits href: IntroDisassemblyAllJits.md - name: IntroDisassemblyDry href: IntroDisassemblyDry.md - name: IntroDisassemblyRyuJit href: IntroDisassemblyRyuJit.md - name: IntroDotTraceDiagnoser href: IntroDotTraceDiagnoser.md - name: IntroDotMemoryDiagnoser href: IntroDotMemoryDiagnoser.md - name: IntroEnvVars href: IntroEnvVars.md - name: IntroEventPipeProfiler href: IntroEventPipeProfiler.md - name: IntroEventPipeProfilerAdvanced href: IntroEventPipeProfilerAdvanced.md - name: IntroExport href: IntroExport.md - name: IntroExportJson href: IntroExportJson.md - name: IntroExportXml href: IntroExportXml.md - name: IntroFilters href: IntroFilters.md - name: IntroFluentConfigBuilder href: IntroFluentConfigBuilder.md - name: IntroGcMode href: IntroGcMode.md - name: IntroGenericTypeArguments href: IntroGenericTypeArguments.md - name: IntroHardwareCounters href: IntroHardwareCounters.md - name: IntroInliningDiagnoser href: IntroInliningDiagnoser.md - name: IntroInProcess href: IntroInProcess.md - name: IntroInProcessWrongEnv href: IntroInProcessWrongEnv.md - name: IntroJitStatsDiagnoser href: IntroJitStatsDiagnoser.md - name: IntroJobBaseline href: IntroJobBaseline.md - name: IntroJoin href: IntroJoin.md - name: IntroLargeAddressAware href: IntroLargeAddressAware.md - name: IntroMonitoring href: IntroMonitoring.md - name: IntroMultimodal href: IntroMultimodal.md - name: IntroNativeMemory href: IntroNativeMemory.md - name: IntroNuGet href: IntroNuGet.md - name: IntroOrderAttr href: IntroOrderAttr.md - name: IntroOrderManual href: IntroOrderManual.md - name: IntroOutliers href: IntroOutliers.md - name: IntroParams href: IntroParams.md - name: IntroParamsAllValues href: IntroParamsAllValues.md - name: IntroParamsSource href: IntroParamsSource.md - name: IntroPercentiles href: IntroPercentiles.md - name: IntroPowerPlan href: IntroPowerPlan.md - name: IntroRankColumn href: IntroRankColumn.md - name: IntroRatioSD href: IntroRatioSD.md - name: IntroRatioStyle href: IntroRatioStyle.md - name: IntroSetupCleanupGlobal href: IntroSetupCleanupGlobal.md - name: IntroSetupCleanupIteration href: IntroSetupCleanupIteration.md - name: IntroSetupCleanupTarget href: IntroSetupCleanupTarget.md - name: IntroStaThread href: IntroStaThread.md - name: IntroStatisticalTesting href: IntroStatisticalTesting.md - name: IntroStatisticsColumns href: IntroStatisticsColumns.md - name: IntroStopOnFirstError href: IntroStopOnFirstError.md - name: IntroTagColumn href: IntroTagColumn.md - name: IntroTailcall href: IntroTailcall.md - name: IntroVisualStudioProfiler href: IntroVisualStudioProfiler.md - name: IntroWakeLock href: IntroWakeLock.md - name: IntroWasm href: IntroWasm.md - name: IntroUnicode href: IntroUnicode.md - name: IntroMaui href: IntroMaui.md ================================================ FILE: docs/articles/team.md ================================================ # Team Maintainers: [Andrey Akinshin](https://github.com/AndreyAkinshin) (Project Lead), [Adam Sitnik](https://github.com/adamsitnik), [Yegor Stepanov](https://github.com/YegorStepanov), [Tim Cassell](https://github.com/timcassell). [All contributors on GitHub (200+)](https://github.com/dotnet/BenchmarkDotNet/graphs/contributors) BenchmarkDotNet is a part of the [.NET Foundation](https://dotnetfoundation.org). ================================================ FILE: docs/articles/toc.yml ================================================ - name: Overview href: overview.md - name: Guides href: guides/toc.yml - name: Features href: features/toc.yml - name: Configs href: configs/toc.yml - name: Samples href: samples/toc.yml - name: Contributing href: contributing/toc.yml - name: FAQ href: faq.md - name: Team href: team.md - name: License href: license.md - name: Analyzers href: analyzers.md ================================================ FILE: docs/docfx.json ================================================ { "metadata": [ { "src": [ { "files": [ "src/BenchmarkDotNet/bin/Release/netstandard2.0/BenchmarkDotNet.dll", "src/BenchmarkDotNet/bin/Release/netstandard2.0/BenchmarkDotNet.Annotations.dll" ], "src": ".." } ], "dest": "api", "filter": "filter.yml", "properties": { "TargetFramework": "netstandard2.0" }, "disableGitFeatures": true } ], "build": { "content": [ { "files": [ "api/**.yml", "api/index.md" ] }, { "files": [ "articles/**.md", "articles/**/toc.yml", "changelog/toc.yml", "changelog/*.md", "toc.yml", "*.md" ] } ], "resource": [ { "files": [ "images/**", "logo/**" ] } ], "dest": "_site", "globalMetadataFiles": [], "fileMetadataFiles": [], "template": [ "default", "modern", "template" ], "xrefService": [ "https://xref.docs.microsoft.com/query?uid={uid}" ], "markdownEngineName": "markdig", "noLangKeyword": false, "keepFileLink": false, "cleanupCacheHistory": false, "disableGitFeatures": false, "globalMetadata" : { "_appTitle" : "BenchmarkDotNet", "_appFooter" : "Copyright © 2013–2024 .NET Foundation and contributors", "_appLogoPath" : "logo/icon.svg", "_appFaviconPath": "logo/icon-32.png", "_enableSearch": true, "_disableContribution": true }, "sitemap": { "baseUrl": "https://benchmarkdotnet.org/", "priority": 0.8, "changefreq": "monthly", "fileOptions":{ "**/articles/**": { "priority": 0.8 }, "**/changelog/**": { "priority": 0.5 }, "**/api/**": { "priority": 0.3 } } } } } ================================================ FILE: docs/filter.yml ================================================ apiRules: - exclude: uidRegex: ^System\.Object type: member # Avoid list of inherited Object members for each type. ================================================ FILE: docs/guide/README.md ================================================ Old versions of the NuGet package use `https://raw.githubusercontent.com/dotnet/BenchmarkDotNet/master/docs/guide/logo.png` as a logo url. So, we have to keep the `logo.png` file for backward compatibility. ================================================ FILE: docs/template/public/main.css ================================================ #logo { width: 50px; height: 50px; } #breadcrumb { display: none; } .affix { display: none !important; } ================================================ FILE: docs/template/public/main.js ================================================ export default { iconLinks: [ { icon: 'github', href: 'https://github.com/dotnet/BenchmarkDotNet', title: 'GitHub' }, { icon: 'twitter', href: 'https://twitter.com/BenchmarkDotNet', title: 'Twitter' }, { icon: 'heart', href: 'https://github.com/sponsors/AndreyAkinshin', title: 'Sponsor' } ] } ================================================ FILE: docs/toc.yml ================================================ - name: Articles href: articles/ - name: API href: api/ homepage: api/index.md - name: ChangeLog href: changelog/ homepage: changelog/index.md ================================================ FILE: samples/BenchmarkDotNet.Maui.slnx ================================================ ================================================ FILE: samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj ================================================  BenchmarkDotNet.Samples net8.0;net462 true BenchmarkDotNet.Samples Exe BenchmarkDotNet.Samples AnyCPU true $(NoWarn);CA1018;CA5351;CA1825 false false 8.0.0 ================================================ FILE: samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj.DotSettings ================================================  True True True DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW True DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW DO_NOT_SHOW ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroArguments.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroArguments { [Params(true, false)] // Arguments can be combined with Params public bool AddExtra5Milliseconds; [Benchmark] [Arguments(100, 10)] [Arguments(100, 20)] [Arguments(200, 10)] [Arguments(200, 20)] public void Benchmark(int a, int b) { if (AddExtra5Milliseconds) Thread.Sleep(a + b + 5); else Thread.Sleep(a + b); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroArgumentsPriority.cs ================================================ using System.Collections.Generic; using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroArgumentsPriority { [Params(100, Priority = 0)] // Argument priority can be combined with Params priority public int A { get; set; } [Arguments(5, Priority = -10)] // Define priority just once for multiple argument attributes [Arguments(10)] [Arguments(20)] [Benchmark] public void Benchmark(int b) => Thread.Sleep(A + b); [Benchmark] [ArgumentsSource(nameof(Numbers), Priority = 10)] public void ManyArguments(int c, int d) => Thread.Sleep(A + c + d); public IEnumerable Numbers() { yield return new object[] { 1, 2 }; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs ================================================ using System; using System.Collections.Generic; using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroArgumentsSource { [Benchmark] [ArgumentsSource(nameof(Numbers))] public double ManyArguments(double x, double y) => Math.Pow(x, y); public IEnumerable Numbers() // for multiple arguments it's an IEnumerable of array of objects (object[]) { yield return new object[] { 1.0, 1.0 }; yield return new object[] { 2.0, 2.0 }; yield return new object[] { 4.0, 4.0 }; yield return new object[] { 10.0, 10.0 }; } [Benchmark] [ArgumentsSource(typeof(BenchmarkArguments), nameof(BenchmarkArguments.TimeSpans))] // when the arguments come from a different type, specify that type here public void SingleArgument(TimeSpan time) => Thread.Sleep(time); } public static class BenchmarkArguments { public static IEnumerable TimeSpans() // for single argument it's an IEnumerable of objects (object) { yield return TimeSpan.FromMilliseconds(10); yield return TimeSpan.FromMilliseconds(100); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroArrayParam.cs ================================================ using System; using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroArrayParam { [Benchmark] [ArgumentsSource(nameof(Data))] public int ArrayIndexOf(int[] array, int value) => Array.IndexOf(array, value); [Benchmark] [ArgumentsSource(nameof(Data))] public int ManualIndexOf(int[] array, int value) { for (int i = 0; i < array.Length; i++) if (array[i] == value) return i; return -1; } public IEnumerable Data() { yield return new object[] { new int[] { 1, 2, 3 }, 4 }; yield return new object[] { Enumerable.Range(0, 100).ToArray(), 4 }; yield return new object[] { Enumerable.Range(0, 100).ToArray(), 101 }; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroBasic.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { // It is very easy to use BenchmarkDotNet. You should just create a class public class IntroBasic { // And define a method with the Benchmark attribute [Benchmark] public void Sleep() => Thread.Sleep(10); // You can write a description for your method. [Benchmark(Description = "Thread.Sleep(10)")] public void SleepWithDescription() => Thread.Sleep(10); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroBenchmarkBaseline.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroBenchmarkBaseline { [Benchmark] public void Time50() => Thread.Sleep(50); [Benchmark(Baseline = true)] public void Time100() => Thread.Sleep(100); [Benchmark] public void Time150() => Thread.Sleep(150); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroCategories.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [DryJob] [CategoriesColumn] [BenchmarkCategory("Awesome")] [AnyCategoriesFilter("A", "1")] public class IntroCategories { [Benchmark] [BenchmarkCategory("A", "1")] public void A1() => Thread.Sleep(10); // Will be benchmarked [Benchmark] [BenchmarkCategory("A", "2")] public void A2() => Thread.Sleep(10); // Will be benchmarked [Benchmark] [BenchmarkCategory("B", "1")] public void B1() => Thread.Sleep(10); // Will be benchmarked [Benchmark] [BenchmarkCategory("B", "2")] public void B2() => Thread.Sleep(10); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroCategoryBaseline.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; namespace BenchmarkDotNet.Samples { [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] [CategoriesColumn] public class IntroCategoryBaseline { [BenchmarkCategory("Fast"), Benchmark(Baseline = true)] public void Time50() => Thread.Sleep(50); [BenchmarkCategory("Fast"), Benchmark] public void Time100() => Thread.Sleep(100); [BenchmarkCategory("Slow"), Benchmark(Baseline = true)] public void Time550() => Thread.Sleep(550); [BenchmarkCategory("Slow"), Benchmark] public void Time600() => Thread.Sleep(600); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs ================================================ using System; using System.Collections.Generic; using System.Reflection; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Samples { [DryJob] [CategoriesColumn] [CustomCategoryDiscoverer] public class IntroCategoryDiscoverer { private class CustomCategoryDiscoverer : DefaultCategoryDiscoverer { public override string[] GetCategories(MethodInfo method) { var categories = new List(); categories.AddRange(base.GetCategories(method)); categories.Add("All"); categories.Add(method.Name.Substring(0, 1)); return categories.ToArray(); } } [AttributeUsage(AttributeTargets.Class)] private class CustomCategoryDiscovererAttribute : Attribute, IConfigSource { public CustomCategoryDiscovererAttribute() { Config = ManualConfig.CreateEmpty() .WithCategoryDiscoverer(new CustomCategoryDiscoverer()); } public IConfig Config { get; } } [Benchmark] public void Foo() { } [Benchmark] public void Bar() { } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroColdStart.cs ================================================ using System; using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; namespace BenchmarkDotNet.Samples { [SimpleJob(RunStrategy.ColdStart, iterationCount: 5)] [MinColumn, MaxColumn, MeanColumn, MedianColumn] public class IntroColdStart { private bool firstCall; [Benchmark] public void Foo() { if (firstCall == false) { firstCall = true; Console.WriteLine("// First call"); Thread.Sleep(1000); } else Thread.Sleep(10); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroComparableComplexParam.cs ================================================ using System; using System.Collections.Generic; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroComparableComplexParam { [ParamsSource(nameof(ValuesForA))] public ComplexParam? A { get; set; } public IEnumerable ValuesForA => [new ComplexParam(1, "First"), new ComplexParam(2, "Second")]; [Benchmark] public object? Benchmark() => A; // Only non generic IComparable is required to provide custom order behavior, but implementing IComparable<> too is customary. public class ComplexParam : IComparable, IComparable { public ComplexParam(int value, string name) { Value = value; Name = name; } public int Value { get; set; } public string Name { get; set; } public override string ToString() => Name; public int CompareTo(ComplexParam? other) => other == null ? 1 : Value.CompareTo(other.Value); public int CompareTo(object? obj) { if (obj == null) return 1; if (obj is not ComplexParam other) throw new ArgumentException(); return CompareTo(other); } } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroConfigSource.cs ================================================ using System; using System.Linq; using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { [MyConfigSource(Jit.LegacyJit, Jit.RyuJit)] public class IntroConfigSource { /// /// Dry-x64 jobs for specific jits /// private class MyConfigSourceAttribute : Attribute, IConfigSource { public IConfig Config { get; } public MyConfigSourceAttribute(params Jit[] jits) { var jobs = jits .Select(jit => new Job(Job.Dry) { Environment = { Jit = jit, Platform = Platform.X64 } }) .ToArray(); Config = ManualConfig.CreateEmpty().AddJob(jobs); } } [Benchmark] public void Foo() { Thread.Sleep(10); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroConfigUnion.cs ================================================ using System.Threading; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] public class IntroConfigUnion { private class Config : ManualConfig { public Config() { AddJob(Job.Dry); AddLogger(ConsoleLogger.Default); AddColumn(TargetMethodColumn.Method, StatisticColumn.Max); AddExporter(RPlotExporter.Default, CsvExporter.Default); AddAnalyser(EnvironmentAnalyser.Default); UnionRule = ConfigUnionRule.AlwaysUseLocal; } } [Benchmark] public void Foo() { Thread.Sleep(10); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroCultureInfo.cs ================================================ using System.Globalization; using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] [ShortRunJob] public class IntroCultureInfo { private class Config : ManualConfig { public Config() { CultureInfo = (CultureInfo) CultureInfo.InvariantCulture.Clone(); CultureInfo.NumberFormat.NumberDecimalSeparator = "@"; } } [Benchmark] public void Foo() => Thread.Sleep(100); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroCustomMono.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Samples { // *** Attribute Style *** [MonoJob("Mono x64", @"C:\Program Files\Mono\bin\mono.exe")] [MonoJob("Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe")] public class IntroCustomMono { [Benchmark] public void Foo() { // Benchmark body } } // *** Object Style *** [Config(typeof(Config))] public class IntroCustomMonoObjectStyle { private class Config : ManualConfig { public Config() { AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( "Mono x64", @"C:\Program Files\Mono\bin\mono.exe"))); AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( "Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe"))); } } [Benchmark] public void Foo() { // Benchmark body } } // ** Object Style, Using AOT ** [Config(typeof(Config))] public class IntroCustomMonoObjectStyleAot { private class Config : ManualConfig { public void AddMono (string name, string mono_top_dir) { var aot_compile_args = "--aot=llvm"; var mono_bcl = $@"{mono_top_dir}\lib\mono\4.5"; var mono_bin = $@"{mono_top_dir}\bin\mono.exe"; AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( name, mono_bin, aot_compile_args, mono_bcl))); } public Config() { AddMono("Mono x64", @"C:\Program Files\Mono"); AddMono("Mono x86", @"C:\Program Files (x86)\Mono"); } } [Benchmark] public void Foo() { // Benchmark body } } // *** Fluent Config *** public class IntroCustomMonoFluentConfig { public static void Run() { BenchmarkRunner.Run(ManualConfig .CreateMinimumViable() .AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( "Mono x64", @"C:\Program Files\Mono\bin\mono.exe"))) .AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( "Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe")))); } [Benchmark] public void Foo() { // Benchmark body } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroCustomMonoArguments.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { [Config(typeof(ConfigWithCustomArguments))] public class IntroCustomMonoArguments { public class ConfigWithCustomArguments : ManualConfig { public ConfigWithCustomArguments() { // --optimize=MODE , -O=mode // MODE is a comma separated list of optimizations. They also allow // optimizations to be turned off by prefixing the optimization // name with a minus sign. AddJob(Job.Default .WithRuntime(MonoRuntime.Default) .WithArguments([new MonoArgument("--optimize=inline")]) .WithId("Inlining enabled")); AddJob(Job.Default .WithRuntime(MonoRuntime.Default) .WithArguments([new MonoArgument("--optimize=-inline")]) .WithId("Inlining disabled")); } } [Benchmark] public void Sample() { ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined(); ShouldGetInlined(); } private void ShouldGetInlined() { } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroDeferredExecution.cs ================================================ using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; namespace BenchmarkDotNet.Samples { public class IntroDeferredExecution { private readonly int[] numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; private readonly Consumer consumer = new Consumer(); /// /// this benchmark returns a deferred LINQ query which is NOT executed /// so the benchmark measures the cost of creating the query, not the actual execution /// this is WRONG /// You can read more about LINQ and Deferred Execution here /// /// deferred LINQ query [Benchmark] public IEnumerable Wrong() => from number in numbers orderby number descending select number; /// /// this benchmark uses .Consume extension method which executes given deferred query and consumes its result /// so the benchmark measures the cost of creating the query and executing it /// [Benchmark] public void Ok() => (from number in numbers orderby number descending select number).Consume(consumer); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroDisassembly.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnosers; using System.Linq; namespace BenchmarkDotNet.Samples { [DisassemblyDiagnoser(printInstructionAddresses: true, syntax: DisassemblySyntax.Masm)] public class IntroDisassembly { private int[] field = Enumerable.Range(0, 100).ToArray(); [Benchmark] public int SumLocal() { var local = field; // we use local variable that points to the field int sum = 0; for (int i = 0; i < local.Length; i++) sum += local[i]; return sum; } [Benchmark] public int SumField() { int sum = 0; for (int i = 0; i < field.Length; i++) sum += field[i]; return sum; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroDisassemblyAllJits.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { [Config(typeof(MultipleJits))] public class IntroDisassemblyAllJits { public class MultipleJits : ManualConfig { public MultipleJits() { AddJob(Job.ShortRun.WithPlatform(Platform.X86).WithRuntime(new MonoRuntime(name: "Mono x86", customPath: @"C:\Program Files (x86)\Mono\bin\mono.exe"))); AddJob(Job.ShortRun.WithPlatform(Platform.X64).WithRuntime(new MonoRuntime(name: "Mono x64", customPath: @"C:\Program Files\Mono\bin\mono.exe"))); AddJob(Job.ShortRun.WithJit(Jit.LegacyJit).WithPlatform(Platform.X86).WithRuntime(ClrRuntime.Net462)); AddJob(Job.ShortRun.WithJit(Jit.LegacyJit).WithPlatform(Platform.X64).WithRuntime(ClrRuntime.Net462)); AddJob(Job.ShortRun.WithJit(Jit.RyuJit).WithPlatform(Platform.X64).WithRuntime(ClrRuntime.Net462)); // RyuJit for .NET Core 5.0 AddJob(Job.ShortRun.WithJit(Jit.RyuJit).WithPlatform(Platform.X64).WithRuntime(CoreRuntime.Core50)); AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig(maxDepth: 3, exportDiff: true))); } } private Increment increment = new Increment(); [Benchmark] public int CallVirtualMethod() => increment.OperateTwice(10); public abstract class Operation // abstract unary integer operation { public abstract int Operate(int input); public int OperateTwice(int input) => Operate(Operate(input)); // two virtual calls to Operate } public sealed class Increment : Operation // concrete, sealed operation: increment by fixed amount { public readonly int Amount; public Increment(int amount = 1) { Amount = amount; } public override int Operate(int input) => input + Amount; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroDisassemblyDry.cs ================================================ using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [DisassemblyDiagnoser(maxDepth: 3)] [DryJob] public class IntroDisassemblyDry { [Benchmark] public void Foo() { } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroDisassemblyRyuJit.cs ================================================ using System.Linq; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [DisassemblyDiagnoser(printSource: true)] [RyuJitX64Job] public class IntroDisassemblyRyuJit { private int[] field = Enumerable.Range(0, 100).ToArray(); [Benchmark] public int SumLocal() { var local = field; // we use local variable that points to the field int sum = 0; for (int i = 0; i < local.Length; i++) sum += local[i]; return sum; } [Benchmark] public int SumField() { int sum = 0; for (int i = 0; i < field.Length; i++) sum += field[i]; return sum; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnostics.dotMemory; using System.Collections.Generic; namespace BenchmarkDotNet.Samples { // Profile benchmarks via dotMemory SelfApi profiling for all jobs [DotMemoryDiagnoser] [SimpleJob] // external-process execution [InProcess] // in-process execution public class IntroDotMemoryDiagnoser { [Params(1024)] public int Size; private byte[] dataArray = default!; private IEnumerable dataEnumerable = default!; [GlobalSetup] public void Setup() { dataArray = new byte[Size]; dataEnumerable = dataArray; } [Benchmark] public int IterateArray() { var count = 0; foreach (var _ in dataArray) count++; return count; } [Benchmark] public int IterateEnumerable() { var count = 0; foreach (var _ in dataEnumerable) count++; return count; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnostics.dotTrace; namespace BenchmarkDotNet.Samples { // Profile benchmarks via dotTrace SelfApi profiling for all jobs // See: https://www.nuget.org/packages/JetBrains.Profiler.SelfApi [DotTraceDiagnoser] [SimpleJob] // external-process execution [InProcess] // in-process execution public class IntroDotTraceDiagnoser { [Benchmark] public void Fibonacci() => Fibonacci(30); private static int Fibonacci(int n) { return n <= 1 ? n : Fibonacci(n - 1) + Fibonacci(n - 2); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroEnvVars.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { [Config(typeof(ConfigWithCustomEnvVars))] public class IntroEnvVars { private class ConfigWithCustomEnvVars : ManualConfig { public ConfigWithCustomEnvVars() { AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithId("Inlining enabled")); AddJob(Job.Default.WithRuntime(CoreRuntime.Core80) .WithEnvironmentVariables([ new EnvironmentVariable("DOTNET_JitNoInline", "1"), new EnvironmentVariable("COMPlus_JitNoInline", "1") ]) .WithId("Inlining disabled")); } } [Benchmark] public void Foo() { // Benchmark body } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroEventPipeProfiler.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnosers; namespace BenchmarkDotNet.Samples { [ShortRunJob] [EventPipeProfiler(EventPipeProfile.CpuSampling)] public class IntroEventPipeProfiler { [Benchmark] public void Sleep() => Thread.Sleep(2000); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroEventPipeProfilerAdvanced.cs ================================================ using System.Buffers; using System.Diagnostics.Tracing; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using Microsoft.Diagnostics.NETCore.Client; using Microsoft.Diagnostics.Tracing.Parsers; namespace BenchmarkDotNet.Samples { [Config(typeof(CustomConfig))] public class IntroEventPipeProfilerAdvanced { private class CustomConfig : ManualConfig { public CustomConfig() { AddJob(Job.ShortRun.WithRuntime(CoreRuntime.Core50)); var providers = new[] { new EventPipeProvider(ClrTraceEventParser.ProviderName, EventLevel.Verbose, (long) (ClrTraceEventParser.Keywords.Exception | ClrTraceEventParser.Keywords.GC | ClrTraceEventParser.Keywords.Jit | ClrTraceEventParser.Keywords.JitTracing // for the inlining events | ClrTraceEventParser.Keywords.Loader | ClrTraceEventParser.Keywords.NGen)), new EventPipeProvider("System.Buffers.ArrayPoolEventSource", EventLevel.Informational, long.MaxValue), }; AddDiagnoser(new EventPipeProfiler(providers: providers)); } } [Benchmark] public void RentAndReturn_Shared() { var pool = ArrayPool.Shared; byte[] array = pool.Rent(10000); pool.Return(array); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs ================================================ using BenchmarkDotNet.Attributes; using System; namespace BenchmarkDotNet.Samples { [ExceptionDiagnoser] public class IntroExceptionDiagnoser { [Benchmark] public void ThrowExceptionRandomly() { try { if (new Random().Next(0, 5) > 1) { throw new Exception(); } } catch { // ignored } } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroExport.cs ================================================ using System; using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [ShortRunJob] [MediumRunJob] [KeepBenchmarkFiles] [AsciiDocExporter] [CsvExporter] [CsvMeasurementsExporter] [HtmlExporter] [PlainExporter] [RPlotExporter] [JsonExporterAttribute.Brief] [JsonExporterAttribute.BriefCompressed] [JsonExporterAttribute.Full] [JsonExporterAttribute.FullCompressed] [MarkdownExporterAttribute.Default] [MarkdownExporterAttribute.GitHub] [MarkdownExporterAttribute.StackOverflow] [MarkdownExporterAttribute.Atlassian] [XmlExporterAttribute.Brief] [XmlExporterAttribute.BriefCompressed] [XmlExporterAttribute.Full] [XmlExporterAttribute.FullCompressed] public class IntroExport { private Random random = new Random(42); [Benchmark(Baseline = true)] public void Sleep10() => Thread.Sleep(10); [Benchmark] public void Sleep50Noisy() => Thread.Sleep(random.Next(100)); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroExportJson.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Exporters.Json; namespace BenchmarkDotNet.Samples { // *** Attribute style *** [DryJob] [JsonExporterAttribute.Brief] [JsonExporterAttribute.Full] [JsonExporterAttribute.BriefCompressed] [JsonExporterAttribute.FullCompressed] [JsonExporter("-custom", indentJson: true, excludeMeasurements: true)] public class IntroExportJson { [Benchmark] public void Sleep10() => Thread.Sleep(10); [Benchmark] public void Sleep20() => Thread.Sleep(20); } // *** Object style *** [Config(typeof(Config))] public class IntroJsonExportObjectStyle { private class Config : ManualConfig { public Config() { AddExporter(JsonExporter.Brief); AddExporter(JsonExporter.Brief); AddExporter(JsonExporter.Full); AddExporter(JsonExporter.BriefCompressed); AddExporter(JsonExporter.FullCompressed); AddExporter(JsonExporter.Custom("-custom", indentJson: true, excludeMeasurements: true)); } } [Benchmark] public void Sleep10() => Thread.Sleep(10); [Benchmark] public void Sleep20() => Thread.Sleep(20); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroExportXml.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [DryJob] [XmlExporterAttribute.Brief] [XmlExporterAttribute.Full] [XmlExporterAttribute.BriefCompressed] [XmlExporterAttribute.FullCompressed] [XmlExporter("-custom", indentXml: true, excludeMeasurements: true)] public class IntroExportXml { [Benchmark] public void Sleep10() => Thread.Sleep(10); [Benchmark] public void Sleep20() => Thread.Sleep(20); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroFilters.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Filters; namespace BenchmarkDotNet.Samples { [DryJob] [Config(typeof(Config))] public class IntroFilters { private class Config : ManualConfig { // We will benchmark ONLY method with // names (which contains "A" OR "1") AND (have length < 3) public Config() { // benchmark with names which contains "A" OR "1" AddFilter(new DisjunctionFilter( new NameFilter(name => name.Contains("A")), new NameFilter(name => name.Contains("1")) )); // benchmark with names with length < 3 AddFilter(new NameFilter(name => name.Length < 3)); } } [Benchmark] public void A1() => Thread.Sleep(10); // Will be benchmarked [Benchmark] public void A2() => Thread.Sleep(10); // Will be benchmarked [Benchmark] public void A3() => Thread.Sleep(10); // Will be benchmarked [Benchmark] public void B1() => Thread.Sleep(10); // Will be benchmarked [Benchmark] public void B2() => Thread.Sleep(10); [Benchmark] public void B3() => Thread.Sleep(10); [Benchmark] public void C1() => Thread.Sleep(10); // Will be benchmarked [Benchmark] public void C2() => Thread.Sleep(10); [Benchmark] public void C3() => Thread.Sleep(10); [Benchmark] public void Aaa() => Thread.Sleep(10); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs ================================================ using System; using System.Security.Cryptography; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Samples { public class Algo_Md5VsSha256 { private const int N = 10000; private readonly byte[] data; private readonly MD5 md5 = MD5.Create(); private readonly SHA256 sha256 = SHA256.Create(); public Algo_Md5VsSha256() { data = new byte[N]; new Random(42).NextBytes(data); } [Benchmark(Baseline = true)] public byte[] Md5() => md5.ComputeHash(data); [Benchmark] public byte[] Sha256() => sha256.ComputeHash(data); } public class IntroFluentConfigBuilder { public static void Run() { BenchmarkRunner .Run( DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(ClrRuntime.Net462)) .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80)) .AddValidator(ExecutionValidator.FailOnError)); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroGcMode.cs ================================================ using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Order; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] [Orderer(SummaryOrderPolicy.FastestToSlowest)] [MemoryDiagnoser] public class IntroGcMode { private class Config : ManualConfig { public Config() { AddJob(Job.MediumRun.WithGcServer(true).WithGcForce(true).WithId("ServerForce")); AddJob(Job.MediumRun.WithGcServer(true).WithGcForce(false).WithId("Server")); AddJob(Job.MediumRun.WithGcServer(false).WithGcForce(true).WithId("Workstation")); AddJob(Job.MediumRun.WithGcServer(false).WithGcForce(false).WithId("WorkstationForce")); } } [Benchmark(Description = "new byte[10kB]")] public byte[] Allocate() { return new byte[10000]; } [Benchmark(Description = "stackalloc byte[10kB]")] public unsafe void AllocateWithStackalloc() { var array = stackalloc byte[10000]; Consume(array); } [MethodImpl(MethodImplOptions.NoInlining)] private static unsafe void Consume(byte* input) { } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroGenericTypeArguments.cs ================================================ using System; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [GenericTypeArguments(typeof(int))] [GenericTypeArguments(typeof(char))] public class IntroGenericTypeArguments { [Benchmark] public T Create() => Activator.CreateInstance(); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroHardwareCounters.cs ================================================ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnosers; namespace BenchmarkDotNet.Samples { [HardwareCounters( HardwareCounter.BranchMispredictions, HardwareCounter.BranchInstructions)] public class IntroHardwareCounters { private const int N = 32767; private readonly int[] sorted, unsorted; public IntroHardwareCounters() { var random = new Random(0); unsorted = new int[N]; sorted = new int[N]; for (int i = 0; i < N; i++) sorted[i] = unsorted[i] = random.Next(256); Array.Sort(sorted); } private static int Branch(int[] data) { int sum = 0; for (int i = 0; i < N; i++) if (data[i] >= 128) sum += data[i]; return sum; } private static int Branchless(int[] data) { int sum = 0; for (int i = 0; i < N; i++) { int t = (data[i] - 128) >> 31; sum += ~t & data[i]; } return sum; } [Benchmark] public int SortedBranch() => Branch(sorted); [Benchmark] public int UnsortedBranch() => Branch(unsorted); [Benchmark] public int SortedBranchless() => Branchless(sorted); [Benchmark] public int UnsortedBranchless() => Branchless(unsorted); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroHidingColumns.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; namespace BenchmarkDotNet.Samples { [MemoryDiagnoser] // adds Gen0, Gen1, Gen2 and Allocated Bytes columns [HideColumns(Column.Gen0, Column.Gen1, Column.Gen2)] // dont display GenX columns public class IntroHidingColumns { [Benchmark] public byte[] AllocateArray() => new byte[100_000]; } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroInProcess.cs ================================================ using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Order; using BenchmarkDotNet.Toolchains.InProcess.Emit; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] [Orderer(SummaryOrderPolicy.FastestToSlowest)] [MemoryDiagnoser] public class IntroInProcess { private class Config : ManualConfig { public Config() { AddJob(Job.MediumRun .WithLaunchCount(1) .WithId("OutOfProc")); AddJob(Job.MediumRun .WithLaunchCount(1) .WithToolchain(InProcessEmitToolchain.Default) .WithId("InProcess")); } } [Benchmark(Description = "new byte[10kB]")] public byte[] Allocate() { return new byte[10000]; } [Benchmark(Description = "stackalloc byte[10kB]")] public unsafe void AllocateWithStackalloc() { var array = stackalloc byte[10000]; Consume(array); } [MethodImpl(MethodImplOptions.NoInlining)] private static unsafe void Consume(byte* input) { } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroInProcessWrongEnv.cs ================================================ using System; using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Order; using BenchmarkDotNet.Toolchains.InProcess; using BenchmarkDotNet.Toolchains.InProcess.Emit; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] [Orderer(SummaryOrderPolicy.FastestToSlowest)] [MemoryDiagnoser] public class IntroInProcessWrongEnv { private class Config : ManualConfig { public Config() { var wrongPlatform = Environment.Is64BitProcess ? Platform.X64 : Platform.X86; AddJob(Job.MediumRun .WithLaunchCount(1) .WithPlatform(wrongPlatform) .WithToolchain(InProcessEmitToolchain.Default) .WithId("InProcess")); AddValidator(InProcessValidator.DontFailOnError); } } [Benchmark(Description = "new byte[10kB]")] public byte[] Allocate() { return new byte[10000]; } [Benchmark(Description = "stackalloc byte[10kB]")] public unsafe void AllocateWithStackalloc() { var array = stackalloc byte[10000]; Consume(array); } [MethodImpl(MethodImplOptions.NoInlining)] private static unsafe void Consume(byte* input) { } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroInliningDiagnoser.cs ================================================ using BenchmarkDotNet.Attributes; using System.Runtime.CompilerServices; namespace BenchmarkDotNet.Samples { [Diagnostics.Windows.Configs.InliningDiagnoser(logFailuresOnly: false, allowedNamespaces: ["BenchmarkDotNet.Samples"])] public class IntroInliningDiagnoser { [Benchmark] public int IterationTest() { int j = 0; for (int i = 0; i < short.MaxValue; ++i) { j = i + AddThree(i); } return j + ReturnFive() + AddThree(ReturnFive()); } [Benchmark] public int SplitJoin() => string.Join(",", new string[1000]).Split(',').Length; private int ReturnFive() { return 5; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddThree(int a) { return a + 3; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroJitStatsDiagnoser.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [Diagnostics.Windows.Configs.JitStatsDiagnoser] public class IntroJitStatsDiagnoser { [Benchmark] public void Sleep() => Thread.Sleep(10); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroJobBaseline.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { [SimpleJob(runtimeMoniker: RuntimeMoniker.Net462, baseline: true)] [SimpleJob(runtimeMoniker: RuntimeMoniker.Mono)] [SimpleJob(runtimeMoniker: RuntimeMoniker.Net50)] public class IntroJobBaseline { [Benchmark] public int SplitJoin() => string.Join(",", new string[1000]).Split(',').Length; } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroJoin.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { // Run BenchmarkSwitcher with arguments: "--join --category=IntroJoinA" [DryJob] public class IntroJoin1 { [Benchmark] [BenchmarkCategory("IntroJoinA")] public void A() => Thread.Sleep(10); [Benchmark] [BenchmarkCategory("IntroJoinB")] public void B() => Thread.Sleep(10); } [DryJob] public class IntroJoin2 { [Benchmark] [BenchmarkCategory("IntroJoinA")] public void A() => Thread.Sleep(10); [Benchmark] [BenchmarkCategory("IntroJoinB")] public void B() => Thread.Sleep(10); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs ================================================ using System; using System.Runtime.InteropServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { [MemoryDiagnoser] [Config(typeof(Config))] public class IntroLargeAddressAware { private class Config : ManualConfig { public Config() { AddJob(Job.Default .WithRuntime(ClrRuntime.Net462) .WithPlatform(Platform.X86) .WithLargeAddressAware(value: RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) .WithId("Framework")); } } [Benchmark] public void AllocateMoreThan2GB() { const int oneGB = 1024 * 1024 * 1024; const int halfGB = oneGB / 2; byte[] bytes1 = new byte[oneGB]; byte[] bytes2 = new byte[oneGB]; byte[] bytes3 = new byte[halfGB]; GC.KeepAlive(bytes1); GC.KeepAlive(bytes2); GC.KeepAlive(bytes3); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs ================================================ using BenchmarkDotNet.Attributes; using System; namespace BenchmarkDotNet.Samples { public class IntroMemoryRandomization { [Params(512 * 4)] public int Size; private int[] array = default!; private int[] destination = default!; [GlobalSetup] public void Setup() { array = new int[Size]; destination = new int[Size]; } [Benchmark] [MemoryRandomization(false)] public void Array_RandomizationDisabled() => Array.Copy(array, destination, Size); [Benchmark] [MemoryRandomization(true)] [MaxIterationCount(40)] // the benchmark becomes multimodal and need a lower limit of max iterations than the default public void Array_RandomizationEnabled() => Array.Copy(array, destination, Size); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroMonitoring.cs ================================================ using System; using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; namespace BenchmarkDotNet.Samples { [SimpleJob(RunStrategy.Monitoring, iterationCount: 10, id: "MonitoringJob")] [MinColumn, Q1Column, Q3Column, MaxColumn] public class IntroMonitoring { private Random random = new Random(42); [Benchmark] public void Foo() { Thread.Sleep(random.Next(10) * 10); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroMultimodal.cs ================================================ using System; using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; namespace BenchmarkDotNet.Samples { [MValueColumn] [SimpleJob(RunStrategy.Throughput, 1, 0, -1, 1, "MyJob")] public class IntroMultimodal { private readonly Random rnd = new Random(42); private void Multimodal(int n) => Thread.Sleep((rnd.Next(n) + 1) * 100); [Benchmark] public void Unimodal() => Multimodal(1); [Benchmark] public void Bimodal() => Multimodal(2); [Benchmark] public void Trimodal() => Multimodal(3); [Benchmark] public void Quadrimodal() => Multimodal(4); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs ================================================ using System; using System.Drawing; using System.Runtime.InteropServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnostics.Windows.Configs; using BenchmarkDotNet.Filters; namespace BenchmarkDotNet.Samples { [ShortRunJob] [NativeMemoryProfiler] [MemoryDiagnoser] public class IntroNativeMemory { #pragma warning disable CA1416 [Benchmark, WindowsOnly] public void BitmapWithLeaks() { var flag = new Bitmap(200, 100); var graphics = Graphics.FromImage(flag); var blackPen = new Pen(Color.Black, 3); graphics.DrawLine(blackPen, 100, 100, 500, 100); } [Benchmark, WindowsOnly] public void Bitmap() { using (var flag = new Bitmap(200, 100)) { using (var graphics = Graphics.FromImage(flag)) { using (var blackPen = new Pen(Color.Black, 3)) { graphics.DrawLine(blackPen, 100, 100, 500, 100); } } } } #pragma warning restore CA1416 private const int Size = 20; // Greater value could cause System.OutOfMemoryException for test with memory leaks. private int ArraySize = Size * Marshal.SizeOf(); [Benchmark] public unsafe void AllocHGlobal() { IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); Marshal.FreeHGlobal(unmanagedHandle); } [Benchmark] public unsafe void AllocHGlobalWithLeaks() { IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); } private class WindowsOnlyAttribute : FilterConfigBaseAttribute { public WindowsOnlyAttribute() : base(new SimpleFilter(_ => RuntimeInformation.IsOSPlatform(OSPlatform.Windows))) { } } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroNuGet.cs ================================================ using System; using System.IO.Hashing; using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { /// /// Benchmarks between various versions of a NuGet package /// /// /// Only supported with CsProj toolchains. /// [Config(typeof(Config))] public class IntroNuGet { // Setup your csproj like this: /* 8.0.0 */ // All versions of the package must be source-compatible with your benchmark code. private class Config : ManualConfig { public Config() { string[] targetVersions = [ "8.0.0", "9.0.0", "10.0.0", ]; foreach (var version in targetVersions) { AddJob(Job.MediumRun .WithMsBuildArguments($"/p:SihVersion={version}") .WithId($"v{version}") ); } } } private static readonly byte[] values; static IntroNuGet() { var rand = new Random(Seed: 0); values = new byte[10_000]; rand.NextBytes(values); } [Benchmark] public void XxHash3Benchmark() { var results = XxHash3.Hash(values); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroOrderAttr.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Order; namespace BenchmarkDotNet.Samples { [Orderer(SummaryOrderPolicy.FastestToSlowest, MethodOrderPolicy.Declared, jobOrderPolicy: JobOrderPolicy.Numeric)] [DryJob] public class IntroOrderAttr { [Params(1, 2, 3)] public int X { get; set; } [Benchmark] public void Slow() => Thread.Sleep(X * 100); [Benchmark] public void Fast() => Thread.Sleep(X * 50); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroOrderManual.cs ================================================ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Order; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] [DryJob] [RankColumn] public class IntroOrderManual { private class Config : ManualConfig { public Config() => Orderer = new FastestToSlowestOrderer(); private class FastestToSlowestOrderer : IOrderer { public IEnumerable GetExecutionOrder(ImmutableArray benchmarksCase, IEnumerable? order = null) => from benchmark in benchmarksCase orderby benchmark.Parameters["X"] descending, benchmark.Descriptor.WorkloadMethodDisplayInfo select benchmark; public IEnumerable GetSummaryOrder(ImmutableArray benchmarksCase, Summary summary) => from benchmark in benchmarksCase orderby summary[benchmark]?.ResultStatistics?.Mean select benchmark; public string? GetHighlightGroupKey(BenchmarkCase benchmarkCase) => null; public string GetLogicalGroupKey(ImmutableArray allBenchmarksCases, BenchmarkCase benchmarkCase) => benchmarkCase.Job.DisplayInfo + "_" + benchmarkCase.Parameters.DisplayInfo; public IEnumerable> GetLogicalGroupOrder(IEnumerable> logicalGroups, IEnumerable? order = null) => logicalGroups.OrderBy(it => it.Key); public bool SeparateLogicalGroups => true; } } [Params(1, 2, 3)] public int X { get; set; } [Benchmark] public void Fast() => Thread.Sleep(X * 50); [Benchmark] public void Slow() => Thread.Sleep(X * 100); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroOutliers.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; using Perfolizer.Mathematics.OutlierDetection; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] public class IntroOutliers { private class Config : ManualConfig { public Config() { var jobBase = Job.Default.WithWarmupCount(0).WithIterationCount(10).WithInvocationCount(1).WithUnrollFactor(1); AddJob(jobBase.WithOutlierMode(OutlierMode.DontRemove).WithId("DontRemoveOutliers")); AddJob(jobBase.WithOutlierMode(OutlierMode.RemoveUpper).WithId("RemoveUpperOutliers")); } } private int counter; [Benchmark] public void Foo() { counter++; int noise = counter % 10 == 0 ? 500 : 0; Thread.Sleep(100 + noise); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroParams.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroParams { [Params(100, 200)] public int A { get; set; } [Params(10, 20)] public int B { get; set; } [Benchmark] public void Benchmark() => Thread.Sleep(A + B + 5); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroParamsAllValues.cs ================================================ using BenchmarkDotNet.Attributes; using System.Threading; namespace BenchmarkDotNet.Samples { [DryJob] public class IntroParamsAllValues { public enum CustomEnum { One = 1, Two, Three } [ParamsAllValues] public CustomEnum E { get; set; } [ParamsAllValues] public bool? B { get; set; } [Benchmark] public void Benchmark() { Thread.Sleep( (int)E * 100 + (B == true ? 20 : B == false ? 10 : 0)); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroParamsPriority.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroParamsPriority { [Params(100)] public int A { get; set; } [Params(10, Priority = -100)] public int B { get; set; } [Benchmark] public void Benchmark() => Thread.Sleep(A + B + 5); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroParamsSource.cs ================================================ using System.Collections.Generic; using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroParamsSource { // property with public setter [ParamsSource(nameof(ValuesForA))] public int A { get; set; } // public field [ParamsSource(nameof(ValuesForB))] public int B; // public property public IEnumerable ValuesForA => [100, 200]; // public static method public static IEnumerable ValuesForB() => [10, 20]; // public field getting its params from a method in another type [ParamsSource(typeof(ParamsValues), nameof(ParamsValues.ValuesForC))] public int C; [Benchmark] public void Benchmark() => Thread.Sleep(A + B + C + 5); } public static class ParamsValues { public static IEnumerable ValuesForC() => [1000, 2000]; } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroPercentiles.cs ================================================ using System; using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; namespace BenchmarkDotNet.Samples { // Using percentiles for adequate timings representation [Config(typeof(Config))] [SimpleJob(RunStrategy.ColdStart, launchCount: 4, warmupCount: 3, iterationCount: 20, id: "MyJob")] public class IntroPercentiles { // To share between runs. // DO NOT do this in production code. The System.Random IS NOT thread safe. private static readonly Random Rnd = new Random(); private class Config : ManualConfig { public Config() { AddColumn( StatisticColumn.P0, StatisticColumn.P25, StatisticColumn.P50, StatisticColumn.P67, StatisticColumn.P80, StatisticColumn.P85, StatisticColumn.P90, StatisticColumn.P95, StatisticColumn.P100); } } [Benchmark(Baseline = true)] public void ConstantDelays() => Thread.Sleep(20); [Benchmark] public void RandomDelays() => Thread.Sleep(10 + (int) (20 * Rnd.NextDouble())); [Benchmark] public void RareDelays() { int rndTime = 10; // Bigger delays for 15% of the runs if (Rnd.NextDouble() > 0.85) { rndTime += 30; } Thread.Sleep(rndTime); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroPerfCollectProfiler.cs ================================================ using System.IO; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [PerfCollectProfiler(performExtraBenchmarksRun: false)] public class IntroPerfCollectProfiler { private readonly string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); private readonly string content = new string('a', 100_000); [Benchmark] public void WriteAllText() => File.WriteAllText(path, content); [GlobalCleanup] public void Delete() => File.Delete(path); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroPowerPlan.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using System; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] public class IntroPowerPlan { private class Config : ManualConfig { public Config() { AddJob(Job.MediumRun.WithPowerPlan(new Guid("e9a42b02-d5df-448d-aa00-03f14749eb61"))); AddJob(Job.MediumRun.WithPowerPlan(PowerPlan.UltimatePerformance)); AddJob(Job.MediumRun.WithPowerPlan(PowerPlan.UserPowerPlan)); AddJob(Job.MediumRun.WithPowerPlan(PowerPlan.HighPerformance)); AddJob(Job.MediumRun.WithPowerPlan(PowerPlan.Balanced)); AddJob(Job.MediumRun.WithPowerPlan(PowerPlan.PowerSaver)); } } [Benchmark] public int IterationTest() { int j = 0; for (int i = 0; i < short.MaxValue; ++i) { j = i; } return j; } [Benchmark] public int SplitJoin() => string.Join(",", new string[1000]).Split(',').Length; } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroRankColumn.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Order; namespace BenchmarkDotNet.Samples { [ShortRunJob] [Orderer(SummaryOrderPolicy.FastestToSlowest)] [RankColumn(NumeralSystem.Arabic)] [RankColumn(NumeralSystem.Roman)] [RankColumn(NumeralSystem.Stars)] public class IntroRankColumn { [Params(1, 2)] public int Factor; [Benchmark] public void Foo() => Thread.Sleep(Factor * 100); [Benchmark] public void Bar() => Thread.Sleep(Factor * 200); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroRatioSD.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; using Perfolizer.Mathematics.OutlierDetection; namespace BenchmarkDotNet.Samples { // Don't remove outliers [Outliers(OutlierMode.DontRemove)] // Skip jitting, pilot, warmup; measure 10 iterations [SimpleJob(RunStrategy.Monitoring, iterationCount: 10, invocationCount: 1)] public class IntroRatioSD { private int counter; [GlobalSetup] public void Setup() => counter = 0; [Benchmark(Baseline = true)] public void Base() { Thread.Sleep(100); if (++counter % 7 == 0) Thread.Sleep(5000); // Emulate outlier } [Benchmark] public void Slow() => Thread.Sleep(200); [Benchmark] public void Fast() => Thread.Sleep(50); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroRatioStyle.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Samples { [ShortRunJob, Config(typeof(Config))] public class IntroRatioStyle { [Benchmark(Baseline = true)] public void Baseline() => Thread.Sleep(1000); [Benchmark] public void Bar() => Thread.Sleep(150); [Benchmark] public void Foo() => Thread.Sleep(1150); private class Config : ManualConfig { public Config() { SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); } } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroSetupCleanupGlobal.cs ================================================ using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroSetupCleanupGlobal { [Params(10, 100, 1000)] public int N; private int[] data = default!; [GlobalSetup] public void GlobalSetup() { data = new int[N]; // executed once per each N value } [Benchmark] public int Logic() { int res = 0; for (int i = 0; i < N; i++) res += data[i]; return res; } [GlobalCleanup] public void GlobalCleanup() { // Disposing logic } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroSetupCleanupIteration.cs ================================================ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; namespace BenchmarkDotNet.Samples { [SimpleJob(RunStrategy.Monitoring, launchCount: 1, warmupCount: 2, iterationCount: 3)] public class IntroSetupCleanupIteration { private int setupCounter; private int cleanupCounter; [IterationSetup] public void IterationSetup() => Console.WriteLine($"// IterationSetup ({++setupCounter})"); [IterationCleanup] public void IterationCleanup() => Console.WriteLine($"// IterationCleanup ({++cleanupCounter})"); [GlobalSetup] public void GlobalSetup() => Console.WriteLine("// " + "GlobalSetup"); [GlobalCleanup] public void GlobalCleanup() => Console.WriteLine("// " + "GlobalCleanup"); [Benchmark] public void Benchmark() => Console.WriteLine("// " + "Benchmark"); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroSetupCleanupTarget.cs ================================================ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; namespace BenchmarkDotNet.Samples { [SimpleJob(RunStrategy.Monitoring, launchCount: 0, warmupCount: 0, iterationCount: 1)] public class IntroSetupCleanupTarget { [GlobalSetup(Target = nameof(BenchmarkA))] public void GlobalSetupA() => Console.WriteLine("// " + "GlobalSetup A"); [Benchmark] public void BenchmarkA() => Console.WriteLine("// " + "Benchmark A"); [GlobalSetup(Targets = new[] { nameof(BenchmarkB), nameof(BenchmarkC) })] public void GlobalSetupB() => Console.WriteLine("// " + "GlobalSetup B"); [Benchmark] public void BenchmarkB() => Console.WriteLine("// " + "Benchmark B"); [Benchmark] public void BenchmarkC() => Console.WriteLine("// " + "Benchmark C"); [Benchmark] public void BenchmarkD() => Console.WriteLine("// " + "Benchmark D"); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs ================================================ using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples; [DisassemblyDiagnoser] public class IntroSmokeEmptyBasic { [Benchmark] public void Void1() {} [Benchmark] public void Void2() {} [Benchmark] public void Void3() {} [Benchmark] public void Void4() {} [Benchmark] public byte Byte1() => 0; [Benchmark] public byte Byte2() => 0; [Benchmark] public byte Byte3() => 0; [Benchmark] public byte Byte4() => 0; [Benchmark] public sbyte Sbyte1() => 0; [Benchmark] public sbyte Sbyte2() => 0; [Benchmark] public sbyte Sbyte3() => 0; [Benchmark] public sbyte Sbyte4() => 0; [Benchmark] public short Short1() => 0; [Benchmark] public short Short2() => 0; [Benchmark] public short Short3() => 0; [Benchmark] public short Short4() => 0; [Benchmark] public ushort Ushort1() => 0; [Benchmark] public ushort Ushort2() => 0; [Benchmark] public ushort Ushort3() => 0; [Benchmark] public ushort Ushort4() => 0; [Benchmark] public int Int1() => 0; [Benchmark] public int Int2() => 0; [Benchmark] public int Int3() => 0; [Benchmark] public int Int4() => 0; [Benchmark] public uint Uint1() => 0u; [Benchmark] public uint Uint2() => 0u; [Benchmark] public uint Uint3() => 0u; [Benchmark] public uint Uint4() => 0u; [Benchmark] public bool Bool1() => false; [Benchmark] public bool Bool2() => false; [Benchmark] public bool Bool3() => false; [Benchmark] public bool Bool4() => false; [Benchmark] public char Char1() => 'a'; [Benchmark] public char Char2() => 'a'; [Benchmark] public char Char3() => 'a'; [Benchmark] public char Char4() => 'a'; [Benchmark] public float Float1() => 0f; [Benchmark] public float Float2() => 0f; [Benchmark] public float Float3() => 0f; [Benchmark] public float Float4() => 0f; [Benchmark] public double Double1() => 0d; [Benchmark] public double Double2() => 0d; [Benchmark] public double Double3() => 0d; [Benchmark] public double Double4() => 0d; [Benchmark] public long Long1() => 0L; [Benchmark] public long Long2() => 0L; [Benchmark] public long Long3() => 0L; [Benchmark] public long Long4() => 0L; [Benchmark] public ulong Ulong1() => 0uL; [Benchmark] public ulong Ulong2() => 0uL; [Benchmark] public ulong Ulong3() => 0uL; [Benchmark] public ulong Ulong4() => 0uL; [Benchmark] public string String1() => ""; [Benchmark] public string String2() => ""; [Benchmark] public string String3() => ""; [Benchmark] public string String4() => ""; [Benchmark] public object? Object1() => null; [Benchmark] public object? Object2() => null; [Benchmark] public object? Object3() => null; [Benchmark] public object? Object4() => null; } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs ================================================ using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples; public class IntroSmokeIncrements { public int Field; [Benchmark] public void Increment01() { Field++; } [Benchmark] public void Increment02() { Field++; Field++; } [Benchmark] public void Increment03() { Field++; Field++; Field++; } [Benchmark] public void Increment04() { Field++; Field++; Field++; Field++; } [Benchmark] public void Increment05() { Field++; Field++; Field++; Field++; Field++; } [Benchmark] public void Increment06() { Field++; Field++; Field++; Field++; Field++; Field++; } [Benchmark] public void Increment07() { Field++; Field++; Field++; Field++; Field++; Field++; Field++; } [Benchmark] public void Increment08() { Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; } [Benchmark] public void Increment09() { Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; } [Benchmark] public void Increment10() { Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; } [Benchmark] public void Increment20() { Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; Field++; } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroSmokeStringBuilder.cs ================================================ using BenchmarkDotNet.Attributes; using System.Text; namespace BenchmarkDotNet.Samples { [MemoryDiagnoser(false)] public class IntroSmokeStringBuilder { [Benchmark] [Arguments(1)] [Arguments(1_000)] public StringBuilder Append_Strings(int repeat) { StringBuilder builder = new StringBuilder(); // strings are not sorted by length to mimic real input for (int i = 0; i < repeat; i++) { builder.Append("12345"); builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN"); builder.Append("1234567890abcdefghijklmnopqrstuvwxy"); builder.Append("1234567890"); builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHI"); builder.Append("1234567890abcde"); builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCD"); builder.Append("1234567890abcdefghijklmnopqrst"); builder.Append("1234567890abcdefghij"); builder.Append("1234567890abcdefghijklmno"); } return builder; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs ================================================ using System; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Environments; namespace BenchmarkDotNet.Samples; [MemoryDiagnoser, DisassemblyDiagnoser] public class IntroSmokeValueTypes { [Benchmark] public Jit ReturnEnum() => Jit.RyuJit; [Benchmark] public DateTime ReturnDateTime() => new DateTime(); [Benchmark] public DateTime? ReturnNullableDateTime() => new DateTime(); [Benchmark] public int? ReturnNullableInt() => 0; public struct StructWithReferencesOnly { public object _ref; } [Benchmark] public StructWithReferencesOnly ReturnStructWithReferencesOnly() => new StructWithReferencesOnly(); public struct EmptyStruct { } [Benchmark] public EmptyStruct ReturnEmptyStruct() => new EmptyStruct(); [Benchmark] public ValueTuple ReturnGenericStructOfValueType() => new ValueTuple(0); [Benchmark] public ValueTuple ReturnGenericStructOfReferenceType() => new ValueTuple(null); [Benchmark] public ValueTask ReturnValueTaskOfValueType() => new ValueTask(0); [Benchmark] public ValueTask ReturnValueTaskOfReferenceType() => new ValueTask(result: null); [Benchmark] public byte ReturnByte() => 0; public struct Byte1 { public byte _1; } [Benchmark] public Byte1 ReturnByte1() => new Byte1(); public struct Byte2 { public byte _1, _2; } [Benchmark] public Byte2 ReturnByte2() => new Byte2(); public struct Byte3 { public byte _1, _2, _3; } [Benchmark] public Byte3 ReturnByte3() => new Byte3(); public struct Byte4 { public byte _1, _2, _3, _4; } [Benchmark] public Byte4 ReturnByte4() => new Byte4(); [Benchmark] public short ReturnShort() => 0; public struct Short1 { public short _1; } [Benchmark] public Short1 ReturnShort1() => new Short1(); public struct Short2 { public short _1, _2; } [Benchmark] public Short2 ReturnShort2() => new Short2(); public struct Short3 { public short _1, _2, _3; } [Benchmark] public Short3 ReturnShort3() => new Short3(); public struct Short4 { public short _1, _2, _3, _4; } [Benchmark] public Short4 ReturnShort4() => new Short4(); [Benchmark] public int ReturnInt() => 0; public struct Int1 { public int _1; } [Benchmark] public Int1 ReturnInt1() => new Int1(); public struct Int2 { public int _1, _2; } [Benchmark] public Int2 ReturnInt2() => new Int2(); public struct Int3 { public int _1, _2, _3; } [Benchmark] public Int3 ReturnInt3() => new Int3(); public struct Int4 { public int _1, _2, _3, _4; } [Benchmark] public Int4 ReturnInt4() => new Int4(); [Benchmark] public long ReturnLong() => 0; public struct Long1 { public long _1; } [Benchmark] public Long1 ReturnLong1() => new Long1(); public struct Long2 { public long _1, _2; } [Benchmark] public Long2 ReturnLong2() => new Long2(); public struct Long3 { public long _1, _2, _3; } [Benchmark] public Long3 ReturnLong3() => new Long3(); public struct Long4 { public long _1, _2, _3, _4; } [Benchmark] public Long4 ReturnLong4() => new Long4(); } // ReSharper restore InconsistentNaming ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroStaThread.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { public class IntroStaThread { [Benchmark, System.STAThread] public void CheckForSTA() { if (Thread.CurrentThread.GetApartmentState() != ApartmentState.STA) { throw new ThreadStateException( "The current threads apartment state is not STA"); } } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [StatisticalTestColumn("500us")] [StatisticalTestColumn("3%")] [SimpleJob(warmupCount: 0, iterationCount: 5)] public class IntroStatisticalTesting { [Benchmark] public void Sleep50() => Thread.Sleep(50); [Benchmark] public void Sleep97() => Thread.Sleep(97); [Benchmark] public void Sleep99() => Thread.Sleep(99); [Benchmark(Baseline = true)] public void Sleep100() => Thread.Sleep(100); [Benchmark] public void Sleep101() => Thread.Sleep(101); [Benchmark] public void Sleep103() => Thread.Sleep(103); [Benchmark] public void Sleep150() => Thread.Sleep(150); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroStatisticsColumns.cs ================================================ using System; using System.Security.Cryptography; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [MediumRunJob, SkewnessColumn, KurtosisColumn] public class IntroStatisticsColumns { private const int N = 10000; private readonly byte[] data; private readonly MD5 md5 = MD5.Create(); private readonly SHA256 sha256 = SHA256.Create(); public IntroStatisticsColumns() { data = new byte[N]; new Random(42).NextBytes(data); } [Benchmark(Baseline = true)] public byte[] Md5A() => md5.ComputeHash(data); [Benchmark] public byte[] Md5B() => md5.ComputeHash(data); [Benchmark] public byte[] Sha256() => sha256.ComputeHash(data); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroStopOnFirstError.cs ================================================ using System; using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [StopOnFirstError] public class IntroStopOnFirstError { [Benchmark(Baseline = true)] public int FirstMethod() => throw new Exception("Example exception."); [Benchmark] public int SecondMethod() => 1; } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs ================================================ using System.Globalization; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Reports; using Perfolizer.Horology; using Perfolizer.Metrology; namespace BenchmarkDotNet.Samples { [Config(typeof(Config))] public class IntroSummaryStyle { private class Config : ManualConfig { public Config() { // Configure the summary style here var summaryStyle = new SummaryStyle ( cultureInfo: CultureInfo.InvariantCulture, printUnitsInHeader: true, printUnitsInContent: false, sizeUnit: SizeUnit.KB, timeUnit: TimeUnit.Nanosecond, maxParameterColumnWidth: 20 ); WithSummaryStyle(summaryStyle); } } [Params(10, 100)] public int N; [Benchmark] public void Sleep() => System.Threading.Thread.Sleep(N); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroTagColumn.cs ================================================ using System.Threading; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Samples { // You can add custom tags per each method using Columns [Config(typeof(Config))] public class IntroTagColumn { private class Config : ManualConfig { public Config() { AddJob(Job.Dry); AddColumn(new TagColumn("Kind", name => name.Substring(0, 3))); AddColumn(new TagColumn("Number", name => name.Substring(3))); } } [Benchmark] public void Foo1() => Thread.Sleep(10); [Benchmark] public void Foo12() => Thread.Sleep(10); [Benchmark] public void Bar3() => Thread.Sleep(10); [Benchmark] public void Bar34() => Thread.Sleep(10); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroTailcall.cs ================================================ using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Samples { [Diagnostics.Windows.Configs.TailCallDiagnoser] [LegacyJitX86Job, LegacyJitX64Job, RyuJitX64Job] public class IntroTailcall { [Benchmark] public long Calc() => FactorialWithoutTailing(7) - FactorialWithTailing(7); private static long FactorialWithoutTailing(int depth) => depth == 0 ? 1 : depth * FactorialWithoutTailing(depth - 1); private static long FactorialWithTailing(int pos, int depth) => pos == 0 ? depth : FactorialWithTailing(pos - 1, depth * pos); private static long FactorialWithTailing(int depth) => FactorialWithTailing(depth - 1, depth); } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroThreadingDiagnoser.cs ================================================ using BenchmarkDotNet.Attributes; using System.Threading; namespace BenchmarkDotNet.Samples { [ThreadingDiagnoser] // ENABLE the diagnoser public class IntroThreadingDiagnoser { [Benchmark] public void CompleteOneWorkItem() { ManualResetEvent done = new ManualResetEvent(initialState: false); ThreadPool.QueueUserWorkItem(m => (m as ManualResetEvent)?.Set(), done); done.WaitOne(); } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroUnicode.cs ================================================ using System.Diagnostics; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Samples { // *** Attribute Style *** [UnicodeConsoleLogger] public class IntroUnicode { [Benchmark] public long Foo() { long waitUntil = Stopwatch.GetTimestamp() + 1000; while (Stopwatch.GetTimestamp() < waitUntil) { } return waitUntil; } } // *** Object Style *** [Config(typeof(Config))] public class IntroUnicodeObjectStyle { private class Config : ManualConfig { public Config() => AddLogger(ConsoleLogger.Unicode); } [Benchmark] public long Foo() { long waitUntil = Stopwatch.GetTimestamp() + 1000; while (Stopwatch.GetTimestamp() < waitUntil) { } return waitUntil; } } // *** Fluent Config *** public class IntroUnicodeFluentConfig { public static void Run() { BenchmarkRunner.Run( DefaultConfig.Instance .AddLogger(ConsoleLogger.Unicode)); } [Benchmark] public long Foo() { long waitUntil = Stopwatch.GetTimestamp() + 1000; while (Stopwatch.GetTimestamp() < waitUntil) { } return waitUntil; } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs ================================================ using System; using BenchmarkDotNet.Attributes; using Microsoft.VSDiagnostics; namespace BenchmarkDotNet.Samples { // Enables profiling with the CPU Usage tool // See: https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet [CPUUsageDiagnoser] public class IntroVisualStudioProfiler { private readonly Random rand = new Random(42); [Benchmark] public void BurnCPU() { for (int i = 0; i < 100000; ++i) { rand.Next(1, 100); } } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroWakeLock.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using System; using System.Threading; // *** Attribute Style applied to Assembly *** [assembly: WakeLock(WakeLockType.System)] namespace BenchmarkDotNet.Samples; // *** Attribute Style *** [WakeLock(WakeLockType.Display)] public class IntroWakeLock { [Benchmark] public void LongRunning() => Thread.Sleep(TimeSpan.FromSeconds(10)); } // *** Object Style *** [Config(typeof(Config))] public class IntroWakeLockObjectStyle { private class Config : ManualConfig { public Config() => WakeLock = WakeLockType.System; } [Benchmark] public void LongRunning() => Thread.Sleep(TimeSpan.FromSeconds(10)); } ================================================ FILE: samples/BenchmarkDotNet.Samples/IntroWasm.cs ================================================ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.MonoWasm; namespace BenchmarkDotNet.Samples { // *** Command Line Arguments *** public class IntroWasmCmdConfig { // Example: // --runtimes wasmnet8.0 // --cli /path/to/dotnet (optional) // --wasmEngine v8 (optional) // --wasmArgs "--expose_wasm" (optional) // --wasmDataDir /path/to/data (optional) public static void Run(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(IntroWasmCmdConfig).Assembly).Run(args); [Benchmark] public void Foo() { // Benchmark body } } // *** Fluent Config *** public class IntroWasmFluentConfig { public static void Run() { // Optional: set this to use a custom `dotnet` (for example, a local dotnet/runtime build). const string cliPath = ""; WasmRuntime runtime = new WasmRuntime(msBuildMoniker: "net8.0", RuntimeMoniker.WasmNet80, "Wasm .net8.0", false, "v8"); NetCoreAppSettings netCoreAppSettings = new NetCoreAppSettings( targetFrameworkMoniker: "net8.0", runtimeFrameworkVersion: "", name: "Wasm", customDotNetCliPath: cliPath); var toolChain = WasmToolchain.From(netCoreAppSettings); BenchmarkRunner.Run(DefaultConfig.Instance .AddJob(Job.ShortRun.WithRuntime(runtime).WithToolchain(toolChain))); } [Benchmark] public void Foo() { // Benchmark body } } } ================================================ FILE: samples/BenchmarkDotNet.Samples/Program.cs ================================================ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using System; using System.Linq; namespace BenchmarkDotNet.Samples; public class Program { public static int Main(string[] args) { #if DEBUG ConsoleLogger.Default.WriteLineWarning("Benchmark is executed with DEBUG configuration."); ConsoleLogger.Default.WriteLine(); #endif if (args.Length != 0) { ConsoleLogger.Default.WriteLine($"Start benchmarks with args: {string.Join(" ", args)}"); ConsoleLogger.Default.WriteLine(); } IConfig? config = GetConfig(ref args); var summaries = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly) .Run(args, config) .ToArray(); if (summaries.HasError()) return 1; return 0; } private static IConfig? GetConfig(ref string[] args) { #if !DEBUG return null; // `DefaultConfig.Instance` is used. #else bool isInProcess = args.Contains("--inProcess"); if (isInProcess) args = args.Where(x => x != "--inProcess").ToArray(); DebugConfig config = isInProcess ? new DebugInProcessConfig() : new DebugBuildConfig(); return config.AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) .AddExporter(MarkdownExporter.Default) .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) .WithArtifactsPath(DefaultConfig.Instance.ArtifactsPath!); #endif } } ================================================ FILE: samples/BenchmarkDotNet.Samples/Properties/AssemblyInfo.cs ================================================ using System.Runtime.InteropServices; [assembly: Guid("6f2232a9-0d0c-46cf-b08c-f6e28ab612e3")] ================================================ FILE: samples/BenchmarkDotNet.Samples/Properties/launchSettings.json ================================================ { "profiles": { "Default": { "commandName": "Project", "commandLineArgs": "", "environmentVariables": { } }, "Run IntroBasic Benchmarks": { "commandName": "Project", "commandLineArgs": "--filter *IntroBasic*", "environmentVariables": { } }, "Run IntroBasic Benchmarks with --inProcess": { "commandName": "Project", "commandLineArgs": "--filter *IntroBasic* --inProcess", "environmentVariables": { } }, "--list": { "commandName": "Project", "commandLineArgs": "--list" }, "--info": { "commandName": "Project", "commandLineArgs": "--info" }, "--help": { "commandName": "Project", "commandLineArgs": "--help" }, "--version": { "commandName": "Project", "commandLineArgs": "--version" } } } ================================================ FILE: samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj ================================================ Exe net462;net8.0 false false ================================================ FILE: samples/BenchmarkDotNet.Samples.FSharp/Program.fs ================================================ module Program open System open System.IO open System.Collections.Concurrent open BenchmarkDotNet.Attributes open BenchmarkDotNet.Running let getStrings len = Array.init len (fun _ -> Path.GetRandomFileName()) let lookup arr (dict:ConcurrentDictionary) = arr |> Array.iteri(fun idx elm -> let mutable b = dict.[elm] b <- dict.[arr.[0]]) type StringKeyComparison () = let mutable arr : string [] = [||] let dict1 = ConcurrentDictionary<_,_>() let dict2 = ConcurrentDictionary<_,_>(StringComparer.Ordinal) [] member val public DictSize = 0 with get, set [] member self.GlobalSetupData() = dict1.Clear(); dict2.Clear() arr <- getStrings self.DictSize arr |> Array.iter (fun x -> dict1.[x] <- true ; dict2.[x] <- true) [] member self.StandardLookup () = lookup arr dict1 [] member self.OrdinalLookup () = lookup arr dict2 #if NETFRAMEWORK [] #endif type TailCallDetector () = let rec factorial n = match n with | 0 | 1 -> 1 | _ -> n * factorial(n-1) let factorial1 n = let rec loop i acc = match i with | 0 | 1 -> acc | _ -> loop (i-1) (acc * i) loop n 1 let factorial2 n = let rec tailCall n f = if n <= 1 then f() else tailCall (n - 1) (fun () -> n * f()) tailCall n (fun () -> 1) [] member val public facRank = 0 with get, set [] member self.test () = factorial self.facRank [] member self.test1 () = factorial1 self.facRank [] member self.test2 () = factorial2 self.facRank let defaultSwitch () = BenchmarkSwitcher [|typeof; typeof|] [] let Main args = let summary = defaultSwitch().Run args 0 ================================================ FILE: samples/BenchmarkDotNet.Samples.Maui/App.xaml ================================================  ================================================ FILE: samples/BenchmarkDotNet.Samples.Maui/App.xaml.cs ================================================ using Microsoft.Extensions.DependencyInjection; namespace BenchmarkDotNet.Samples.Maui; public partial class App : Application { public App() { InitializeComponent(); } protected override Window CreateWindow(IActivationState? activationState) { return new Window(new AppShell()); } } ================================================ FILE: samples/BenchmarkDotNet.Samples.Maui/AppShell.xaml ================================================ ================================================ FILE: samples/BenchmarkDotNet.Samples.Maui/AppShell.xaml.cs ================================================ namespace BenchmarkDotNet.Samples.Maui; public partial class AppShell : Shell { public AppShell() { InitializeComponent(); } } ================================================ FILE: samples/BenchmarkDotNet.Samples.Maui/BenchmarkDotNet.Samples.Maui.csproj ================================================ net10.0-android;net10.0-ios;net10.0-maccatalyst $(TargetFrameworks);net10.0-windows10.0.19041.0 Exe BenchmarkDotNet.Samples.Maui true true enable enable BenchmarkDotNet Samples com.benchmarkdotnet.samples 1.0 1 None 15.0 15.0 21.0 10.0.17763.0 10.0.17763.0 false ================================================ FILE: samples/BenchmarkDotNet.Samples.Maui/MainPage.xaml ================================================