diff --git a/benchmark_cherrypick/bin/main.dart b/benchmark_cherrypick/bin/main.dart index 2d045d9..3aa79fb 100644 --- a/benchmark_cherrypick/bin/main.dart +++ b/benchmark_cherrypick/bin/main.dart @@ -1,15 +1,59 @@ import 'package:benchmark_cherrypick/benchmarks/universal_chain_benchmark.dart'; +import 'package:benchmark_cherrypick/benchmarks/universal_chain_async_benchmark.dart'; import 'package:benchmark_cherrypick/di_adapters/cherrypick_adapter.dart'; import 'package:benchmark_cherrypick/scenarios/universal_chain_module.dart'; import 'package:args/args.dart'; import 'dart:io'; import 'dart:math'; +enum UniversalBenchmark { + registerSingleton, + chainSingleton, + chainFactory, + chainAsync, + named, + override, +} + +UniversalScenario _toScenario(UniversalBenchmark b) { + switch (b) { + case UniversalBenchmark.registerSingleton: + return UniversalScenario.register; + case UniversalBenchmark.chainSingleton: + return UniversalScenario.chain; + case UniversalBenchmark.chainFactory: + return UniversalScenario.chain; + case UniversalBenchmark.chainAsync: + return UniversalScenario.asyncChain; + case UniversalBenchmark.named: + return UniversalScenario.named; + case UniversalBenchmark.override: + return UniversalScenario.override; + } +} + +UniversalBindingMode _toMode(UniversalBenchmark b) { + switch (b) { + case UniversalBenchmark.registerSingleton: + return UniversalBindingMode.singletonStrategy; + case UniversalBenchmark.chainSingleton: + return UniversalBindingMode.singletonStrategy; + case UniversalBenchmark.chainFactory: + return UniversalBindingMode.factoryStrategy; + case UniversalBenchmark.chainAsync: + return UniversalBindingMode.asyncStrategy; + case UniversalBenchmark.named: + return UniversalBindingMode.singletonStrategy; + case UniversalBenchmark.override: + return UniversalBindingMode.singletonStrategy; + } +} + Future main(List args) async { final parser = ArgParser() + ..addOption('benchmark', abbr: 'b', help: 'One of: registerSingleton, chainSingleton, chainFactory, chainAsync, named, override, all', defaultsTo: 'chainSingleton') ..addOption('chainCount', abbr: 'c', help: 'Comma-separated chainCounts', defaultsTo: '10') ..addOption('nestingDepth', abbr: 'd', help: 'Comma-separated depths', defaultsTo: '5') - ..addOption('mode', abbr: 'm', help: 'Mode (singletonStrategy,factoryStrategy,asyncStrategy)', defaultsTo: 'singletonStrategy') ..addOption('repeat', abbr: 'r', help: 'Repeats for each run (>=2)', defaultsTo: '2') ..addOption('warmup', abbr: 'w', help: 'Warmup runs', defaultsTo: '1') ..addOption('format', abbr: 'f', help: 'Output format (pretty, csv, json)', defaultsTo: 'pretty') @@ -20,88 +64,151 @@ Future main(List args) async { if (result['help'] == true) { print('UniversalChainBenchmark'); print(parser.usage); + print('Example:'); + print(' dart run bin/main.dart --benchmark=chainFactory --chainCount=10 --nestingDepth=5 --format=csv'); return; } + final benchName = result['benchmark'] as String; + final isAll = benchName == 'all'; + final List allBenches = [ + UniversalBenchmark.registerSingleton, + UniversalBenchmark.chainSingleton, + UniversalBenchmark.chainFactory, + UniversalBenchmark.chainAsync, + UniversalBenchmark.named, + UniversalBenchmark.override, + ]; + + final List benchesToRun = isAll + ? allBenches + : [ + UniversalBenchmark.values.firstWhere( + (b) => b.toString().split('.').last == benchName, + orElse: () => UniversalBenchmark.chainSingleton, + ), + ]; + final chainCounts = _parseIntList(result['chainCount'] as String); final nestDepths = _parseIntList(result['nestingDepth'] as String); - final modeName = result['mode'] as String; - final mode = UniversalBindingMode.values.firstWhere( - (m) => m.toString().split('.').last == modeName, - orElse: () => UniversalBindingMode.singletonStrategy, - ); final repeats = int.tryParse(result['repeat'] as String? ?? "") ?? 2; final warmups = int.tryParse(result['warmup'] as String? ?? "") ?? 1; final format = result['format'] as String; - final di = CherrypickDIAdapter(); final results = >[]; - void addResult( - String name, - int chainCount, - int nestingDepth, - List timings, - int? memoryDiffKb, - int? deltaPeakKb, - int? peakRssKb, - ) { - timings.sort(); - var mean = timings.reduce((a, b) => a + b) / timings.length; - var median = timings[timings.length ~/ 2]; - var minVal = timings.first; - var maxVal = timings.last; - var stddev = sqrt(timings.map((x) => pow(x - mean, 2)).reduce((a, b) => a + b) / timings.length); - results.add({ - 'benchmark': name, - 'chainCount': chainCount, - 'nestingDepth': nestingDepth, - 'mean_us': mean.round(), - 'median_us': median.round(), - 'stddev_us': stddev.round(), - 'min_us': minVal.round(), - 'max_us': maxVal.round(), - 'trials': timings.length, - 'timings_us': timings.map((t) => t.round()).toList(), - 'memory_diff_kb': memoryDiffKb, - 'delta_peak_kb': deltaPeakKb, - 'peak_rss_kb': peakRssKb, - }); - } - - Future runAndCollect( - String name, - Future Function() fn, { - required int chainCount, - required int nestingDepth, - }) async { - for (int i = 0; i < warmups; i++) { - await fn(); - } - final timings = []; - final rssValues = []; - final memBefore = ProcessInfo.currentRss; - for (int i = 0; i < repeats; i++) { - timings.add(await fn()); - rssValues.add(ProcessInfo.currentRss); - } - final memAfter = ProcessInfo.currentRss; - final memDiffKB = ((memAfter - memBefore) / 1024).round(); - final peakRss = [...rssValues, memBefore].reduce(max); - final deltaPeakKb = ((peakRss - memBefore) / 1024).round(); - addResult(name, chainCount, nestingDepth, timings, memDiffKB, deltaPeakKb, (peakRss/1024).round()); - } - - for (final c in chainCounts) { - for (final d in nestDepths) { - await runAndCollect('UniversalChain_$mode', () async { - return _captureReport(() => UniversalChainBenchmark( + for (final bench in benchesToRun) { + final scenario = _toScenario(bench); + final mode = _toMode(bench); + for (final c in chainCounts) { + for (final d in nestDepths) { + // --- asyncChain special case --- + if (scenario == UniversalScenario.asyncChain) { + final di = CherrypickDIAdapter(); + final benchAsync = UniversalChainAsyncBenchmark( + di, + chainCount: c, + nestingDepth: d, + mode: mode, + ); + final timings = []; + final rssValues = []; + // Warmup + for (int i = 0; i < warmups; i++) { + await benchAsync.setup(); + await benchAsync.run(); + await benchAsync.teardown(); + } + final memBefore = ProcessInfo.currentRss; + for (int i = 0; i < repeats; i++) { + await benchAsync.setup(); + final sw = Stopwatch()..start(); + await benchAsync.run(); + sw.stop(); + timings.add(sw.elapsedMicroseconds); + rssValues.add(ProcessInfo.currentRss); + await benchAsync.teardown(); + } + final memAfter = ProcessInfo.currentRss; + final memDiffKB = ((memAfter - memBefore) / 1024).round(); + final peakRss = [...rssValues, memBefore].reduce(max); + final deltaPeakKb = ((peakRss - memBefore) / 1024).round(); + timings.sort(); + var mean = timings.reduce((a, b) => a + b) / timings.length; + var median = timings[timings.length ~/ 2]; + var minVal = timings.first; + var maxVal = timings.last; + var stddev = sqrt(timings.map((x) => pow(x - mean, 2)).reduce((a, b) => a + b) / timings.length); + results.add({ + 'benchmark': 'Universal_$bench', + 'chainCount': c, + 'nestingDepth': d, + 'mean_us': mean.round(), + 'median_us': median.round(), + 'stddev_us': stddev.round(), + 'min_us': minVal.round(), + 'max_us': maxVal.round(), + 'trials': timings.length, + 'timings_us': timings.map((t) => t.round()).toList(), + 'memory_diff_kb': memDiffKB, + 'delta_peak_kb': deltaPeakKb, + 'peak_rss_kb': (peakRss / 1024).round(), + }); + continue; + } + // --- Sync-case --- + final di = CherrypickDIAdapter(); + final benchSync = UniversalChainBenchmark( di, chainCount: c, nestingDepth: d, mode: mode, - ).report()); - }, chainCount: c, nestingDepth: d); + scenario: scenario, + ); + final timings = []; + final rssValues = []; + // Warmup + for (int i = 0; i < warmups; i++) { + benchSync.setup(); + benchSync.run(); + benchSync.teardown(); + } + final memBefore = ProcessInfo.currentRss; + for (int i = 0; i < repeats; i++) { + benchSync.setup(); + final sw = Stopwatch()..start(); + benchSync.run(); + sw.stop(); + timings.add(sw.elapsedMicroseconds); + rssValues.add(ProcessInfo.currentRss); + benchSync.teardown(); + } + final memAfter = ProcessInfo.currentRss; + final memDiffKB = ((memAfter - memBefore) / 1024).round(); + final peakRss = [...rssValues, memBefore].reduce(max); + final deltaPeakKb = ((peakRss - memBefore) / 1024).round(); + timings.sort(); + var mean = timings.reduce((a, b) => a + b) / timings.length; + var median = timings[timings.length ~/ 2]; + var minVal = timings.first; + var maxVal = timings.last; + var stddev = sqrt(timings.map((x) => pow(x - mean, 2)).reduce((a, b) => a + b) / timings.length); + results.add({ + 'benchmark': 'Universal_$bench', + 'chainCount': c, + 'nestingDepth': d, + 'mean_us': mean.round(), + 'median_us': median.round(), + 'stddev_us': stddev.round(), + 'min_us': minVal.round(), + 'max_us': maxVal.round(), + 'trials': timings.length, + 'timings_us': timings.map((t) => t.round()).toList(), + 'memory_diff_kb': memDiffKB, + 'delta_peak_kb': deltaPeakKb, + 'peak_rss_kb': (peakRss / 1024).round(), + }); + } } } @@ -117,13 +224,6 @@ Future main(List args) async { // --- helpers --- List _parseIntList(String s) => s.split(',').map((e) => int.tryParse(e.trim()) ?? 0).where((x) => x > 0).toList(); -Future _captureReport(void Function() fn) async { - final sw = Stopwatch()..start(); - fn(); - sw.stop(); - return sw.elapsedMicroseconds; -} - String _toPretty(List> rows) { final keys = [ 'benchmark','chainCount','nestingDepth','mean_us','median_us','stddev_us', @@ -133,7 +233,6 @@ String _toPretty(List> rows) { final lines = rows.map((r) => keys.map((k) => (r[k] ?? '').toString()).join('\t')).toList(); return ([header] + lines).join('\n'); } - String _toCsv(List> rows) { final keys = [ 'benchmark','chainCount','nestingDepth','mean_us','median_us','stddev_us', @@ -149,7 +248,6 @@ String _toCsv(List> rows) { ).toList(); return ([header] + lines).join('\n'); } - String _toJson(List> rows) { return '[\n${rows.map((r) => ' $r').join(',\n')}\n]'; } diff --git a/benchmark_cherrypick/lib/benchmarks/universal_chain_async_benchmark.dart b/benchmark_cherrypick/lib/benchmarks/universal_chain_async_benchmark.dart new file mode 100644 index 0000000..524dadb --- /dev/null +++ b/benchmark_cherrypick/lib/benchmarks/universal_chain_async_benchmark.dart @@ -0,0 +1,41 @@ +import 'package:benchmark_harness/benchmark_harness.dart'; +import 'package:benchmark_cherrypick/di_adapters/di_adapter.dart'; +import 'package:benchmark_cherrypick/scenarios/universal_chain_module.dart'; +import 'package:benchmark_cherrypick/scenarios/universal_service.dart'; + +class UniversalChainAsyncBenchmark extends AsyncBenchmarkBase { + final DIAdapter di; + final int chainCount; + final int nestingDepth; + final UniversalBindingMode mode; + + UniversalChainAsyncBenchmark( + this.di, { + this.chainCount = 1, + this.nestingDepth = 3, + this.mode = UniversalBindingMode.asyncStrategy, + }) : super('UniversalAsync: asyncChain/$mode CD=$chainCount/$nestingDepth'); + + @override + Future setup() async { + di.setupModules([ + UniversalChainModule( + chainCount: chainCount, + nestingDepth: nestingDepth, + bindingMode: mode, + scenario: UniversalScenario.asyncChain, + ) + ]); + } + + @override + Future teardown() async { + di.teardown(); + } + + @override + Future run() async { + final serviceName = '${chainCount}_$nestingDepth'; + await di.resolveAsync(named: serviceName); + } +} diff --git a/benchmark_cherrypick/lib/benchmarks/universal_chain_benchmark.dart b/benchmark_cherrypick/lib/benchmarks/universal_chain_benchmark.dart index da166cf..0818608 100644 --- a/benchmark_cherrypick/lib/benchmarks/universal_chain_benchmark.dart +++ b/benchmark_cherrypick/lib/benchmarks/universal_chain_benchmark.dart @@ -4,37 +4,77 @@ import 'package:benchmark_cherrypick/scenarios/universal_chain_module.dart'; import 'package:benchmark_cherrypick/scenarios/universal_service.dart'; class UniversalChainBenchmark extends BenchmarkBase { - final DIAdapter di; + final DIAdapter _di; final int chainCount; final int nestingDepth; final UniversalBindingMode mode; + final UniversalScenario scenario; + DIAdapter? _childDi; UniversalChainBenchmark( - this.di, { - this.chainCount = 1, - this.nestingDepth = 3, - this.mode = UniversalBindingMode.singletonStrategy, - }) : super( - 'UniversalChain: $mode. C/D = $chainCount/$nestingDepth', - ); + this._di, { + this.chainCount = 1, + this.nestingDepth = 3, + this.mode = UniversalBindingMode.singletonStrategy, + this.scenario = UniversalScenario.chain, + }) : super('Universal: $scenario/$mode CD=$chainCount/$nestingDepth'); @override void setup() { - di.setupModules([ - UniversalChainModule( - chainCount: chainCount, - nestingDepth: nestingDepth, - bindingMode: mode, - ), - ]); + switch (scenario) { + case UniversalScenario.override: + _di.setupModules([ + UniversalChainModule( + chainCount: chainCount, + nestingDepth: nestingDepth, + bindingMode: UniversalBindingMode.singletonStrategy, + scenario: UniversalScenario.register, + ) + ]); + _childDi = _di.openSubScope('child'); + _childDi!.setupModules([ + UniversalChainModule( + chainCount: chainCount, + nestingDepth: nestingDepth, + bindingMode: UniversalBindingMode.singletonStrategy, + scenario: UniversalScenario.register, + ) + ]); + break; + default: + _di.setupModules([ + UniversalChainModule( + chainCount: chainCount, + nestingDepth: nestingDepth, + bindingMode: mode, + scenario: scenario, + ) + ]); + break; + } } @override - void teardown() => di.teardown(); + void teardown() => _di.teardown(); @override void run() { - final serviceName = '${chainCount}_$nestingDepth'; - di.resolve(named: serviceName); + switch (scenario) { + case UniversalScenario.register: + _di.resolve(); + break; + case UniversalScenario.named: + _di.resolve(named: 'impl2'); + break; + case UniversalScenario.chain: + final serviceName = '${chainCount}_$nestingDepth'; + _di.resolve(named: serviceName); + break; + case UniversalScenario.override: + _childDi!.resolve(); + break; + case UniversalScenario.asyncChain: + throw UnsupportedError('asyncChain supported only in UniversalChainAsyncBenchmark'); + } } } diff --git a/benchmark_cherrypick/lib/scenarios/universal_chain_module.dart b/benchmark_cherrypick/lib/scenarios/universal_chain_module.dart index 3c9d6b1..dd89644 100644 --- a/benchmark_cherrypick/lib/scenarios/universal_chain_module.dart +++ b/benchmark_cherrypick/lib/scenarios/universal_chain_module.dart @@ -7,54 +7,107 @@ enum UniversalBindingMode { asyncStrategy, } +enum UniversalScenario { + register, + chain, + named, + override, + asyncChain, +} + class UniversalChainModule extends Module { final int chainCount; final int nestingDepth; final UniversalBindingMode bindingMode; + final UniversalScenario scenario; UniversalChainModule({ required this.chainCount, required this.nestingDepth, this.bindingMode = UniversalBindingMode.singletonStrategy, + this.scenario = UniversalScenario.chain, }); @override void builder(Scope currentScope) { - for (var chainIndex = 0; chainIndex < chainCount; chainIndex++) { - for (var levelIndex = 0; levelIndex < nestingDepth; levelIndex++) { - final chain = chainIndex + 1; - final level = levelIndex + 1; - final prevDepName = '${chain}_${level - 1}'; - final depName = '${chain}_$level'; - switch (bindingMode) { - case UniversalBindingMode.singletonStrategy: - bind() - .toProvide(() => UniversalServiceImpl( - value: depName, - dependency: currentScope.tryResolve(named: prevDepName), - )) - .withName(depName) - .singleton(); - break; - case UniversalBindingMode.factoryStrategy: - bind() - .toProvide(() => UniversalServiceImpl( - value: depName, - dependency: currentScope.tryResolve(named: prevDepName), - )) - .withName(depName); - break; - case UniversalBindingMode.asyncStrategy: - bind() - .toProvideAsync(() async => UniversalServiceImpl( - value: depName, - dependency: await currentScope.resolveAsync(named: prevDepName), - )) - .withName(depName) - .singleton(); - break; + if (scenario == UniversalScenario.asyncChain) { + for (var chainIndex = 0; chainIndex < chainCount; chainIndex++) { + for (var levelIndex = 0; levelIndex < nestingDepth; levelIndex++) { + final chain = chainIndex + 1; + final level = levelIndex + 1; + final prevDepName = '${chain}_${level - 1}'; + final depName = '${chain}_$level'; + bind() + .toProvideAsync(() async { + final prev = level > 1 + ? await currentScope.resolveAsync(named: prevDepName) + : null; + return UniversalServiceImpl( + value: depName, + dependency: prev, + ); + }) + .withName(depName) + .singleton(); } } + return; + } + + switch (scenario) { + case UniversalScenario.register: + bind() + .toProvide(() => UniversalServiceImpl(value: 'reg', dependency: null)) + .singleton(); + break; + case UniversalScenario.named: + bind().toProvide(() => UniversalServiceImpl(value: 'impl1')).withName('impl1'); + bind().toProvide(() => UniversalServiceImpl(value: 'impl2')).withName('impl2'); + break; + case UniversalScenario.chain: + for (var chainIndex = 0; chainIndex < chainCount; chainIndex++) { + for (var levelIndex = 0; levelIndex < nestingDepth; levelIndex++) { + final chain = chainIndex + 1; + final level = levelIndex + 1; + final prevDepName = '${chain}_${level - 1}'; + final depName = '${chain}_$level'; + switch (bindingMode) { + case UniversalBindingMode.singletonStrategy: + bind() + .toProvide(() => UniversalServiceImpl( + value: depName, + dependency: currentScope.tryResolve(named: prevDepName), + )) + .withName(depName) + .singleton(); + break; + case UniversalBindingMode.factoryStrategy: + bind() + .toProvide(() => UniversalServiceImpl( + value: depName, + dependency: currentScope.tryResolve(named: prevDepName), + )) + .withName(depName); + break; + case UniversalBindingMode.asyncStrategy: + bind() + .toProvideAsync(() async => UniversalServiceImpl( + value: depName, + dependency: await currentScope.resolveAsync(named: prevDepName), + )) + .withName(depName) + .singleton(); + break; + } + } + } + break; + case UniversalScenario.override: + // handled at benchmark level + break; + case UniversalScenario.asyncChain: + // already handled above + break; } } }