From 09ed1865443b488a886a459bffe574b3851f3e73 Mon Sep 17 00:00:00 2001 From: Sergey Penkovsky Date: Wed, 6 Aug 2025 22:19:13 +0300 Subject: [PATCH] =?UTF-8?q?refactor(cli):=20modularize=20CLI=20=E2=80=94?= =?UTF-8?q?=20extract=20parser,=20runner,=20report=20and=20main=20logic=20?= =?UTF-8?q?into=20dedicated=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmark_cherrypick/bin/main.dart | 294 +----------------- .../lib/cli/benchmark_cli.dart | 76 +++++ benchmark_cherrypick/lib/cli/parser.dart | 104 +++++++ .../lib/cli/report/csv_report.dart | 21 ++ .../lib/cli/report/json_report.dart | 10 + .../lib/cli/report/pretty_report.dart | 15 + .../lib/cli/report/report_generator.dart | 4 + benchmark_cherrypick/lib/cli/runner.dart | 85 +++++ 8 files changed, 318 insertions(+), 291 deletions(-) create mode 100644 benchmark_cherrypick/lib/cli/benchmark_cli.dart create mode 100644 benchmark_cherrypick/lib/cli/parser.dart create mode 100644 benchmark_cherrypick/lib/cli/report/csv_report.dart create mode 100644 benchmark_cherrypick/lib/cli/report/json_report.dart create mode 100644 benchmark_cherrypick/lib/cli/report/pretty_report.dart create mode 100644 benchmark_cherrypick/lib/cli/report/report_generator.dart create mode 100644 benchmark_cherrypick/lib/cli/runner.dart diff --git a/benchmark_cherrypick/bin/main.dart b/benchmark_cherrypick/bin/main.dart index 586e1df..3c51a09 100644 --- a/benchmark_cherrypick/bin/main.dart +++ b/benchmark_cherrypick/bin/main.dart @@ -1,293 +1,5 @@ -import 'package:benchmark_cherrypick/benchmarks/universal_chain_benchmark.dart'; -import 'package:benchmark_cherrypick/benchmarks/universal_chain_async_benchmark.dart'; -import 'package:benchmark_cherrypick/di_adapters/cherrypick_adapter.dart'; -import 'package:benchmark_cherrypick/scenarios/universal_chain_module.dart'; -import 'package:args/args.dart'; -import 'dart:io'; -import 'dart:math'; - -enum UniversalBenchmark { - registerSingleton, - chainSingleton, - chainFactory, - chainAsync, - named, - override, -} - -UniversalScenario _toScenario(UniversalBenchmark b) { - switch (b) { - case UniversalBenchmark.registerSingleton: - return UniversalScenario.register; - case UniversalBenchmark.chainSingleton: - return UniversalScenario.chain; - case UniversalBenchmark.chainFactory: - return UniversalScenario.chain; - case UniversalBenchmark.chainAsync: - return UniversalScenario.asyncChain; - case UniversalBenchmark.named: - return UniversalScenario.named; - case UniversalBenchmark.override: - return UniversalScenario.override; - } -} -UniversalBindingMode _toMode(UniversalBenchmark b) { - switch (b) { - case UniversalBenchmark.registerSingleton: - return UniversalBindingMode.singletonStrategy; - case UniversalBenchmark.chainSingleton: - return UniversalBindingMode.singletonStrategy; - case UniversalBenchmark.chainFactory: - return UniversalBindingMode.factoryStrategy; - case UniversalBenchmark.chainAsync: - return UniversalBindingMode.asyncStrategy; - case UniversalBenchmark.named: - return UniversalBindingMode.singletonStrategy; - case UniversalBenchmark.override: - return UniversalBindingMode.singletonStrategy; - } -} - -typedef SyncBench = UniversalChainBenchmark; -typedef AsyncBench = UniversalChainAsyncBenchmark; - -class BenchmarkResult { - final List timings; - final int memoryDiffKb; - final int deltaPeakKb; - final int peakRssKb; - BenchmarkResult({ - required this.timings, - required this.memoryDiffKb, - required this.deltaPeakKb, - required this.peakRssKb, - }); - factory BenchmarkResult.collect({ - required List timings, - required List rssValues, - required int memBefore, - }) { - final memAfter = ProcessInfo.currentRss; - final memDiffKB = ((memAfter - memBefore) / 1024).round(); - final peakRss = [...rssValues, memBefore].reduce(max); - final deltaPeakKb = ((peakRss - memBefore) / 1024).round(); - return BenchmarkResult( - timings: timings, - memoryDiffKb: memDiffKB, - deltaPeakKb: deltaPeakKb, - peakRssKb: (peakRss / 1024).round(), - ); - } -} - -class BenchmarkRunner { - static Future runSync({ - required SyncBench benchmark, - required int warmups, - required int repeats, - }) async { - final timings = []; - final rssValues = []; - for (int i = 0; i < warmups; i++) { - benchmark.setup(); - benchmark.run(); - benchmark.teardown(); - } - final memBefore = ProcessInfo.currentRss; - for (int i = 0; i < repeats; i++) { - benchmark.setup(); - final sw = Stopwatch()..start(); - benchmark.run(); - sw.stop(); - timings.add(sw.elapsedMicroseconds); - rssValues.add(ProcessInfo.currentRss); - benchmark.teardown(); - } - return BenchmarkResult.collect( - timings: timings, - rssValues: rssValues, - memBefore: memBefore, - ); - } - static Future runAsync({ - required AsyncBench benchmark, - required int warmups, - required int repeats, - }) async { - final timings = []; - final rssValues = []; - for (int i = 0; i < warmups; i++) { - await benchmark.setup(); - await benchmark.run(); - await benchmark.teardown(); - } - final memBefore = ProcessInfo.currentRss; - for (int i = 0; i < repeats; i++) { - await benchmark.setup(); - final sw = Stopwatch()..start(); - await benchmark.run(); - sw.stop(); - timings.add(sw.elapsedMicroseconds); - rssValues.add(ProcessInfo.currentRss); - await benchmark.teardown(); - } - return BenchmarkResult.collect( - timings: timings, - rssValues: rssValues, - memBefore: memBefore, - ); - } -} - -T parseEnum(String value, List values, T defaultValue) { - return values.firstWhere( - (v) => v.toString().split('.').last.toLowerCase() == value.toLowerCase(), - orElse: () => defaultValue, - ); -} - -/// --- Report Generation --- -abstract class ReportGenerator { - String render(List> results); - List get keys; -} -class PrettyReport extends ReportGenerator { - @override - final List keys = [ - 'benchmark','chainCount','nestingDepth','mean_us','median_us','stddev_us', - 'min_us','max_us','trials','memory_diff_kb','delta_peak_kb','peak_rss_kb' - ]; - @override - String render(List> rows) { - final header = keys.join('\t'); - final lines = rows.map((r) => keys.map((k) => (r[k] ?? '').toString()).join('\t')).toList(); - return ([header] + lines).join('\n'); - } -} -class CsvReport extends ReportGenerator { - @override - final List keys = [ - 'benchmark','chainCount','nestingDepth','mean_us','median_us','stddev_us', - 'min_us','max_us','trials','timings_us','memory_diff_kb','delta_peak_kb','peak_rss_kb' - ]; - @override - String render(List> rows) { - final header = keys.join(','); - final lines = rows.map((r) => - keys.map((k) { - final v = r[k]; - if (v is List) return '"${v.join(';')}"'; - return (v ?? '').toString(); - }).join(',') - ).toList(); - return ([header] + lines).join('\n'); - } -} -class JsonReport extends ReportGenerator { - @override - List get keys => []; - @override - String render(List> rows) { - return '[\n${rows.map((r) => ' $r').join(',\n')}\n]'; - } -} +import 'package:benchmark_cherrypick/cli/benchmark_cli.dart'; Future main(List args) async { - final parser = ArgParser() - ..addOption('benchmark', abbr: 'b', defaultsTo: 'chainSingleton') - ..addOption('chainCount', abbr: 'c', defaultsTo: '10') - ..addOption('nestingDepth', abbr: 'd', defaultsTo: '5') - ..addOption('repeat', abbr: 'r', defaultsTo: '2') - ..addOption('warmup', abbr: 'w', defaultsTo: '1') - ..addOption('format', abbr: 'f', defaultsTo: 'pretty') - ..addFlag('help', abbr: 'h', negatable: false, help: 'Show help'); - final result = parser.parse(args); - if (result['help'] == true) { - print('UniversalChainBenchmark'); - print(parser.usage); - print('Example:'); - print(' dart run bin/main.dart --benchmark=chainFactory --chainCount=10 --nestingDepth=5 --format=csv'); - return; - } - final benchName = result['benchmark'] as String; - final isAll = benchName == 'all'; - final List allBenches = UniversalBenchmark.values; - final List benchesToRun = isAll - ? allBenches - : [parseEnum(benchName, UniversalBenchmark.values, UniversalBenchmark.chainSingleton)]; - final chainCounts = _parseIntList(result['chainCount'] as String); - final nestDepths = _parseIntList(result['nestingDepth'] as String); - final repeats = int.tryParse(result['repeat'] as String? ?? "") ?? 2; - final warmups = int.tryParse(result['warmup'] as String? ?? "") ?? 1; - final format = result['format'] as String; - - final results = >[]; - for (final bench in benchesToRun) { - final scenario = _toScenario(bench); - final mode = _toMode(bench); - for (final c in chainCounts) { - for (final d in nestDepths) { - BenchmarkResult benchResult; - if (scenario == UniversalScenario.asyncChain) { - final di = CherrypickDIAdapter(); - final benchAsync = UniversalChainAsyncBenchmark( - di, - chainCount: c, - nestingDepth: d, - mode: mode, - ); - benchResult = await BenchmarkRunner.runAsync( - benchmark: benchAsync, - warmups: warmups, - repeats: repeats, - ); - } else { - final di = CherrypickDIAdapter(); - final benchSync = UniversalChainBenchmark( - di, - chainCount: c, - nestingDepth: d, - mode: mode, - scenario: scenario, - ); - benchResult = await BenchmarkRunner.runSync( - benchmark: benchSync, - warmups: warmups, - repeats: repeats, - ); - } - final timings = benchResult.timings; - timings.sort(); - var mean = timings.reduce((a, b) => a + b) / timings.length; - var median = timings[timings.length ~/ 2]; - var minVal = timings.first; - var maxVal = timings.last; - var stddev = sqrt(timings.map((x) => pow(x - mean, 2)).reduce((a, b) => a + b) / timings.length); - results.add({ - 'benchmark': 'Universal_$bench', - 'chainCount': c, - 'nestingDepth': d, - 'mean_us': mean.round(), - 'median_us': median.round(), - 'stddev_us': stddev.round(), - 'min_us': minVal.round(), - 'max_us': maxVal.round(), - 'trials': timings.length, - 'timings_us': timings.map((t) => t.round()).toList(), - 'memory_diff_kb': benchResult.memoryDiffKb, - 'delta_peak_kb': benchResult.deltaPeakKb, - 'peak_rss_kb': benchResult.peakRssKb, - }); - } - } - } - - final reportGenerators = { - 'pretty': PrettyReport(), - 'csv': CsvReport(), - 'json': JsonReport(), - }; - print(reportGenerators[format]?.render(results) ?? PrettyReport().render(results)); -} - -List _parseIntList(String s) => s.split(',').map((e) => int.tryParse(e.trim()) ?? 0).where((x) => x > 0).toList(); + await BenchmarkCliRunner().run(args); +} \ No newline at end of file diff --git a/benchmark_cherrypick/lib/cli/benchmark_cli.dart b/benchmark_cherrypick/lib/cli/benchmark_cli.dart new file mode 100644 index 0000000..406be3e --- /dev/null +++ b/benchmark_cherrypick/lib/cli/benchmark_cli.dart @@ -0,0 +1,76 @@ +import 'dart:math'; + +import '../scenarios/universal_chain_module.dart'; +import 'report/pretty_report.dart'; +import 'report/csv_report.dart'; +import 'report/json_report.dart'; +import 'parser.dart'; +import 'runner.dart'; +import 'package:benchmark_cherrypick/benchmarks/universal_chain_benchmark.dart'; +import 'package:benchmark_cherrypick/benchmarks/universal_chain_async_benchmark.dart'; +import 'package:benchmark_cherrypick/di_adapters/cherrypick_adapter.dart'; + +class BenchmarkCliRunner { + Future run(List args) async { + final config = parseBenchmarkCli(args); + final results = >[]; + for (final bench in config.benchesToRun) { + final scenario = toScenario(bench); + final mode = toMode(bench); + for (final c in config.chainCounts) { + for (final d in config.nestDepths) { + BenchmarkResult benchResult; + if (scenario == UniversalScenario.asyncChain) { + final di = CherrypickDIAdapter(); + final benchAsync = UniversalChainAsyncBenchmark(di, + chainCount: c, nestingDepth: d, mode: mode, + ); + benchResult = await BenchmarkRunner.runAsync( + benchmark: benchAsync, + warmups: config.warmups, + repeats: config.repeats, + ); + } else { + final di = CherrypickDIAdapter(); + final benchSync = UniversalChainBenchmark(di, + chainCount: c, nestingDepth: d, mode: mode, scenario: scenario, + ); + benchResult = await BenchmarkRunner.runSync( + benchmark: benchSync, + warmups: config.warmups, + repeats: config.repeats, + ); + } + final timings = benchResult.timings; + timings.sort(); + var mean = timings.reduce((a, b) => a + b) / timings.length; + var median = timings[timings.length ~/ 2]; + var minVal = timings.first; + var maxVal = timings.last; + var stddev = timings.isEmpty ? 0 : sqrt(timings.map((x) => pow(x - mean, 2)).reduce((a, b) => a + b) / timings.length); + results.add({ + 'benchmark': 'Universal_$bench', + 'chainCount': c, + 'nestingDepth': d, + 'mean_us': mean.round(), + 'median_us': median.round(), + 'stddev_us': stddev.round(), + 'min_us': minVal.round(), + 'max_us': maxVal.round(), + 'trials': timings.length, + 'timings_us': timings.map((t) => t.round()).toList(), + 'memory_diff_kb': benchResult.memoryDiffKb, + 'delta_peak_kb': benchResult.deltaPeakKb, + 'peak_rss_kb': benchResult.peakRssKb, + }); + } + } + } + final reportGenerators = { + 'pretty': PrettyReport(), + 'csv': CsvReport(), + 'json': JsonReport(), + }; + print(reportGenerators[config.format]?.render(results) ?? PrettyReport().render(results)); + } +} \ No newline at end of file diff --git a/benchmark_cherrypick/lib/cli/parser.dart b/benchmark_cherrypick/lib/cli/parser.dart new file mode 100644 index 0000000..3d8450b --- /dev/null +++ b/benchmark_cherrypick/lib/cli/parser.dart @@ -0,0 +1,104 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:benchmark_cherrypick/scenarios/universal_chain_module.dart'; + +enum UniversalBenchmark { + registerSingleton, + chainSingleton, + chainFactory, + chainAsync, + named, + override, +} + +UniversalScenario toScenario(UniversalBenchmark b) { + switch (b) { + case UniversalBenchmark.registerSingleton: + return UniversalScenario.register; + case UniversalBenchmark.chainSingleton: + return UniversalScenario.chain; + case UniversalBenchmark.chainFactory: + return UniversalScenario.chain; + case UniversalBenchmark.chainAsync: + return UniversalScenario.asyncChain; + case UniversalBenchmark.named: + return UniversalScenario.named; + case UniversalBenchmark.override: + return UniversalScenario.override; + } +} + +UniversalBindingMode toMode(UniversalBenchmark b) { + switch (b) { + case UniversalBenchmark.registerSingleton: + return UniversalBindingMode.singletonStrategy; + case UniversalBenchmark.chainSingleton: + return UniversalBindingMode.singletonStrategy; + case UniversalBenchmark.chainFactory: + return UniversalBindingMode.factoryStrategy; + case UniversalBenchmark.chainAsync: + return UniversalBindingMode.asyncStrategy; + case UniversalBenchmark.named: + return UniversalBindingMode.singletonStrategy; + case UniversalBenchmark.override: + return UniversalBindingMode.singletonStrategy; + } +} + +T parseEnum(String value, List values, T defaultValue) { + return values.firstWhere( + (v) => v.toString().split('.').last.toLowerCase() == value.toLowerCase(), + orElse: () => defaultValue, + ); +} + +List parseIntList(String s) => + s.split(',').map((e) => int.tryParse(e.trim()) ?? 0).where((x) => x > 0).toList(); + +class BenchmarkCliConfig { + final List benchesToRun; + final List chainCounts; + final List nestDepths; + final int repeats; + final int warmups; + final String format; + BenchmarkCliConfig({ + required this.benchesToRun, + required this.chainCounts, + required this.nestDepths, + required this.repeats, + required this.warmups, + required this.format, + }); +} + +BenchmarkCliConfig parseBenchmarkCli(List args) { + final parser = ArgParser() + ..addOption('benchmark', abbr: 'b', defaultsTo: 'chainSingleton') + ..addOption('chainCount', abbr: 'c', defaultsTo: '10') + ..addOption('nestingDepth', abbr: 'd', defaultsTo: '5') + ..addOption('repeat', abbr: 'r', defaultsTo: '2') + ..addOption('warmup', abbr: 'w', defaultsTo: '1') + ..addOption('format', abbr: 'f', defaultsTo: 'pretty') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show help'); + final result = parser.parse(args); + if (result['help'] == true) { + print(parser.usage); + exit(0); + } + final benchName = result['benchmark'] as String; + final isAll = benchName == 'all'; + final allBenches = UniversalBenchmark.values; + final benchesToRun = isAll + ? allBenches + : [parseEnum(benchName, allBenches, UniversalBenchmark.chainSingleton)]; + return BenchmarkCliConfig( + benchesToRun: benchesToRun, + chainCounts: parseIntList(result['chainCount'] as String), + nestDepths: parseIntList(result['nestingDepth'] as String), + repeats: int.tryParse(result['repeat'] as String? ?? "") ?? 2, + warmups: int.tryParse(result['warmup'] as String? ?? "") ?? 1, + format: result['format'] as String, + ); +} \ No newline at end of file diff --git a/benchmark_cherrypick/lib/cli/report/csv_report.dart b/benchmark_cherrypick/lib/cli/report/csv_report.dart new file mode 100644 index 0000000..03e2449 --- /dev/null +++ b/benchmark_cherrypick/lib/cli/report/csv_report.dart @@ -0,0 +1,21 @@ +import 'report_generator.dart'; + +class CsvReport extends ReportGenerator { + @override + final List keys = [ + 'benchmark','chainCount','nestingDepth','mean_us','median_us','stddev_us', + 'min_us','max_us','trials','timings_us','memory_diff_kb','delta_peak_kb','peak_rss_kb' + ]; + @override + String render(List> rows) { + final header = keys.join(','); + final lines = rows.map((r) => + keys.map((k) { + final v = r[k]; + if (v is List) return '"${v.join(';')}"'; + return (v ?? '').toString(); + }).join(',') + ).toList(); + return ([header] + lines).join('\n'); + } +} \ No newline at end of file diff --git a/benchmark_cherrypick/lib/cli/report/json_report.dart b/benchmark_cherrypick/lib/cli/report/json_report.dart new file mode 100644 index 0000000..a21d662 --- /dev/null +++ b/benchmark_cherrypick/lib/cli/report/json_report.dart @@ -0,0 +1,10 @@ +import 'report_generator.dart'; + +class JsonReport extends ReportGenerator { + @override + List get keys => []; + @override + String render(List> rows) { + return '[\n${rows.map((r) => ' $r').join(',\n')}\n]'; + } +} \ No newline at end of file diff --git a/benchmark_cherrypick/lib/cli/report/pretty_report.dart b/benchmark_cherrypick/lib/cli/report/pretty_report.dart new file mode 100644 index 0000000..66dd2de --- /dev/null +++ b/benchmark_cherrypick/lib/cli/report/pretty_report.dart @@ -0,0 +1,15 @@ +import 'report_generator.dart'; + +class PrettyReport extends ReportGenerator { + @override + final List keys = [ + 'benchmark','chainCount','nestingDepth','mean_us','median_us','stddev_us', + 'min_us','max_us','trials','memory_diff_kb','delta_peak_kb','peak_rss_kb' + ]; + @override + String render(List> rows) { + final header = keys.join('\t'); + final lines = rows.map((r) => keys.map((k) => (r[k] ?? '').toString()).join('\t')).toList(); + return ([header] + lines).join('\n'); + } +} \ No newline at end of file diff --git a/benchmark_cherrypick/lib/cli/report/report_generator.dart b/benchmark_cherrypick/lib/cli/report/report_generator.dart new file mode 100644 index 0000000..c46645d --- /dev/null +++ b/benchmark_cherrypick/lib/cli/report/report_generator.dart @@ -0,0 +1,4 @@ +abstract class ReportGenerator { + String render(List> results); + List get keys; +} \ No newline at end of file diff --git a/benchmark_cherrypick/lib/cli/runner.dart b/benchmark_cherrypick/lib/cli/runner.dart new file mode 100644 index 0000000..b2ea3e1 --- /dev/null +++ b/benchmark_cherrypick/lib/cli/runner.dart @@ -0,0 +1,85 @@ +import 'dart:io'; +import 'dart:math'; +import 'package:benchmark_cherrypick/benchmarks/universal_chain_benchmark.dart'; +import 'package:benchmark_cherrypick/benchmarks/universal_chain_async_benchmark.dart'; + +class BenchmarkResult { + final List timings; + final int memoryDiffKb; + final int deltaPeakKb; + final int peakRssKb; + BenchmarkResult({ + required this.timings, + required this.memoryDiffKb, + required this.deltaPeakKb, + required this.peakRssKb, + }); + factory BenchmarkResult.collect({ + required List timings, + required List rssValues, + required int memBefore, + }) { + final memAfter = ProcessInfo.currentRss; + final memDiffKB = ((memAfter - memBefore) / 1024).round(); + final peakRss = [...rssValues, memBefore].reduce(max); + final deltaPeakKb = ((peakRss - memBefore) / 1024).round(); + return BenchmarkResult( + timings: timings, + memoryDiffKb: memDiffKB, + deltaPeakKb: deltaPeakKb, + peakRssKb: (peakRss / 1024).round(), + ); + } +} + +class BenchmarkRunner { + static Future runSync({ + required UniversalChainBenchmark benchmark, + required int warmups, + required int repeats, + }) async { + final timings = []; + final rssValues = []; + for (int i = 0; i < warmups; i++) { + benchmark.setup(); + benchmark.run(); + benchmark.teardown(); + } + final memBefore = ProcessInfo.currentRss; + for (int i = 0; i < repeats; i++) { + benchmark.setup(); + final sw = Stopwatch()..start(); + benchmark.run(); + sw.stop(); + timings.add(sw.elapsedMicroseconds); + rssValues.add(ProcessInfo.currentRss); + benchmark.teardown(); + } + return BenchmarkResult.collect(timings: timings, rssValues: rssValues, memBefore: memBefore); + } + + static Future runAsync({ + required UniversalChainAsyncBenchmark benchmark, + required int warmups, + required int repeats, + }) async { + final timings = []; + final rssValues = []; + for (int i = 0; i < warmups; i++) { + await benchmark.setup(); + await benchmark.run(); + await benchmark.teardown(); + } + final memBefore = ProcessInfo.currentRss; + for (int i = 0; i < repeats; i++) { + await benchmark.setup(); + final sw = Stopwatch()..start(); + await benchmark.run(); + sw.stop(); + timings.add(sw.elapsedMicroseconds); + rssValues.add(ProcessInfo.currentRss); + await benchmark.teardown(); + } + return BenchmarkResult.collect(timings: timings, rssValues: rssValues, memBefore: memBefore); + } +} \ No newline at end of file