mirror of
https://github.com/pese-git/cherrypick.git
synced 2026-01-23 21:13:35 +00:00
refactor(main): extract BenchmarkRunner and BenchmarkResult, simplify main loop, unify sync/async cases
This commit is contained in:
@@ -31,7 +31,6 @@ UniversalScenario _toScenario(UniversalBenchmark b) {
|
|||||||
return UniversalScenario.override;
|
return UniversalScenario.override;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
UniversalBindingMode _toMode(UniversalBenchmark b) {
|
UniversalBindingMode _toMode(UniversalBenchmark b) {
|
||||||
switch (b) {
|
switch (b) {
|
||||||
case UniversalBenchmark.registerSingleton:
|
case UniversalBenchmark.registerSingleton:
|
||||||
@@ -49,18 +48,114 @@ UniversalBindingMode _toMode(UniversalBenchmark b) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
typedef SyncBench = UniversalChainBenchmark;
|
||||||
|
typedef AsyncBench = UniversalChainAsyncBenchmark;
|
||||||
|
|
||||||
|
class BenchmarkResult {
|
||||||
|
final List<num> timings;
|
||||||
|
final int memoryDiffKb;
|
||||||
|
final int deltaPeakKb;
|
||||||
|
final int peakRssKb;
|
||||||
|
BenchmarkResult({
|
||||||
|
required this.timings,
|
||||||
|
required this.memoryDiffKb,
|
||||||
|
required this.deltaPeakKb,
|
||||||
|
required this.peakRssKb,
|
||||||
|
});
|
||||||
|
factory BenchmarkResult.collect({
|
||||||
|
required List<num> timings,
|
||||||
|
required List<int> rssValues,
|
||||||
|
required int memBefore,
|
||||||
|
}) {
|
||||||
|
final memAfter = ProcessInfo.currentRss;
|
||||||
|
final memDiffKB = ((memAfter - memBefore) / 1024).round();
|
||||||
|
final peakRss = [...rssValues, memBefore].reduce(max);
|
||||||
|
final deltaPeakKb = ((peakRss - memBefore) / 1024).round();
|
||||||
|
return BenchmarkResult(
|
||||||
|
timings: timings,
|
||||||
|
memoryDiffKb: memDiffKB,
|
||||||
|
deltaPeakKb: deltaPeakKb,
|
||||||
|
peakRssKb: (peakRss / 1024).round(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class BenchmarkRunner {
|
||||||
|
static Future<BenchmarkResult> runSync({
|
||||||
|
required SyncBench benchmark,
|
||||||
|
required int warmups,
|
||||||
|
required int repeats,
|
||||||
|
}) async {
|
||||||
|
final timings = <num>[];
|
||||||
|
final rssValues = <int>[];
|
||||||
|
for (int i = 0; i < warmups; i++) {
|
||||||
|
benchmark.setup();
|
||||||
|
benchmark.run();
|
||||||
|
benchmark.teardown();
|
||||||
|
}
|
||||||
|
final memBefore = ProcessInfo.currentRss;
|
||||||
|
for (int i = 0; i < repeats; i++) {
|
||||||
|
benchmark.setup();
|
||||||
|
final sw = Stopwatch()..start();
|
||||||
|
benchmark.run();
|
||||||
|
sw.stop();
|
||||||
|
timings.add(sw.elapsedMicroseconds);
|
||||||
|
rssValues.add(ProcessInfo.currentRss);
|
||||||
|
benchmark.teardown();
|
||||||
|
}
|
||||||
|
return BenchmarkResult.collect(
|
||||||
|
timings: timings,
|
||||||
|
rssValues: rssValues,
|
||||||
|
memBefore: memBefore,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
static Future<BenchmarkResult> runAsync({
|
||||||
|
required AsyncBench benchmark,
|
||||||
|
required int warmups,
|
||||||
|
required int repeats,
|
||||||
|
}) async {
|
||||||
|
final timings = <num>[];
|
||||||
|
final rssValues = <int>[];
|
||||||
|
for (int i = 0; i < warmups; i++) {
|
||||||
|
await benchmark.setup();
|
||||||
|
await benchmark.run();
|
||||||
|
await benchmark.teardown();
|
||||||
|
}
|
||||||
|
final memBefore = ProcessInfo.currentRss;
|
||||||
|
for (int i = 0; i < repeats; i++) {
|
||||||
|
await benchmark.setup();
|
||||||
|
final sw = Stopwatch()..start();
|
||||||
|
await benchmark.run();
|
||||||
|
sw.stop();
|
||||||
|
timings.add(sw.elapsedMicroseconds);
|
||||||
|
rssValues.add(ProcessInfo.currentRss);
|
||||||
|
await benchmark.teardown();
|
||||||
|
}
|
||||||
|
return BenchmarkResult.collect(
|
||||||
|
timings: timings,
|
||||||
|
rssValues: rssValues,
|
||||||
|
memBefore: memBefore,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
T parseEnum<T>(String value, List<T> values, T defaultValue) {
|
||||||
|
return values.firstWhere(
|
||||||
|
(v) => v.toString().split('.').last.toLowerCase() == value.toLowerCase(),
|
||||||
|
orElse: () => defaultValue,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
Future<void> main(List<String> args) async {
|
Future<void> main(List<String> args) async {
|
||||||
final parser = ArgParser()
|
final parser = ArgParser()
|
||||||
..addOption('benchmark', abbr: 'b', help: 'One of: registerSingleton, chainSingleton, chainFactory, chainAsync, named, override, all', defaultsTo: 'chainSingleton')
|
..addOption('benchmark', abbr: 'b', defaultsTo: 'chainSingleton')
|
||||||
..addOption('chainCount', abbr: 'c', help: 'Comma-separated chainCounts', defaultsTo: '10')
|
..addOption('chainCount', abbr: 'c', defaultsTo: '10')
|
||||||
..addOption('nestingDepth', abbr: 'd', help: 'Comma-separated depths', defaultsTo: '5')
|
..addOption('nestingDepth', abbr: 'd', defaultsTo: '5')
|
||||||
..addOption('repeat', abbr: 'r', help: 'Repeats for each run (>=2)', defaultsTo: '2')
|
..addOption('repeat', abbr: 'r', defaultsTo: '2')
|
||||||
..addOption('warmup', abbr: 'w', help: 'Warmup runs', defaultsTo: '1')
|
..addOption('warmup', abbr: 'w', defaultsTo: '1')
|
||||||
..addOption('format', abbr: 'f', help: 'Output format (pretty, csv, json)', defaultsTo: 'pretty')
|
..addOption('format', abbr: 'f', defaultsTo: 'pretty')
|
||||||
..addFlag('help', abbr: 'h', negatable: false, help: 'Show help');
|
..addFlag('help', abbr: 'h', negatable: false, help: 'Show help');
|
||||||
|
|
||||||
final result = parser.parse(args);
|
final result = parser.parse(args);
|
||||||
|
|
||||||
if (result['help'] == true) {
|
if (result['help'] == true) {
|
||||||
print('UniversalChainBenchmark');
|
print('UniversalChainBenchmark');
|
||||||
print(parser.usage);
|
print(parser.usage);
|
||||||
@@ -68,41 +163,25 @@ Future<void> main(List<String> args) async {
|
|||||||
print(' dart run bin/main.dart --benchmark=chainFactory --chainCount=10 --nestingDepth=5 --format=csv');
|
print(' dart run bin/main.dart --benchmark=chainFactory --chainCount=10 --nestingDepth=5 --format=csv');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
final benchName = result['benchmark'] as String;
|
||||||
final benchName = result['benchmark'] as String;
|
|
||||||
final isAll = benchName == 'all';
|
final isAll = benchName == 'all';
|
||||||
final List<UniversalBenchmark> allBenches = [
|
final List<UniversalBenchmark> allBenches = UniversalBenchmark.values;
|
||||||
UniversalBenchmark.registerSingleton,
|
|
||||||
UniversalBenchmark.chainSingleton,
|
|
||||||
UniversalBenchmark.chainFactory,
|
|
||||||
UniversalBenchmark.chainAsync,
|
|
||||||
UniversalBenchmark.named,
|
|
||||||
UniversalBenchmark.override,
|
|
||||||
];
|
|
||||||
|
|
||||||
final List<UniversalBenchmark> benchesToRun = isAll
|
final List<UniversalBenchmark> benchesToRun = isAll
|
||||||
? allBenches
|
? allBenches
|
||||||
: [
|
: [parseEnum(benchName, UniversalBenchmark.values, UniversalBenchmark.chainSingleton)];
|
||||||
UniversalBenchmark.values.firstWhere(
|
|
||||||
(b) => b.toString().split('.').last == benchName,
|
|
||||||
orElse: () => UniversalBenchmark.chainSingleton,
|
|
||||||
),
|
|
||||||
];
|
|
||||||
|
|
||||||
final chainCounts = _parseIntList(result['chainCount'] as String);
|
final chainCounts = _parseIntList(result['chainCount'] as String);
|
||||||
final nestDepths = _parseIntList(result['nestingDepth'] as String);
|
final nestDepths = _parseIntList(result['nestingDepth'] as String);
|
||||||
final repeats = int.tryParse(result['repeat'] as String? ?? "") ?? 2;
|
final repeats = int.tryParse(result['repeat'] as String? ?? "") ?? 2;
|
||||||
final warmups = int.tryParse(result['warmup'] as String? ?? "") ?? 1;
|
final warmups = int.tryParse(result['warmup'] as String? ?? "") ?? 1;
|
||||||
final format = result['format'] as String;
|
final format = result['format'] as String;
|
||||||
|
|
||||||
final results = <Map<String, dynamic>>[];
|
final results = <Map<String, dynamic>>[];
|
||||||
|
|
||||||
for (final bench in benchesToRun) {
|
for (final bench in benchesToRun) {
|
||||||
final scenario = _toScenario(bench);
|
final scenario = _toScenario(bench);
|
||||||
final mode = _toMode(bench);
|
final mode = _toMode(bench);
|
||||||
for (final c in chainCounts) {
|
for (final c in chainCounts) {
|
||||||
for (final d in nestDepths) {
|
for (final d in nestDepths) {
|
||||||
// --- asyncChain special case ---
|
BenchmarkResult benchResult;
|
||||||
if (scenario == UniversalScenario.asyncChain) {
|
if (scenario == UniversalScenario.asyncChain) {
|
||||||
final di = CherrypickDIAdapter();
|
final di = CherrypickDIAdapter();
|
||||||
final benchAsync = UniversalChainAsyncBenchmark(
|
final benchAsync = UniversalChainAsyncBenchmark(
|
||||||
@@ -111,82 +190,27 @@ Future<void> main(List<String> args) async {
|
|||||||
nestingDepth: d,
|
nestingDepth: d,
|
||||||
mode: mode,
|
mode: mode,
|
||||||
);
|
);
|
||||||
final timings = <num>[];
|
benchResult = await BenchmarkRunner.runAsync(
|
||||||
final rssValues = <int>[];
|
benchmark: benchAsync,
|
||||||
// Warmup
|
warmups: warmups,
|
||||||
for (int i = 0; i < warmups; i++) {
|
repeats: repeats,
|
||||||
await benchAsync.setup();
|
);
|
||||||
await benchAsync.run();
|
} else {
|
||||||
await benchAsync.teardown();
|
final di = CherrypickDIAdapter();
|
||||||
}
|
final benchSync = UniversalChainBenchmark(
|
||||||
final memBefore = ProcessInfo.currentRss;
|
di,
|
||||||
for (int i = 0; i < repeats; i++) {
|
chainCount: c,
|
||||||
await benchAsync.setup();
|
nestingDepth: d,
|
||||||
final sw = Stopwatch()..start();
|
mode: mode,
|
||||||
await benchAsync.run();
|
scenario: scenario,
|
||||||
sw.stop();
|
);
|
||||||
timings.add(sw.elapsedMicroseconds);
|
benchResult = await BenchmarkRunner.runSync(
|
||||||
rssValues.add(ProcessInfo.currentRss);
|
benchmark: benchSync,
|
||||||
await benchAsync.teardown();
|
warmups: warmups,
|
||||||
}
|
repeats: repeats,
|
||||||
final memAfter = ProcessInfo.currentRss;
|
);
|
||||||
final memDiffKB = ((memAfter - memBefore) / 1024).round();
|
|
||||||
final peakRss = [...rssValues, memBefore].reduce(max);
|
|
||||||
final deltaPeakKb = ((peakRss - memBefore) / 1024).round();
|
|
||||||
timings.sort();
|
|
||||||
var mean = timings.reduce((a, b) => a + b) / timings.length;
|
|
||||||
var median = timings[timings.length ~/ 2];
|
|
||||||
var minVal = timings.first;
|
|
||||||
var maxVal = timings.last;
|
|
||||||
var stddev = sqrt(timings.map((x) => pow(x - mean, 2)).reduce((a, b) => a + b) / timings.length);
|
|
||||||
results.add({
|
|
||||||
'benchmark': 'Universal_$bench',
|
|
||||||
'chainCount': c,
|
|
||||||
'nestingDepth': d,
|
|
||||||
'mean_us': mean.round(),
|
|
||||||
'median_us': median.round(),
|
|
||||||
'stddev_us': stddev.round(),
|
|
||||||
'min_us': minVal.round(),
|
|
||||||
'max_us': maxVal.round(),
|
|
||||||
'trials': timings.length,
|
|
||||||
'timings_us': timings.map((t) => t.round()).toList(),
|
|
||||||
'memory_diff_kb': memDiffKB,
|
|
||||||
'delta_peak_kb': deltaPeakKb,
|
|
||||||
'peak_rss_kb': (peakRss / 1024).round(),
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
// --- Sync-case ---
|
final timings = benchResult.timings;
|
||||||
final di = CherrypickDIAdapter();
|
|
||||||
final benchSync = UniversalChainBenchmark(
|
|
||||||
di,
|
|
||||||
chainCount: c,
|
|
||||||
nestingDepth: d,
|
|
||||||
mode: mode,
|
|
||||||
scenario: scenario,
|
|
||||||
);
|
|
||||||
final timings = <num>[];
|
|
||||||
final rssValues = <int>[];
|
|
||||||
// Warmup
|
|
||||||
for (int i = 0; i < warmups; i++) {
|
|
||||||
benchSync.setup();
|
|
||||||
benchSync.run();
|
|
||||||
benchSync.teardown();
|
|
||||||
}
|
|
||||||
final memBefore = ProcessInfo.currentRss;
|
|
||||||
for (int i = 0; i < repeats; i++) {
|
|
||||||
benchSync.setup();
|
|
||||||
final sw = Stopwatch()..start();
|
|
||||||
benchSync.run();
|
|
||||||
sw.stop();
|
|
||||||
timings.add(sw.elapsedMicroseconds);
|
|
||||||
rssValues.add(ProcessInfo.currentRss);
|
|
||||||
benchSync.teardown();
|
|
||||||
}
|
|
||||||
final memAfter = ProcessInfo.currentRss;
|
|
||||||
final memDiffKB = ((memAfter - memBefore) / 1024).round();
|
|
||||||
final peakRss = [...rssValues, memBefore].reduce(max);
|
|
||||||
final deltaPeakKb = ((peakRss - memBefore) / 1024).round();
|
|
||||||
timings.sort();
|
timings.sort();
|
||||||
var mean = timings.reduce((a, b) => a + b) / timings.length;
|
var mean = timings.reduce((a, b) => a + b) / timings.length;
|
||||||
var median = timings[timings.length ~/ 2];
|
var median = timings[timings.length ~/ 2];
|
||||||
@@ -204,14 +228,13 @@ Future<void> main(List<String> args) async {
|
|||||||
'max_us': maxVal.round(),
|
'max_us': maxVal.round(),
|
||||||
'trials': timings.length,
|
'trials': timings.length,
|
||||||
'timings_us': timings.map((t) => t.round()).toList(),
|
'timings_us': timings.map((t) => t.round()).toList(),
|
||||||
'memory_diff_kb': memDiffKB,
|
'memory_diff_kb': benchResult.memoryDiffKb,
|
||||||
'delta_peak_kb': deltaPeakKb,
|
'delta_peak_kb': benchResult.deltaPeakKb,
|
||||||
'peak_rss_kb': (peakRss / 1024).round(),
|
'peak_rss_kb': benchResult.peakRssKb,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (format == 'json') {
|
if (format == 'json') {
|
||||||
print(_toJson(results));
|
print(_toJson(results));
|
||||||
} else if (format == 'csv') {
|
} else if (format == 'csv') {
|
||||||
@@ -223,7 +246,6 @@ Future<void> main(List<String> args) async {
|
|||||||
|
|
||||||
// --- helpers ---
|
// --- helpers ---
|
||||||
List<int> _parseIntList(String s) => s.split(',').map((e) => int.tryParse(e.trim()) ?? 0).where((x) => x > 0).toList();
|
List<int> _parseIntList(String s) => s.split(',').map((e) => int.tryParse(e.trim()) ?? 0).where((x) => x > 0).toList();
|
||||||
|
|
||||||
String _toPretty(List<Map<String, dynamic>> rows) {
|
String _toPretty(List<Map<String, dynamic>> rows) {
|
||||||
final keys = [
|
final keys = [
|
||||||
'benchmark','chainCount','nestingDepth','mean_us','median_us','stddev_us',
|
'benchmark','chainCount','nestingDepth','mean_us','median_us','stddev_us',
|
||||||
|
|||||||
Reference in New Issue
Block a user