refactor(benchmarks): unify benchmark structure, enable CLI parameterization, run matrix, add CSV/JSON/pretty output

- All benchmarks now use a unified base mixin for setup/teardown (BenchmarkWithScope).
- Added args package support: CLI flags for choosing benchmarks, chain counts, nesting depths, output format.
- Support for running benchmarks in matrix mode (multiple parameter sets).
- Machine-readable output: csv, json, pretty-table.
- Loop and naming lint fixes, unused imports removed.
This commit is contained in:
Sergey Penkovsky
2025-08-06 13:29:23 +03:00
parent a5ef0dc437
commit 926bbf15f4
8 changed files with 211 additions and 71 deletions

View File

@@ -2,16 +2,124 @@ import 'package:benchmark_cherrypick/cherrypick_benchmark.dart';
import 'package:benchmark_cherrypick/complex_bindings_benchmark.dart'; import 'package:benchmark_cherrypick/complex_bindings_benchmark.dart';
import 'package:benchmark_cherrypick/async_chain_benchmark.dart'; import 'package:benchmark_cherrypick/async_chain_benchmark.dart';
import 'package:benchmark_cherrypick/scope_override_benchmark.dart'; import 'package:benchmark_cherrypick/scope_override_benchmark.dart';
import 'package:args/args.dart';
void main(List<String> args) async { Future<void> main(List<String> args) async {
// Синхронные бенчмарки final parser = ArgParser()
RegisterAndResolveBenchmark().report(); ..addOption('benchmark', abbr: 'b', help: 'Benchmark name (register, chain_singleton, chain_factory, named, override, async_chain, all)', defaultsTo: 'all')
ChainSingletonBenchmark().report(); ..addOption('chainCount', abbr: 'c', help: 'Comma-separated chainCounts (используется в chain_singleton/factory)', defaultsTo: '100')
ChainFactoryBenchmark().report(); ..addOption('nestingDepth', abbr: 'd', help: 'Comma-separated depths (используется в chain_singleton/factory)', defaultsTo: '100')
NamedResolveBenchmark().report(); ..addOption('format', abbr: 'f', help: 'Output format (pretty, csv, json)', defaultsTo: 'pretty')
..addFlag('help', abbr: 'h', negatable: false, help: 'Show help');
// Асинхронный бенчмарк final result = parser.parse(args);
await AsyncChainBenchmark().report();
ScopeOverrideBenchmark().report(); if (result['help'] == true) {
print('Dart DI benchmarks');
print(parser.usage);
return;
}
final benchmark = result['benchmark'] as String;
final format = result['format'] as String;
final chainCounts = _parseIntList(result['chainCount'] as String);
final nestDepths = _parseIntList(result['nestingDepth'] as String);
final results = <Map<String, dynamic>>[];
void addResult(String name, int? chainCount, int? nestingDepth, num elapsed) {
results.add({
'benchmark': name,
'chainCount': chainCount,
'nestingDepth': nestingDepth,
'elapsed_us': elapsed.round()
});
}
Future<void> runAndCollect(String name, Future<num> Function() fn, {int? chainCount, int? nestingDepth}) async {
final elapsed = await fn();
addResult(name, chainCount, nestingDepth, elapsed);
}
if (benchmark == 'all' || benchmark == 'register') {
await runAndCollect('RegisterAndResolve', () async {
return _captureReport(RegisterAndResolveBenchmark().report);
});
}
if (benchmark == 'all' || benchmark == 'chain_singleton') {
for (final c in chainCounts) {
for (final d in nestDepths) {
await runAndCollect('ChainSingleton', () async {
return _captureReport(() => ChainSingletonBenchmark(chainCount: c, nestingDepth: d).report());
}, chainCount: c, nestingDepth: d);
}
}
}
if (benchmark == 'all' || benchmark == 'chain_factory') {
for (final c in chainCounts) {
for (final d in nestDepths) {
await runAndCollect('ChainFactory', () async {
return _captureReport(() => ChainFactoryBenchmark(chainCount: c, nestingDepth: d).report());
}, chainCount: c, nestingDepth: d);
}
}
}
if (benchmark == 'all' || benchmark == 'named') {
await runAndCollect('NamedResolve', () async {
return _captureReport(NamedResolveBenchmark().report);
});
}
if (benchmark == 'all' || benchmark == 'override') {
await runAndCollect('ScopeOverride', () async {
return _captureReport(ScopeOverrideBenchmark().report);
});
}
if (benchmark == 'all' || benchmark == 'async_chain') {
await runAndCollect('AsyncChain', () async {
return _captureReportAsync(AsyncChainBenchmark().report);
});
}
if (format == 'json') {
print(_toJson(results));
} else if (format == 'csv') {
print(_toCsv(results));
} else {
print(_toPretty(results));
}
} }
// --- helpers ---
List<int> _parseIntList(String s) => s.split(',').map((e) => int.tryParse(e.trim()) ?? 0).where((x) => x > 0).toList();
Future<num> _captureReport(void Function() fn) async {
final sw = Stopwatch()..start();
fn();
sw.stop();
return sw.elapsedMicroseconds;
}
Future<num> _captureReportAsync(Future<void> Function() fn) async {
final sw = Stopwatch()..start();
await fn();
sw.stop();
return sw.elapsedMicroseconds;
}
String _toPretty(List<Map<String, dynamic>> rows) {
final keys = ['benchmark','chainCount','nestingDepth','elapsed_us'];
final header = keys.join('\t');
final lines = rows.map((r) => keys.map((k) => (r[k] ?? '').toString()).join('\t')).toList();
return ([header] + lines).join('\n');
}
String _toCsv(List<Map<String, dynamic>> rows) {
final keys = ['benchmark','chainCount','nestingDepth','elapsed_us'];
final header = keys.join(',');
final lines = rows.map((r) => keys.map((k) => (r[k] ?? '').toString()).join(',')).toList();
return ([header] + lines).join('\n');
}
String _toJson(List<Map<String, dynamic>> rows) {
return '[\n${rows.map((r) => ' $r').join(',\n')}\n]';
}
// --- end helpers ---

View File

@@ -1,12 +1,15 @@
// ignore: depend_on_referenced_packages // ignore: depend_on_referenced_packages
import 'package:benchmark_harness/benchmark_harness.dart'; import 'package:benchmark_harness/benchmark_harness.dart';
import 'package:cherrypick/cherrypick.dart'; import 'package:cherrypick/cherrypick.dart';
import 'benchmark_utils.dart';
class AsyncA {} class AsyncA {}
class AsyncB { class AsyncB {
final AsyncA a; final AsyncA a;
AsyncB(this.a); AsyncB(this.a);
} }
class AsyncC { class AsyncC {
final AsyncB b; final AsyncB b;
AsyncC(this.b); AsyncC(this.b);
@@ -16,26 +19,30 @@ class AsyncChainModule extends Module {
@override @override
void builder(Scope currentScope) { void builder(Scope currentScope) {
bind<AsyncA>().toProvideAsync(() async => AsyncA()).singleton(); bind<AsyncA>().toProvideAsync(() async => AsyncA()).singleton();
bind<AsyncB>().toProvideAsync(() async => AsyncB(await currentScope.resolveAsync<AsyncA>())).singleton(); bind<AsyncB>()
bind<AsyncC>().toProvideAsync(() async => AsyncC(await currentScope.resolveAsync<AsyncB>())).singleton(); .toProvideAsync(
() async => AsyncB(await currentScope.resolveAsync<AsyncA>()))
.singleton();
bind<AsyncC>()
.toProvideAsync(
() async => AsyncC(await currentScope.resolveAsync<AsyncB>()))
.singleton();
} }
} }
class AsyncChainBenchmark extends AsyncBenchmarkBase { class AsyncChainBenchmark extends AsyncBenchmarkBase with BenchmarkWithScope {
AsyncChainBenchmark() : super('AsyncChain (A->B->C, async)'); AsyncChainBenchmark() : super('AsyncChain (A->B->C, async)');
late Scope scope;
@override @override
Future<void> setup() async { Future<void> setup() async {
CherryPick.disableGlobalCycleDetection(); setupScope([AsyncChainModule()]);
CherryPick.disableGlobalCrossScopeCycleDetection();
scope = CherryPick.openRootScope();
scope.installModules([AsyncChainModule()]);
} }
@override @override
Future<void> teardown() async { Future<void> teardown() async {
CherryPick.closeRootScope(); teardownScope();
} }
@override @override
Future<void> run() async { Future<void> run() async {
await scope.resolveAsync<AsyncC>(); await scope.resolveAsync<AsyncC>();

View File

@@ -0,0 +1,29 @@
import 'package:cherrypick/cherrypick.dart';
/// Миксин для упрощения работы с CherryPick Scope в бенчмарках.
mixin BenchmarkWithScope {
Scope? _scope;
/// Отключить глобальные проверки циклов и создать корневой scope с модулями.
void setupScope(List<Module> modules,
{bool disableCycleDetection = true,
bool disableCrossScopeCycleDetection = true}) {
if (disableCycleDetection) {
CherryPick.disableGlobalCycleDetection();
}
if (disableCrossScopeCycleDetection) {
CherryPick.disableGlobalCrossScopeCycleDetection();
}
_scope = CherryPick.openRootScope();
_scope!.installModules(modules);
}
/// Закрывает текущий scope.
void teardownScope() {
CherryPick.closeRootScope();
_scope = null;
}
/// Получить текущий scope. Не null после setupScope.
Scope get scope => _scope!;
}

View File

@@ -7,7 +7,6 @@ class AppModule extends Module {
void builder(Scope currentScope) { void builder(Scope currentScope) {
bind<FooService>().toProvide(() => FooService()); bind<FooService>().toProvide(() => FooService());
} }
} }
// Dummy service for DI // Dummy service for DI
@@ -23,7 +22,6 @@ class RegisterAndResolveBenchmark extends BenchmarkBase {
CherryPick.disableGlobalCrossScopeCycleDetection(); CherryPick.disableGlobalCrossScopeCycleDetection();
scope = CherryPick.openRootScope(); scope = CherryPick.openRootScope();
scope.installModules([AppModule()]); scope.installModules([AppModule()]);
} }
@override @override

View File

@@ -1,6 +1,7 @@
// ignore: depend_on_referenced_packages // ignore: depend_on_referenced_packages
import 'package:benchmark_harness/benchmark_harness.dart'; import 'package:benchmark_harness/benchmark_harness.dart';
import 'package:cherrypick/cherrypick.dart'; import 'package:cherrypick/cherrypick.dart';
import 'benchmark_utils.dart';
// === DI graph: A -> B -> C (singleton) === // === DI graph: A -> B -> C (singleton) ===
abstract class Service { abstract class Service {
@@ -58,7 +59,7 @@ class ChainSingletonModule extends Module {
} }
} }
class ChainSingletonBenchmark extends BenchmarkBase { class ChainSingletonBenchmark extends BenchmarkBase with BenchmarkWithScope {
final int chainCount; final int chainCount;
final int nestingDepth; final int nestingDepth;
@@ -69,12 +70,10 @@ class ChainSingletonBenchmark extends BenchmarkBase {
'ChainSingleton (A->B->C, singleton). ' 'ChainSingleton (A->B->C, singleton). '
'C/D = $chainCount/$nestingDepth. ', 'C/D = $chainCount/$nestingDepth. ',
); );
late Scope scope;
@override @override
void setup() { void setup() {
scope = CherryPick.openRootScope(); setupScope([
scope.installModules([
ChainSingletonModule( ChainSingletonModule(
chainCount: chainCount, chainCount: chainCount,
nestingDepth: nestingDepth, nestingDepth: nestingDepth,
@@ -83,7 +82,7 @@ class ChainSingletonBenchmark extends BenchmarkBase {
} }
@override @override
void teardown() => CherryPick.closeRootScope(); void teardown() => teardownScope();
@override @override
void run() { void run() {
@@ -130,11 +129,8 @@ class ChainFactoryModule extends Module {
} }
} }
class ChainFactoryBenchmark extends BenchmarkBase { class ChainFactoryBenchmark extends BenchmarkBase with BenchmarkWithScope {
// количество независимых цепочек
final int chainCount; final int chainCount;
// глубина вложенности
final int nestingDepth; final int nestingDepth;
ChainFactoryBenchmark({ ChainFactoryBenchmark({
@@ -145,15 +141,9 @@ class ChainFactoryBenchmark extends BenchmarkBase {
'C/D = $chainCount/$nestingDepth. ', 'C/D = $chainCount/$nestingDepth. ',
); );
late Scope scope;
@override @override
void setup() { void setup() {
CherryPick.disableGlobalCycleDetection(); setupScope([
CherryPick.disableGlobalCrossScopeCycleDetection();
scope = CherryPick.openRootScope();
scope.installModules([
ChainFactoryModule( ChainFactoryModule(
chainCount: chainCount, chainCount: chainCount,
nestingDepth: nestingDepth, nestingDepth: nestingDepth,
@@ -162,7 +152,7 @@ class ChainFactoryBenchmark extends BenchmarkBase {
} }
@override @override
void teardown() => CherryPick.closeRootScope(); void teardown() => teardownScope();
@override @override
void run() { void run() {
@@ -184,18 +174,16 @@ class NamedModule extends Module {
} }
} }
class NamedResolveBenchmark extends BenchmarkBase { class NamedResolveBenchmark extends BenchmarkBase with BenchmarkWithScope {
NamedResolveBenchmark() : super('NamedResolve (by name)'); NamedResolveBenchmark() : super('NamedResolve (by name)');
late Scope scope;
@override @override
void setup() { void setup() {
scope = CherryPick.openRootScope(); setupScope([NamedModule()]);
scope.installModules([NamedModule()]);
} }
@override @override
void teardown() => CherryPick.closeRootScope(); void teardown() => teardownScope();
@override @override
void run() { void run() {
@@ -203,4 +191,3 @@ class NamedResolveBenchmark extends BenchmarkBase {
scope.resolve<Object>(named: 'impl2'); scope.resolve<Object>(named: 'impl2');
} }
} }

View File

@@ -1,9 +1,12 @@
// ignore: depend_on_referenced_packages // ignore: depend_on_referenced_packages
import 'package:benchmark_harness/benchmark_harness.dart'; import 'package:benchmark_harness/benchmark_harness.dart';
import 'package:cherrypick/cherrypick.dart'; import 'package:cherrypick/cherrypick.dart';
import 'benchmark_utils.dart';
class Shared {} class Shared {}
class ParentImpl extends Shared {} class ParentImpl extends Shared {}
class ChildImpl extends Shared {} class ChildImpl extends Shared {}
class ParentModule extends Module { class ParentModule extends Module {
@@ -20,23 +23,22 @@ class ChildOverrideModule extends Module {
} }
} }
class ScopeOverrideBenchmark extends BenchmarkBase { class ScopeOverrideBenchmark extends BenchmarkBase with BenchmarkWithScope {
ScopeOverrideBenchmark() : super('ScopeOverride (child overrides parent)'); ScopeOverrideBenchmark() : super('ScopeOverride (child overrides parent)');
late Scope parent;
late Scope child; late Scope child;
@override @override
void setup() { void setup() {
CherryPick.disableGlobalCycleDetection(); setupScope([ParentModule()]);
CherryPick.disableGlobalCrossScopeCycleDetection(); child = scope.openSubScope('child');
parent = CherryPick.openRootScope();
parent.installModules([ParentModule()]);
child = parent.openSubScope('child');
child.installModules([ChildOverrideModule()]); child.installModules([ChildOverrideModule()]);
} }
@override @override
void teardown() { void teardown() {
CherryPick.closeRootScope(); teardownScope();
} }
@override @override
void run() { void run() {
// Должен возвращать ChildImpl, а не ParentImpl // Должен возвращать ChildImpl, а не ParentImpl

View File

@@ -9,6 +9,14 @@ packages:
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "0.1.4" version: "0.1.4"
args:
dependency: "direct main"
description:
name: args
sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04
url: "https://pub.dev"
source: hosted
version: "2.7.0"
benchmark_harness: benchmark_harness:
dependency: "direct dev" dependency: "direct dev"
description: description:
@@ -31,7 +39,7 @@ packages:
path: "../cherrypick" path: "../cherrypick"
relative: true relative: true
source: path source: path
version: "3.0.0-dev.1" version: "3.0.0-dev.2"
exception_templates: exception_templates:
dependency: transitive dependency: transitive
description: description:

View File

@@ -9,6 +9,7 @@ environment:
dependencies: dependencies:
cherrypick: cherrypick:
path: ../cherrypick path: ../cherrypick
args: ^2.7.0
dev_dependencies: dev_dependencies:
lints: ^5.0.0 lints: ^5.0.0