diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bff068..ed9659e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,11 +43,17 @@ jobs: env: CODSPEED_GO_PKG_VERSION: ${{ github.head_ref || github.ref_name }} - compat-integration-test-walltime: - runs-on: codspeed-macro + compat-integration-test: strategy: matrix: target: [example, example/compat, example/timing, example/very/nested/module, example/external] + mode: [memory] # , walltime] + include: + # - mode: walltime + # runner: codspeed-macro + - mode: memory + runner: ubuntu-latest + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 with: @@ -62,9 +68,11 @@ jobs: env: CODSPEED_GO_PKG_VERSION: ${{ github.head_ref || github.ref_name }} with: - mode: walltime + runner-version: branch:cod-2107-support-golang + mode: ${{ matrix.mode }} working-directory: example run: cargo r --release --manifest-path ../go-runner/Cargo.toml -- test -bench=. ${{ matrix.target }} + upload-url: https://api.staging.preview.codspeed.io/upload go-runner-benchmarks: runs-on: codspeed-macro @@ -99,7 +107,7 @@ jobs: needs: - lint - tests - - compat-integration-test-walltime + - compat-integration-test - go-runner-benchmarks steps: - uses: re-actors/alls-green@release/v1 diff --git a/example/rle_test.go b/example/rle_test.go new file mode 100644 index 0000000..871bd37 --- /dev/null +++ b/example/rle_test.go @@ -0,0 +1,47 @@ +package example + +import ( + "strconv" + "strings" + "testing" +) + +func rleBad(s string) string { + if len(s) == 0 { + return "" + } + var out string + run := 1 + for i := 1; i <= len(s); i++ { + if i < len(s) && s[i] == s[i-1] { + run++ + continue + } + out += string(s[i-1]) + out += strconv.Itoa(run) + out += "|" + run = 1 + } + return out +} + +func BenchmarkRLEBad(b *testing.B) { + in := strings.Repeat("AAAABBBCCDAA", 200) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = rleBad(in) + } +} + +func leakyFunction() { + s := make([]string, 3) + for i := 0; i < 10000000; i++ { + s = append(s, "magical pandas") + } +} + +func BenchmarkLeakyFunction(b *testing.B) { + for i := 0; i < b.N; i++ { + leakyFunction() + } +} diff --git a/go-runner/overlay/benchmark1.24.0.go b/go-runner/overlay/benchmark1.24.0.go index b2a0eeb..b48462b 100644 --- a/go-runner/overlay/benchmark1.24.0.go +++ b/go-runner/overlay/benchmark1.24.0.go @@ -284,7 +284,7 @@ var labelsOnce sync.Once // subbenchmarks. b must not have subbenchmarks. func (b *B) run() { labelsOnce.Do(func() { - fmt.Fprintf(b.w, "Running with CodSpeed (mode: walltime)\n") + fmt.Fprintf(b.w, "Running with CodSpeed (mode: %s)\n", getCodspeedRunnerMode()) fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS) fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH) @@ -347,8 +347,9 @@ func (b *B) launch() { // b.Loop does its own ramp-up logic so we just need to run it once. // If b.loop.n is non zero, it means b.Loop has already run. if b.loop.n == 0 { - // Run the benchmark for at least the specified amount of time. - if b.benchTime.n > 0 { + if isAnalysisMode() { + runBenchmarkSingle(b) + } else if b.benchTime.n > 0 { // We already ran a single iteration in run1. // If -benchtime=1x was requested, use that result. // See https://golang.org/issue/32051. @@ -421,7 +422,9 @@ func (b *B) loopSlowPath() bool { if b.loop.n == 0 { // It's the first call to b.Loop() in the benchmark function. - if b.benchTime.n > 0 { + if isAnalysisMode() { + b.loop.n = 1 + } else if b.benchTime.n > 0 { // Fixed iteration count. b.loop.n = uint64(b.benchTime.n) } else { @@ -442,7 +445,9 @@ func (b *B) loopSlowPath() bool { // Should we keep iterating? var more bool - if b.benchTime.n > 0 { + if isAnalysisMode() { + more = false + } else if b.benchTime.n > 0 { // The iteration count is fixed, so we should have run this many and now // be done. if b.loop.i != uint64(b.benchTime.n) { diff --git a/go-runner/overlay/benchmark1.25.0.go b/go-runner/overlay/benchmark1.25.0.go index 8a1ee29..96ff2d4 100644 --- a/go-runner/overlay/benchmark1.25.0.go +++ b/go-runner/overlay/benchmark1.25.0.go @@ -284,7 +284,7 @@ var labelsOnce sync.Once // subbenchmarks. b must not have subbenchmarks. func (b *B) run() { labelsOnce.Do(func() { - fmt.Fprintf(b.w, "Running with CodSpeed (mode: walltime)\n") + fmt.Fprintf(b.w, "Running with CodSpeed (mode: %s)\n", getCodspeedRunnerMode()) fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS) fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH) @@ -347,8 +347,9 @@ func (b *B) launch() { // b.Loop does its own ramp-up logic so we just need to run it once. // If b.loop.n is non zero, it means b.Loop has already run. if b.loop.n == 0 { - // Run the benchmark for at least the specified amount of time. - if b.benchTime.n > 0 { + if isAnalysisMode() { + runBenchmarkSingle(b) + } else if b.benchTime.n > 0 { // We already ran a single iteration in run1. // If -benchtime=1x was requested, use that result. // See https://golang.org/issue/32051. @@ -421,7 +422,9 @@ func (b *B) loopSlowPath() bool { if b.loop.n == 0 { // It's the first call to b.Loop() in the benchmark function. - if b.benchTime.n > 0 { + if isAnalysisMode() { + b.loop.n = 1 + } else if b.benchTime.n > 0 { // Fixed iteration count. b.loop.n = uint64(b.benchTime.n) } else { @@ -442,7 +445,9 @@ func (b *B) loopSlowPath() bool { // Should we keep iterating? var more bool - if b.benchTime.n > 0 { + if isAnalysisMode() { + more = false + } else if b.benchTime.n > 0 { // The iteration count is fixed, so we should have run this many and now // be done. if b.loop.i != uint64(b.benchTime.n) { diff --git a/go-runner/overlay/codspeed.go b/go-runner/overlay/codspeed.go index a7cc73c..1f55b8c 100644 --- a/go-runner/overlay/codspeed.go +++ b/go-runner/overlay/codspeed.go @@ -13,6 +13,28 @@ import ( "time" ) +type codspeedRunnerMode string + +const ( + modeWalltime codspeedRunnerMode = "walltime" + modeSimulation codspeedRunnerMode = "simulation" + modeMemory codspeedRunnerMode = "memory" +) + +func getCodspeedRunnerMode() codspeedRunnerMode { + v := strings.ToLower(strings.TrimSpace(os.Getenv("CODSPEED_RUNNER_MODE"))) + switch codspeedRunnerMode(v) { + case modeSimulation, modeMemory: + return codspeedRunnerMode(v) + default: + return modeWalltime + } +} + +func isAnalysisMode() bool { + return getCodspeedRunnerMode() != modeWalltime +} + type codspeed struct { instrument_hooks *InstrumentHooks @@ -254,6 +276,9 @@ func (b *B) sendAccumulatedTimestamps() { } func (b *B) SaveMeasurement() { + if isAnalysisMode() { + return + } if b.savedMeasurement { return } @@ -314,6 +339,13 @@ func (b *B) StartTimerWithoutMarker() { } } +func runBenchmarkSingle(b *B) { + b.codspeed.instrument_hooks.StartBenchmark() + b.runN(1) + b.codspeed.instrument_hooks.StopBenchmark() + b.sendAccumulatedTimestamps() +} + func runBenchmarkWithWarmup(b *B) { warmupD := b.benchTime.d / 10 warmupN := int64(1) diff --git a/go-runner/overlay/instrument-hooks.go b/go-runner/overlay/instrument-hooks.go index be647a6..b65114c 100644 --- a/go-runner/overlay/instrument-hooks.go +++ b/go-runner/overlay/instrument-hooks.go @@ -4,8 +4,8 @@ package testing #cgo CFLAGS: -I@@INSTRUMENT_HOOKS_DIR@@/includes -Wno-format -Wno-format-security #include "@@INSTRUMENT_HOOKS_DIR@@/dist/core.c" -#define MARKER_TYPE_BENCHMARK_START c_MARKER_TYPE_BENCHMARK_START__249 -#define MARKER_TYPE_BENCHMARK_END c_MARKER_TYPE_BENCHMARK_END__250 +#define MARKER_TYPE_BENCHMARK_START c_MARKER_TYPE_BENCHMARK_START__247 +#define MARKER_TYPE_BENCHMARK_END c_MARKER_TYPE_BENCHMARK_END__248 typedef struct instruments_root_InstrumentHooks__547 InstrumentHooks; */ import "C" diff --git a/go-runner/src/lib.rs b/go-runner/src/lib.rs index 1b3f2e6..33ff120 100644 --- a/go-runner/src/lib.rs +++ b/go-runner/src/lib.rs @@ -25,8 +25,11 @@ pub fn run_benchmarks>( bail!("Failed to run benchmarks: {error}"); } - let profile_dir = profile_dir.as_ref().to_path_buf(); - collect_walltime_results(&profile_dir).unwrap(); + let mode = std::env::var("CODSPEED_RUNNER_MODE").unwrap_or_else(|_| "walltime".into()); + if mode == "walltime" { + let profile_dir = profile_dir.as_ref().to_path_buf(); + collect_walltime_results(&profile_dir).unwrap(); + } Ok(()) } diff --git a/go-runner/src/runner/mod.rs b/go-runner/src/runner/mod.rs index 3d535d2..8d0019e 100644 --- a/go-runner/src/runner/mod.rs +++ b/go-runner/src/runner/mod.rs @@ -44,6 +44,10 @@ fn run_cmd>( cmd.env("GOCACHE", _dir.path().join("gocache")); cmd.env("GOMODCACHE", _dir.path().join("gomodcache")); + if let Ok(mode) = std::env::var("CODSPEED_RUNNER_MODE") { + cmd.env("CODSPEED_RUNNER_MODE", mode); + } + Ok((_dir, cmd)) } diff --git a/go-runner/src/runner/overlay/instrument_hooks.rs b/go-runner/src/runner/overlay/instrument_hooks.rs index 710231b..2a01fee 100644 --- a/go-runner/src/runner/overlay/instrument_hooks.rs +++ b/go-runner/src/runner/overlay/instrument_hooks.rs @@ -6,7 +6,7 @@ use tar::Archive; use tempfile::TempDir; const INSTRUMENT_HOOKS_REPO: &str = "CodSpeedHQ/instrument-hooks"; -const INSTRUMENT_HOOKS_COMMIT: &str = "1752e9e4eae585e26703932d0055a1473dd77048"; +const INSTRUMENT_HOOKS_COMMIT: &str = "89fb72a076ec71c9eca6eee9bca98bada4b4dfb4"; /// Get the instrument-hooks directory, downloading if necessary /// Downloads to /tmp/codspeed-instrument-hooks-{commit}/ diff --git a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example.snap b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example.snap index b93d29f..d05f6d4 100644 --- a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example.snap +++ b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example.snap @@ -169,6 +169,28 @@ expression: results }, "stats": "[stats]" }, + { + "name": "BenchmarkLeakyFunction", + "uri": "example/rle_test.go::BenchmarkLeakyFunction", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkRLEBad", + "uri": "example/rle_test.go::BenchmarkRLEBad", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, { "name": "BenchmarkWithOutlierMeasurementModern", "uri": "example/setup_test.go::BenchmarkWithOutlierMeasurementModern",