feat: support memory profiling in golang

.github/workflows/ci.yml

@@ -43,11 +43,17 @@ jobs:
         env:
           CODSPEED_GO_PKG_VERSION: ${{ github.head_ref || github.ref_name }}
 
-  compat-integration-test-walltime:
-    runs-on: codspeed-macro
+  compat-integration-test:
     strategy:
       matrix:
         target: [example, example/compat, example/timing, example/very/nested/module, example/external]
+        mode: [memory] # , walltime]
+        include:
+          # - mode: walltime
+          #   runner: codspeed-macro
+          - mode: memory
+            runner: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -62,9 +68,11 @@ jobs:
         env:
           CODSPEED_GO_PKG_VERSION: ${{ github.head_ref || github.ref_name }}
         with:
-          mode: walltime
+          runner-version: branch:cod-2107-support-golang
+          mode: ${{ matrix.mode }}
           working-directory: example
           run: cargo r --release --manifest-path ../go-runner/Cargo.toml -- test -bench=. ${{ matrix.target }}
+          upload-url: https://api.staging.preview.codspeed.io/upload
 
   go-runner-benchmarks:
     runs-on: codspeed-macro
@@ -99,7 +107,7 @@ jobs:
     needs:
       - lint
       - tests
-      - compat-integration-test-walltime
+      - compat-integration-test
       - go-runner-benchmarks
     steps:
       - uses: re-actors/alls-green@release/v1

example/rle_test.go

@@ -0,0 +1,47 @@
+package example
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func rleBad(s string) string {
+	if len(s) == 0 {
+		return ""
+	}
+	var out string
+	run := 1
+	for i := 1; i <= len(s); i++ {
+		if i < len(s) && s[i] == s[i-1] {
+			run++
+			continue
+		}
+		out += string(s[i-1])
+		out += strconv.Itoa(run)
+		out += "|"
+		run = 1
+	}
+	return out
+}
+
+func BenchmarkRLEBad(b *testing.B) {
+	in := strings.Repeat("AAAABBBCCDAA", 200)
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		_ = rleBad(in)
+	}
+}
+
+func leakyFunction() {
+	s := make([]string, 3)
+	for i := 0; i < 10000000; i++ {
+		s = append(s, "magical pandas")
+	}
+}
+
+func BenchmarkLeakyFunction(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		leakyFunction()
+	}
+}

go-runner/overlay/benchmark1.24.0.go

@@ -284,7 +284,7 @@ var labelsOnce sync.Once
 // subbenchmarks. b must not have subbenchmarks.
 func (b *B) run() {
 	labelsOnce.Do(func() {
-		fmt.Fprintf(b.w, "Running with CodSpeed (mode: walltime)\n")
+		fmt.Fprintf(b.w, "Running with CodSpeed (mode: %s)\n", getCodspeedRunnerMode())
 
 		fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS)
 		fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH)
@@ -347,8 +347,9 @@ func (b *B) launch() {
 	// b.Loop does its own ramp-up logic so we just need to run it once.
 	// If b.loop.n is non zero, it means b.Loop has already run.
 	if b.loop.n == 0 {
-		// Run the benchmark for at least the specified amount of time.
-		if b.benchTime.n > 0 {
+		if isAnalysisMode() {
+			runBenchmarkSingle(b)
+		} else if b.benchTime.n > 0 {
 			// We already ran a single iteration in run1.
 			// If -benchtime=1x was requested, use that result.
 			// See https://golang.org/issue/32051.
@@ -421,7 +422,9 @@ func (b *B) loopSlowPath() bool {
 
 	if b.loop.n == 0 {
 		// It's the first call to b.Loop() in the benchmark function.
-		if b.benchTime.n > 0 {
+		if isAnalysisMode() {
+			b.loop.n = 1
+		} else if b.benchTime.n > 0 {
 			// Fixed iteration count.
 			b.loop.n = uint64(b.benchTime.n)
 		} else {
@@ -442,7 +445,9 @@ func (b *B) loopSlowPath() bool {
 
 	// Should we keep iterating?
 	var more bool
-	if b.benchTime.n > 0 {
+	if isAnalysisMode() {
+		more = false
+	} else if b.benchTime.n > 0 {
 		// The iteration count is fixed, so we should have run this many and now
 		// be done.
 		if b.loop.i != uint64(b.benchTime.n) {

go-runner/overlay/benchmark1.25.0.go

@@ -284,7 +284,7 @@ var labelsOnce sync.Once
 // subbenchmarks. b must not have subbenchmarks.
 func (b *B) run() {
 	labelsOnce.Do(func() {
-		fmt.Fprintf(b.w, "Running with CodSpeed (mode: walltime)\n")
+		fmt.Fprintf(b.w, "Running with CodSpeed (mode: %s)\n", getCodspeedRunnerMode())
 
 		fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS)
 		fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH)
@@ -347,8 +347,9 @@ func (b *B) launch() {
 	// b.Loop does its own ramp-up logic so we just need to run it once.
 	// If b.loop.n is non zero, it means b.Loop has already run.
 	if b.loop.n == 0 {
-		// Run the benchmark for at least the specified amount of time.
-		if b.benchTime.n > 0 {
+		if isAnalysisMode() {
+			runBenchmarkSingle(b)
+		} else if b.benchTime.n > 0 {
 			// We already ran a single iteration in run1.
 			// If -benchtime=1x was requested, use that result.
 			// See https://golang.org/issue/32051.
@@ -421,7 +422,9 @@ func (b *B) loopSlowPath() bool {
 
 	if b.loop.n == 0 {
 		// It's the first call to b.Loop() in the benchmark function.
-		if b.benchTime.n > 0 {
+		if isAnalysisMode() {
+			b.loop.n = 1
+		} else if b.benchTime.n > 0 {
 			// Fixed iteration count.
 			b.loop.n = uint64(b.benchTime.n)
 		} else {
@@ -442,7 +445,9 @@ func (b *B) loopSlowPath() bool {
 
 	// Should we keep iterating?
 	var more bool
-	if b.benchTime.n > 0 {
+	if isAnalysisMode() {
+		more = false
+	} else if b.benchTime.n > 0 {
 		// The iteration count is fixed, so we should have run this many and now
 		// be done.
 		if b.loop.i != uint64(b.benchTime.n) {

go-runner/overlay/codspeed.go

@@ -13,6 +13,28 @@ import (
 	"time"
 )
 
+type codspeedRunnerMode string
+
+const (
+	modeWalltime   codspeedRunnerMode = "walltime"
+	modeSimulation codspeedRunnerMode = "simulation"
+	modeMemory     codspeedRunnerMode = "memory"
+)
+
+func getCodspeedRunnerMode() codspeedRunnerMode {
+	v := strings.ToLower(strings.TrimSpace(os.Getenv("CODSPEED_RUNNER_MODE")))
+	switch codspeedRunnerMode(v) {
+	case modeSimulation, modeMemory:
+		return codspeedRunnerMode(v)
+	default:
+		return modeWalltime
+	}
+}
+
+func isAnalysisMode() bool {
+	return getCodspeedRunnerMode() != modeWalltime
+}
+
 type codspeed struct {
 	instrument_hooks *InstrumentHooks
 
@@ -254,6 +276,9 @@ func (b *B) sendAccumulatedTimestamps() {
 }
 
 func (b *B) SaveMeasurement() {
+	if isAnalysisMode() {
+		return
+	}
 	if b.savedMeasurement {
 		return
 	}
@@ -314,6 +339,13 @@ func (b *B) StartTimerWithoutMarker() {
 	}
 }
 
+func runBenchmarkSingle(b *B) {
+	b.codspeed.instrument_hooks.StartBenchmark()
+	b.runN(1)
+	b.codspeed.instrument_hooks.StopBenchmark()
+	b.sendAccumulatedTimestamps()
+}
+
 func runBenchmarkWithWarmup(b *B) {
 	warmupD := b.benchTime.d / 10
 	warmupN := int64(1)

go-runner/overlay/instrument-hooks.go

@@ -4,8 +4,8 @@ package testing
 #cgo CFLAGS: -I@@INSTRUMENT_HOOKS_DIR@@/includes -Wno-format -Wno-format-security
 #include "@@INSTRUMENT_HOOKS_DIR@@/dist/core.c"
 
-#define MARKER_TYPE_BENCHMARK_START c_MARKER_TYPE_BENCHMARK_START__249
-#define MARKER_TYPE_BENCHMARK_END c_MARKER_TYPE_BENCHMARK_END__250
+#define MARKER_TYPE_BENCHMARK_START c_MARKER_TYPE_BENCHMARK_START__247
+#define MARKER_TYPE_BENCHMARK_END c_MARKER_TYPE_BENCHMARK_END__248
 typedef struct instruments_root_InstrumentHooks__547 InstrumentHooks;
 */
 import "C"

go-runner/src/lib.rs

@@ -25,8 +25,11 @@ pub fn run_benchmarks<P: AsRef<Path>>(
         bail!("Failed to run benchmarks: {error}");
     }
 
-    let profile_dir = profile_dir.as_ref().to_path_buf();
-    collect_walltime_results(&profile_dir).unwrap();
+    let mode = std::env::var("CODSPEED_RUNNER_MODE").unwrap_or_else(|_| "walltime".into());
+    if mode == "walltime" {
+        let profile_dir = profile_dir.as_ref().to_path_buf();
+        collect_walltime_results(&profile_dir).unwrap();
+    }
 
     Ok(())
 }

go-runner/src/runner/mod.rs

@@ -44,6 +44,10 @@ fn run_cmd<P: AsRef<Path>>(
     cmd.env("GOCACHE", _dir.path().join("gocache"));
     cmd.env("GOMODCACHE", _dir.path().join("gomodcache"));
 
+    if let Ok(mode) = std::env::var("CODSPEED_RUNNER_MODE") {
+        cmd.env("CODSPEED_RUNNER_MODE", mode);
+    }
+
     Ok((_dir, cmd))
 }
 

go-runner/src/runner/overlay/instrument_hooks.rs

@@ -6,7 +6,7 @@ use tar::Archive;
 use tempfile::TempDir;
 
 const INSTRUMENT_HOOKS_REPO: &str = "CodSpeedHQ/instrument-hooks";
-const INSTRUMENT_HOOKS_COMMIT: &str = "1752e9e4eae585e26703932d0055a1473dd77048";
+const INSTRUMENT_HOOKS_COMMIT: &str = "89fb72a076ec71c9eca6eee9bca98bada4b4dfb4";
 
 /// Get the instrument-hooks directory, downloading if necessary
 /// Downloads to /tmp/codspeed-instrument-hooks-{commit}/

go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example.snap

@@ -169,6 +169,28 @@ expression: results
         },
         "stats": "[stats]"
       },
+      {
+        "name": "BenchmarkLeakyFunction",
+        "uri": "example/rle_test.go::BenchmarkLeakyFunction",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      },
+      {
+        "name": "BenchmarkRLEBad",
+        "uri": "example/rle_test.go::BenchmarkRLEBad",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      },
       {
         "name": "BenchmarkWithOutlierMeasurementModern",
         "uri": "example/setup_test.go::BenchmarkWithOutlierMeasurementModern",