diff --git a/.github/workflows/build-commit0-images.yml b/.github/workflows/build-commit0-images.yml index 5d821d2e5..d54c668f8 100644 --- a/.github/workflows/build-commit0-images.yml +++ b/.github/workflows/build-commit0-images.yml @@ -58,6 +58,7 @@ env: N_LIMIT: '' INSTANCE_IDS: '' SELECT_FILE: '' + LOG_JSON: '1' # Disable rich logging to prevent multiprocessing deadlock concurrency: group: build-commit0-${{ github.ref }} @@ -182,8 +183,7 @@ jobs: echo "Running: $CMD" eval "$CMD" - env: - DOCKER_BUILDKIT: 1 + env: DOCKER_BUILDKIT: 1 BUILDKIT_PROGRESS: plain - name: Archive build logs diff --git a/.github/workflows/build-gaia-images.yml b/.github/workflows/build-gaia-images.yml index 568e63113..d443acb65 100644 --- a/.github/workflows/build-gaia-images.yml +++ b/.github/workflows/build-gaia-images.yml @@ -18,6 +18,9 @@ concurrency: group: build-gaia-${{ github.ref }} cancel-in-progress: false +env: + LOG_JSON: '1' # Disable rich logging to prevent multiprocessing deadlock + jobs: build-and-push: if: > @@ -96,8 +99,7 @@ jobs: eval "$CMD" echo "✅ GAIA image with MCP layer built and pushed successfully" - env: - DOCKER_BUILDKIT: 1 + env: DOCKER_BUILDKIT: 1 BUILDKIT_PROGRESS: plain - name: Archive build logs diff --git a/.github/workflows/build-multiswebench-images.yml b/.github/workflows/build-multiswebench-images.yml index 8999c0a9b..ff969eac1 100644 --- a/.github/workflows/build-multiswebench-images.yml +++ b/.github/workflows/build-multiswebench-images.yml @@ -64,6 +64,7 @@ env: BUILD_BATCH_SIZE: '15' BUILDKIT_PRUNE_KEEP_GB: '60' BUILDKIT_PRUNE_THRESHOLD_PCT: '60' + LOG_JSON: '1' # Disable rich logging to prevent multiprocessing deadlock concurrency: group: build-multiswebench-${{ github.ref }} diff --git a/.github/workflows/build-swebench-images.yml b/.github/workflows/build-swebench-images.yml index 6053db373..88c697ed0 100644 --- a/.github/workflows/build-swebench-images.yml +++ b/.github/workflows/build-swebench-images.yml @@ -62,6 +62,7 @@ env: BUILD_BATCH_SIZE: '15' BUILDKIT_PRUNE_KEEP_GB: '60' BUILDKIT_PRUNE_THRESHOLD_PCT: '60' + LOG_JSON: '1' # Disable rich logging to prevent multiprocessing deadlock concurrency: group: build-swe-bench-${{ github.ref }} diff --git a/.github/workflows/build-swebenchmultimodal-images.yml b/.github/workflows/build-swebenchmultimodal-images.yml index 9b11b0ad5..6ff080583 100644 --- a/.github/workflows/build-swebenchmultimodal-images.yml +++ b/.github/workflows/build-swebenchmultimodal-images.yml @@ -62,6 +62,7 @@ env: BUILD_BATCH_SIZE: '15' BUILDKIT_PRUNE_KEEP_GB: '60' BUILDKIT_PRUNE_THRESHOLD_PCT: '60' + LOG_JSON: '1' # Disable rich logging to prevent multiprocessing deadlock concurrency: group: build-swe-bench-multimodal-${{ github.ref }} diff --git a/.github/workflows/build-swegym-images.yml b/.github/workflows/build-swegym-images.yml index 736ca3a81..e920d1532 100644 --- a/.github/workflows/build-swegym-images.yml +++ b/.github/workflows/build-swegym-images.yml @@ -58,6 +58,7 @@ env: BUILD_BATCH_SIZE: '15' BUILDKIT_PRUNE_KEEP_GB: '60' BUILDKIT_PRUNE_THRESHOLD_PCT: '60' + LOG_JSON: '1' # Disable rich logging to prevent multiprocessing deadlock concurrency: group: build-swe-gym-${{ github.ref }} @@ -240,8 +241,7 @@ jobs: echo "Running: $CMD" eval "$CMD" - env: - DOCKER_BUILDKIT: 1 + env: DOCKER_BUILDKIT: 1 BUILDKIT_PROGRESS: plain BUILDKIT_RESET_ON_FAILURE: 1 diff --git a/.github/workflows/build-swesmith-images.yml b/.github/workflows/build-swesmith-images.yml index a52a5c9f8..524c3981a 100644 --- a/.github/workflows/build-swesmith-images.yml +++ b/.github/workflows/build-swesmith-images.yml @@ -58,6 +58,7 @@ env: BUILD_BATCH_SIZE: '15' BUILDKIT_PRUNE_KEEP_GB: '60' BUILDKIT_PRUNE_THRESHOLD_PCT: '60' + LOG_JSON: '1' # Disable rich logging to prevent multiprocessing deadlock concurrency: group: build-swe-smith-${{ github.ref }} @@ -240,8 +241,7 @@ jobs: echo "Running: $CMD" eval "$CMD" - env: - DOCKER_BUILDKIT: 1 + env: DOCKER_BUILDKIT: 1 BUILDKIT_PROGRESS: plain BUILDKIT_RESET_ON_FAILURE: 1 diff --git a/.github/workflows/build-swtbench-images.yml b/.github/workflows/build-swtbench-images.yml index fe9d07be6..732fae429 100644 --- a/.github/workflows/build-swtbench-images.yml +++ b/.github/workflows/build-swtbench-images.yml @@ -63,6 +63,9 @@ concurrency: group: build-swt-bench-${{ github.ref }} cancel-in-progress: false +env: + LOG_JSON: '1' # Disable rich logging to prevent multiprocessing deadlock + jobs: build-and-push: if: > @@ -178,8 +181,7 @@ jobs: echo "Running: $CMD" eval "$CMD" - env: - DOCKER_BUILDKIT: 1 + env: DOCKER_BUILDKIT: 1 BUILDKIT_PROGRESS: plain - name: Build prebaked eval env images diff --git a/sitecustomize.py b/sitecustomize.py index 50338d1a8..de030efd4 100644 --- a/sitecustomize.py +++ b/sitecustomize.py @@ -6,11 +6,35 @@ this file at the repo root guarantees the patch runs before swebench is used. """ +import os import sys +# ============================================================================ +# CENTRALIZED LOGGING CONFIGURATION +# ============================================================================ +# Disable rich logging to avoid threading issues with multiprocessing. +# Rich's RichHandler creates locks and threads that don't play well with fork(). +os.environ["LOG_JSON"] = "1" + print("benchmarks sitecustomize imported", file=sys.stderr, flush=True) +# Suppress verbose Docker build stderr warnings (75k+ lines from uv copying files). +# This MUST be done in sitecustomize.py to affect forked child processes. +# The SDK logs every "[stderr] copying..." line as WARNING, causing massive log output +# and 3x slowdown with JSON logging (Rich logging throttled these automatically). +try: + import logging + + # Import SDK logger to trigger auto-configuration with LOG_JSON=1 + from openhands.sdk.logger import get_logger # noqa: F401 + + # Now suppress the verbose build logger in ALL processes (including forked children) + logging.getLogger("openhands.agent_server.docker.build").setLevel(logging.ERROR) +except Exception: + # Best-effort: don't break if SDK structure changes + pass + try: # Reuse the actual patch logic that lives alongside the benchmarks package. from benchmarks.utils.sitecustomize import _apply_modal_logging_patch