From 10604fe0c1e089cd7b1b749622dad25a91b72d7e Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Thu, 22 Jan 2026 10:48:21 -0500 Subject: [PATCH] convert wd_test to use sh_test --- build/fixtures/BUILD.bazel | 5 +- build/fixtures/wd_test.sh | 46 +++++ build/wd_test.bzl | 379 ++++++++++--------------------------- 3 files changed, 152 insertions(+), 278 deletions(-) create mode 100755 build/fixtures/wd_test.sh diff --git a/build/fixtures/BUILD.bazel b/build/fixtures/BUILD.bazel index e85ef5d729c..0c9872b7e4a 100644 --- a/build/fixtures/BUILD.bazel +++ b/build/fixtures/BUILD.bazel @@ -1 +1,4 @@ -exports_files(["kj_test.sh"]) +exports_files([ + "kj_test.sh", + "wd_test.sh", +]) diff --git a/build/fixtures/wd_test.sh b/build/fixtures/wd_test.sh new file mode 100755 index 00000000000..019de7deb2d --- /dev/null +++ b/build/fixtures/wd_test.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Test runner script for wd_test. Handles three modes: +# - Normal: runs workerd test directly +# - Sidecar: runs workerd test via supervisor which manages a parallel sidecar process +# - Snapshot: runs workerd twice (save snapshot, then load snapshot) for Python tests +# +# Environment variables (set by wd_test.bzl): +# SIDECAR_COMMAND - If set, enables sidecar mode +# SIDECAR_SUPERVISOR - Path to supervisor executable (required for sidecar mode) +# PORTS_TO_ASSIGN - Comma-separated port binding names for sidecar +# RANDOMIZE_IP - "true"/"false" for sidecar IP randomization +# PYTHON_SNAPSHOT_TEST - If set, enables snapshot mode +# PYTHON_SAVE_SNAPSHOT_ARGS - Additional args for snapshot save phase + +set -euo pipefail + +# Set up coverage for workerd subprocess. +# LLVM_PROFILE_FILE tells the coverage-instrumented workerd binary where to write profraw data. +# KJ_CLEAN_SHUTDOWN ensures proper shutdown so coverage data is flushed. +if [ -n "${COVERAGE_DIR:-}" ]; then + export LLVM_PROFILE_FILE="$COVERAGE_DIR/%p.profraw" + export KJ_CLEAN_SHUTDOWN=1 +fi + +run_test() { + "$@" -dTEST_TMPDIR="${TEST_TMPDIR:-/tmp}" +} + +# Sidecar mode: run via supervisor which starts sidecar process in parallel. +# The supervisor handles port assignment and ensures sidecar is ready before test starts. +if [ -n "${SIDECAR_COMMAND:-}" ]; then + "$SIDECAR_SUPERVISOR" run_test "$@" + +# Snapshot mode: run workerd twice for Python memory snapshot tests. +# First run creates the snapshot, second run uses it. +elif [ -n "${PYTHON_SNAPSHOT_TEST:-}" ]; then + echo "Creating Python Snapshot" + run_test "$@" --python-save-snapshot ${PYTHON_SAVE_SNAPSHOT_ARGS:-} + echo "" + echo "Using Python Snapshot" + run_test "$@" --python-load-snapshot snapshot.bin + +# Normal mode: just run the test. +else + run_test "$@" +fi diff --git a/build/wd_test.bzl b/build/wd_test.bzl index c8e02b489cd..cad630ae224 100644 --- a/build/wd_test.bzl +++ b/build/wd_test.bzl @@ -1,4 +1,5 @@ load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project") +load("@rules_shell//shell:sh_test.bzl", "sh_test") def wd_test( src, @@ -11,43 +12,66 @@ def wd_test( generate_all_autogates_variant = True, generate_all_compat_flags_variant = True, compat_date = "", + sidecar = None, + sidecar_port_bindings = [], + sidecar_randomize_ip = True, + load_snapshot = None, + env = {}, **kwargs): """Rule to define tests that run `workerd test` with a particular config. Args: - src: A .capnp config file defining the test. (`name` will be derived from this if not + src: A .capnp config file defining the test. (`name` will be derived from this if not specified.) The extension `.wd-test` is also permitted instead of `.capnp`, in order to avoid confusing other build systems that may assume a `.capnp` file should be compiled. - data: Additional files which the .capnp config file may embed. All TypeScript files will be compiled, - their resulting files will be passed to the test as well. Usually TypeScript or Javascript source files. - args: Additional arguments to pass to `workerd`. Typically used to pass `--experimental`. - generate_default_variant: If True (default), generate the default variant with oldest compat date. - generate_all_autogates_variant: If True (default), generate @all-autogates variants. - generate_all_compat_flags_variant: If True (default), generate @all-compat-flags variants. - compat_date: If specified, use this compat date for the default variant instead of 2000-01-01. - Does not affect the @all-compat-flags variant which always uses 2999-12-31. + data: Additional files which the .capnp config file may embed. All TypeScript files will be + compiled, their resulting files will be passed to the test as well. Usually TypeScript or + Javascript source files. + args: Additional arguments to pass to `workerd`. Typically used to pass `--experimental`. + generate_default_variant: If True (default), generate the default variant with oldest compat + date. + generate_all_autogates_variant: If True (default), generate @all-autogates variants. + generate_all_compat_flags_variant: If True (default), generate @all-compat-flags variants. + compat_date: If specified, use this compat date for the default variant instead of + 2000-01-01. Does not affect the @all-compat-flags variant which always uses 2999-12-31. + sidecar: If set, an executable that is run in parallel with the test, and provides some + functionality needed for the test. This is usually a backend server, with workerd serving + as the client. The sidecar will be killed once the test completes. + sidecar_port_bindings: A list of binding names which will be filled in with random port + numbers that the sidecar and test can use for communication. The test will only begin once + the sidecar is listening to these ports. In the sidecar, access these bindings as + environment variables. In the wd-test file, add fromEnvironment bindings to expose them to + the test. Reminder: you'll also need to add a network = ( allow = ["private"] ) service. + sidecar_randomize_ip: If true (default), a random IP address will be assigned to the sidecar + process, and provided in the environment variable SIDECAR_HOSTNAME. + load_snapshot: If specified, a label to a snapshot file to load. + env: Environment variables to set when running the test. The following test variants are generated based on the flags: - - name@ (if generate_default_variant): oldest compat date (2000-01-01) - - name@all-compat-flags (if generate_all_compat_flags_variant): newest compat date (2999-12-31) - - name@all-autogates (if generate_all_autogates_variant): all autogates + oldest compat date + - name@ (if generate_default_variant): oldest compat date (2000-01-01) + - name@all-compat-flags (if generate_all_compat_flags_variant): newest compat date + (2999-12-31) + - name@all-autogates (if generate_all_autogates_variant): all autogates + oldest compat date """ - # Add workerd binary to "data" dependencies. - data = data + [src, "//src/workerd/server:workerd_cross"] - - ts_srcs = [src for src in data if src.endswith(".ts")] + # Sidecar and python_snapshot_test cannot be used together due to complexity. + # TODO(later): Implement support for combining these two options if we need it. + if sidecar and python_snapshot_test: + fail("sidecar and python_snapshot_test cannot be used together") # Default name based on src. if name == None: name = src.removesuffix(".capnp").removesuffix(".wd-test").removesuffix(".ts-wd-test") - if len(ts_srcs) != 0: - # Generated declarations are currently not being used, but required based on https://github.com/aspect-build/rules_ts/issues/719 - # TODO(build perf): Consider adopting isolated_typecheck to avoid bottlebecks in TS - # compilation, see https://github.com/aspect-build/rules_ts/blob/f1b7b83/docs/performance.md#isolated-typecheck. - # This will require extensive refactoring and we may only want to enable it for some - # targets, but might be useful if we end up transpiling more code later on. + # Compile TypeScript files. + # Generated declarations are currently not being used, but required based on + # https://github.com/aspect-build/rules_ts/issues/719 + # TODO(build perf): Consider adopting isolated_typecheck to avoid bottlenecks in TS + # compilation, see https://github.com/aspect-build/rules_ts/blob/f1b7b83/docs/performance.md#isolated-typecheck. + # This will require extensive refactoring and we may only want to enable it for some + # targets, but might be useful if we end up transpiling more code later on. + ts_srcs = [s for s in data if s.endswith(".ts")] + if ts_srcs: ts_config( name = name + "@ts_config", src = "tsconfig.json", @@ -62,7 +86,10 @@ def wd_test( declaration = True, deps = ["//src/node:node@tsproject"] + ts_deps, ) - data += [js_src.removesuffix(".ts") + ".js" for js_src in ts_srcs] + data = data + [s.removesuffix(".ts") + ".js" for s in ts_srcs] + + # Add workerd binary and source file to data dependencies. + data = data + [src, "//src/workerd/server:workerd_cross"] # Add initial arguments for `workerd test` command. base_args = [ @@ -71,263 +98,61 @@ def wd_test( "$(location {})".format(src), ] + args - # Define the compat-date args for each variant - # Note: dates must be in range [2000-01-01, 2999-12-31] due to parsing constraints - default_compat_args = ["--compat-date=2000-01-01"] - newest_compat_args = ["--compat-date=2999-12-31"] - - if compat_date: - default_compat_args = ["--compat-date={}".format(compat_date)] - - # Generate variants based on the flags - # Default variant: oldest compat date + # Build environment variables for the test. + test_env = dict(env) + + # Sidecar configuration. + # For detailed documentation, see src/workerd/api/node/tests/sidecar-supervisor.mjs + if sidecar: + supervisor = "//src/workerd/api/node/tests:sidecar-supervisor" + data = data + [sidecar, supervisor] + test_env.update({ + "SIDECAR_COMMAND": "$(location {})".format(sidecar), + "SIDECAR_SUPERVISOR": "$(location {})".format(supervisor), + "PORTS_TO_ASSIGN": ",".join(sidecar_port_bindings), + "RANDOMIZE_IP": "true" if sidecar_randomize_ip else "false", + }) + + # Python snapshot test configuration. + # We need variants of the test for Python memory snapshot tests. We have to invoke + # workerd twice, once with --python-save-snapshot to produce the snapshot and once with + # --python-load-snapshot to use it. + # + # We would like to implement this in py_wd_test and not have to complicate wd_test for it, but + # unfortunately bazel provides no way for a test to create a file that is used by another test. + # So we cannot do this with two separate `wd_test` rules. We _could_ use a build step to create + # the snapshot, but then a failure at this stage would be reported as a build failure when + # really it should count as a test failure. So the only option left is to make this + # modification to wd_test to invoke workerd twice for snapshot tests. + if python_snapshot_test: + test_env["PYTHON_SNAPSHOT_TEST"] = "1" + test_env["PYTHON_SAVE_SNAPSHOT_ARGS"] = "" + + if load_snapshot: + data = data + [load_snapshot] + if python_snapshot_test: + test_env["PYTHON_SAVE_SNAPSHOT_ARGS"] = "--python-load-snapshot load_snapshot.bin" + + # Define the compat-date args for each variant. + # Note: dates must be in range [2000-01-01, 2999-12-31] due to parsing constraints. + default_compat = ["--compat-date={}".format(compat_date or "2000-01-01")] + newest_compat = ["--compat-date=2999-12-31"] + + # Generate test variants based on the flags. + variants = [] if generate_default_variant: - _wd_test( - src = src, - name = name + "@", - data = data, - args = base_args + default_compat_args, - python_snapshot_test = python_snapshot_test, - **kwargs - ) - - # All compat flags variant: newest compat date + variants.append((name + "@", default_compat)) if generate_all_compat_flags_variant: - _wd_test( - src = src, - name = name + "@all-compat-flags", - data = data, - args = base_args + newest_compat_args, - python_snapshot_test = python_snapshot_test, - **kwargs - ) - - # All autogates variant: all autogates + oldest compat date + variants.append((name + "@all-compat-flags", newest_compat)) if generate_all_autogates_variant: - _wd_test( - src = src, - name = name + "@all-autogates", + variants.append((name + "@all-autogates", default_compat + ["--all-autogates"])) + + for variant_name, extra_args in variants: + sh_test( + name = variant_name, + srcs = ["//build/fixtures:wd_test.sh"], data = data, - args = base_args + default_compat_args + ["--all-autogates"], - python_snapshot_test = python_snapshot_test, + args = base_args + extra_args, + env = test_env, **kwargs ) - -WINDOWS_TEMPLATE = """ -@echo off -setlocal EnableDelayedExpansion - -REM Run supervisor to start sidecar if specified -if not "{sidecar}" == "" ( - REM These environment variables are processed by the supervisor executable - set PORTS_TO_ASSIGN={port_bindings} - set RANDOMIZE_IP={randomize_ip} - set SIDECAR_COMMAND="{sidecar}" - powershell -Command \"{supervisor} {runtest}\" -) else ( - {runtest} -) - -set TEST_EXIT=!ERRORLEVEL! -exit /b !TEST_EXIT! -""" - -SH_TEMPLATE = """#!/bin/sh -set -e - -# Set up coverage for workerd subprocess -if [ -n "$COVERAGE_DIR" ]; then - export LLVM_PROFILE_FILE="$COVERAGE_DIR/%p.profraw" - export KJ_CLEAN_SHUTDOWN=1 -fi - -# Run supervisor to start sidecar if specified -if [ ! -z "{sidecar}" ]; then - # These environment variables are processed by the supervisor executable - PORTS_TO_ASSIGN={port_bindings} RANDOMIZE_IP={randomize_ip} SIDECAR_COMMAND="{sidecar}" {supervisor} {runtest} -else - {runtest} -fi -""" - -WINDOWS_RUNTEST_NORMAL = """powershell -Command \"%*\" `-dTEST_TMPDIR=$ENV:TEST_TMPDIR""" - -SH_RUNTEST_NORMAL = """"$@" -dTEST_TMPDIR=$TEST_TMPDIR""" - -# We need variants of the RUN_TEST command for Python memory snapshot tests. We have to invoke -# workerd twice, once with --python-save-snapshot to produce the snapshot and once with -# --python-load-snapshot to use it. -# -# We would like to implement this in py_wd_test and not have to complicate wd_test for it, but -# unfortunately bazel provides no way for a test to create a file that is used by another test. So -# we cannot do this with two separate `wd_test` rules. We _could_ use a build step to create the -# snapshot, but then a failure at this stage would be reported as a build failure when really it -# should count as a test failure. So the only option left is to make this modification to wd_test to -# invoke workerd twice for snapshot tests. - -WINDOWS_RUNTEST_SNAPSHOT = """ -$PYTHON_SAVE_SNAPSHOT_OPTIONS = $ENV:PYTHON_SAVE_SNAPSHOT_ARGS -split ' ' -powershell -Command \"%* --python-save-snapshot @PYTHON_SAVE_SNAPSHOT_OPTIONS\" `-dTEST_TMPDIR=$ENV:TEST_TMPDIR -set TEST_EXIT=!ERRORLEVEL! -if !TEST_EXIT! EQU 0 ( - powershell -Command \"%* --python-load-snapshot snapshot.bin\" `-dTEST_TMPDIR=$ENV:TEST_TMPDIR - set TEST_EXIT=!ERRORLEVEL! -) -""" - -SH_RUNTEST_SNAPSHOT = """ -echo Creating Python Snapshot -"$@" -dTEST_TMPDIR=$TEST_TMPDIR --python-save-snapshot $PYTHON_SAVE_SNAPSHOT_ARGS -echo "" -echo Using Python Snapshot -"$@" -dTEST_TMPDIR=$TEST_TMPDIR --python-load-snapshot snapshot.bin -""" - -def _wd_test_impl(ctx): - is_windows = ctx.target_platform_has_constraint(ctx.attr._platforms_os_windows[platform_common.ConstraintValueInfo]) - - if ctx.file.sidecar and ctx.attr.python_snapshot_test: - # TODO(later): Implement support for generating a combined script with these two options - # if we have a test that requires it. Not doing it for now due to complexity. - return print("sidecar and python_snapshot_test currently cannot be used together") - - # Bazel insists that the rule must actually create the executable that it intends to run; it - # can't just specify some other executable with some args. OK, fine, we'll use a script that - # just execs its args. - if is_windows: - # Batch script executables must end with ".bat" - executable = ctx.actions.declare_file("%s_wd_test.bat" % ctx.label.name) - template = WINDOWS_TEMPLATE - runtest = WINDOWS_RUNTEST_SNAPSHOT if ctx.attr.python_snapshot_test else WINDOWS_RUNTEST_NORMAL - else: - executable = ctx.outputs.executable - template = SH_TEMPLATE - runtest = SH_RUNTEST_SNAPSHOT if ctx.attr.python_snapshot_test else SH_RUNTEST_NORMAL - - content = template.format( - sidecar = ctx.file.sidecar.short_path if ctx.file.sidecar else "", - runtest = runtest, - supervisor = ctx.file.sidecar_supervisor.short_path if ctx.file.sidecar_supervisor else "", - port_bindings = ",".join(ctx.attr.sidecar_port_bindings), - randomize_ip = "true" if ctx.attr.sidecar_randomize_ip else "false", - ) - - ctx.actions.write( - output = executable, - content = content, - is_executable = True, - ) - - runfiles = ctx.runfiles(files = ctx.files.data) - if ctx.file.sidecar: - runfiles = runfiles.merge(ctx.runfiles(files = [ctx.file.sidecar])) - - # Also merge the sidecar's own runfiles if it has any - default_runfiles = ctx.attr.sidecar[DefaultInfo].default_runfiles - if default_runfiles: - runfiles = runfiles.merge(default_runfiles) - - runfiles = runfiles.merge(ctx.runfiles(files = [ctx.file.sidecar_supervisor])) - - # Also merge the supervisor's own runfiles if it has any - default_runfiles = ctx.attr.sidecar_supervisor[DefaultInfo].default_runfiles - if default_runfiles: - runfiles = runfiles.merge(default_runfiles) - - # IMPORTANT: The workerd binary must be listed in dependency_attributes - # to ensure its transitive dependencies (all the C++ source files) are - # included in the coverage instrumentation. Without this, coverage data - # won't be collected for the actual workerd implementation code. - instrumented_files_info = coverage_common.instrumented_files_info( - ctx, - source_attributes = ["src", "data"], - dependency_attributes = ["workerd", "sidecar", "sidecar_supervisor"], - # Include all file types that might contain testable code - extensions = ["cc", "c++", "cpp", "cxx", "c", "h", "hh", "hpp", "hxx", "inc", "js", "ts", "mjs", "wd-test", "capnp"], - ) - environment = dict(ctx.attr.env) - if ctx.attr.python_snapshot_test: - environment["PYTHON_SAVE_SNAPSHOT_ARGS"] = "" - if ctx.attr.load_snapshot: - if ctx.attr.python_snapshot_test: - environment["PYTHON_SAVE_SNAPSHOT_ARGS"] = "--python-load-snapshot load_snapshot.bin" - f = ctx.attr.load_snapshot.files.to_list()[0] - runfiles = runfiles.merge(ctx.runfiles(symlinks = {"load_snapshot.bin": f})) - - return [ - DefaultInfo( - executable = executable, - runfiles = runfiles, - ), - RunEnvironmentInfo( - environment = environment, - ), - instrumented_files_info, - ] - -_wd_test = rule( - implementation = _wd_test_impl, - test = True, - attrs = { - # Implicit dependencies used by Bazel to generate coverage reports. - "_lcov_merger": attr.label( - default = configuration_field(fragment = "coverage", name = "output_generator"), - executable = True, - cfg = config.exec(exec_group = "test"), - ), - "_collect_cc_coverage": attr.label( - default = "@bazel_tools//tools/test:collect_cc_coverage", - executable = True, - cfg = config.exec(exec_group = "test"), - ), - # Source file - "src": attr.label(allow_single_file = True), - # The workerd executable is used to run all tests - "workerd": attr.label( - allow_single_file = True, - executable = True, - cfg = "exec", - default = "//src/workerd/server:workerd_cross", - ), - # A list of files that this test requires to be present in order to run. - "data": attr.label_list(allow_files = True), - # If set, an executable that is run in parallel with the test, and provides some functionality - # needed for the test. This is usually a backend server, with workerd serving as the client. - # The sidecar will be killed once the test completes. - "sidecar": attr.label( - allow_single_file = True, - executable = True, - cfg = "exec", - ), - # Sidecars - # --------- - # For detailed documentation, see src/workerd/api/node/tests/sidecar-supervisor.mjs - - # A list of binding names which will be filled in with random port numbers that the sidecar - # and test can use for communication. The test will only begin once the sidecar is - # listening to these ports. - # - # In the sidecar, access these bindings as environment variables. In the wd-test file, add - # fromEnvironment bindings to expose them to the test. - # - # Reminder: you'll also need to add a network = ( allow = ["private"] ) service as well. - "sidecar_port_bindings": attr.string_list(), - # If true, a random IP address will be assigned to the sidecar process, and provided in the - # environment variable SIDECAR_HOSTNAME, - "sidecar_randomize_ip": attr.bool(default = True), - # An executable that is used to manage port assignments and child process creation when a - # sidecar is specified. - "sidecar_supervisor": attr.label( - allow_single_file = True, - executable = True, - cfg = "exec", - default = "//src/workerd/api/node/tests:sidecar-supervisor", - ), - "python_snapshot_test": attr.bool(), - "load_snapshot": attr.label(allow_single_file = True), - # Environment variables to set when running the test - "env": attr.string_dict(), - # A reference to the Windows platform label, needed for the implementation of wd_test - "_platforms_os_windows": attr.label(default = "@platforms//os:windows"), - }, -)