[nightly] Update common Docker engineering infrastructure with latest

eng/docker-tools/CHANGELOG.md

@@ -4,6 +4,36 @@ All breaking changes and new features in `eng/docker-tools` will be documented i
 
 ---
 
+## 2026-03-12: Service connection OIDC changes
+
+- Pull request: [#2013](https://github.com/dotnet/docker-tools/pull/2013)
+- Issue: [#2012](https://github.com/dotnet/docker-tools/issues/2012)
+
+`setup-service-connections.yml` has been removed. Azure DevOps no longer
+issues OIDC tokens for service connections referenced in a separate stage.
+Service connections are now referenced per-job via
+`reference-service-connections.yml`.
+
+**How to update:**
+
+- Remove any `serviceConnections` parameters passed to `1es-official.yml` or
+  `1es-unofficial.yml` - they are no longer accepted.
+- Remove any calls to `setup-service-connections.yml` from stage templates.
+- Non-registry service connections (e.g., kusto, marStatus) should be passed
+  via `additionalServiceConnections` to the job templates that need them.
+
+---
+
+## 2026-03-04: Pre-build validation gated by `preBuildTestScriptPath` variable
+
+The `PreBuildValidation` job condition now checks the new `preBuildTestScriptPath` variable instead of `testScriptPath`.
+This allows repos to independently control whether pre-build validation runs, without affecting functional tests.
+
+The new variable defaults to `$(testScriptPath)`, so existing repos that have pre-build tests are not affected.
+Repos that do not have pre-build tests can set `preBuildTestScriptPath` to `""` to skip the job entirely.
+
+---
+
 ## 2026-02-19: Separate Registry Endpoints from Authentication
 
 - Pull request: [#1945](https://github.com/dotnet/docker-tools/pull/1945)

eng/docker-tools/DEV-GUIDE.md

@@ -180,14 +180,16 @@ The `stages` variable is a comma-separated string that controls which pipeline s
 ```yaml
 variables:
 - name: stages
-  value: "build,test,publish"    # Run all stages
+  value: "build,test,sign,publish"    # Run all stages
 ```
 
 Common patterns:
-- `"build"` - Build only, no tests or publishing
-- `"build,test"` - Build and test, but don't publish
+- `"build"` - Build only, no tests, signing, or publishing
+- `"build,test"` - Build and test, but don't sign or publish
+- `"build,test,sign"` - Build, test, and sign, but don't publish
+- `"sign"` - Sign only (when re-running failed signing from a previous build)
 - `"publish"` - Publish only (when re-running a failed publish from a previous build)
-- `"build,test,publish"` - Full pipeline
+- `"build,test,sign,publish"` - Full pipeline
 
 **Note:** The `Post_Build` stage is implicitly included whenever `build` is in the stages list. You don't need to specify it separately—it automatically runs after Build to merge image info files and consolidate SBOMs.
 
@@ -372,11 +374,13 @@ Note: For simple retries of failed jobs, use the Azure Pipelines UI "Re-run fail
 
 | Scenario | stages | sourceBuildPipelineRunId |
 |----------|--------|--------------------------|
-| Normal full build | `"build,test,publish"` | `$(Build.BuildId)` (default) |
+| Normal full build | `"build,test,sign,publish"` | `$(Build.BuildId)` (default) |
 | Re-run publish after infra fix | `"publish"` | ID of the successful build run |
 | Re-test after infra fix | `"test"` | ID of the build run to test |
+| Re-sign after infra fix | `"sign"` | ID of the build run to sign |
 | Build only (no publish) | `"build"` | `$(Build.BuildId)` (default) |
 | Test + publish (skip build) | `"test,publish"` | ID of the build run |
+| Sign + publish (skip build/test) | `"sign,publish"` | ID of the build run |
 
 **In the Azure DevOps UI:**
 

eng/docker-tools/templates/1es-official.yml

@@ -17,9 +17,6 @@ parameters:
 - name: stages
   type: stageList
   default: []
-- name: serviceConnections
-  type: object
-  default: []
 - name: pool
   type: object
   default:
@@ -62,9 +59,4 @@ extends:
         tsa:
           enabled: true
     stages:
-    - ${{ if gt(length(parameters.serviceConnections), 0) }}:
-      - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self
-        parameters:
-          pool: ${{ parameters.pool }}
-          serviceConnections: ${{ parameters.serviceConnections }}
     - ${{ parameters.stages }}

eng/docker-tools/templates/1es-unofficial.yml

@@ -19,10 +19,6 @@ parameters:
 - name: stages
   type: stageList
   default: []
-  # 1ES Pipeline Template parameters
-- name: serviceConnections
-  type: object
-  default: []
 - name: pool
   type: object
   default:
@@ -71,9 +67,4 @@ extends:
         tsa:
           enabled: true
     stages:
-    - ${{ if gt(length(parameters.serviceConnections), 0) }}:
-      - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self
-        parameters:
-          pool: ${{ parameters.pool }}
-          serviceConnections: ${{ parameters.serviceConnections }}
     - ${{ parameters.stages }}

eng/docker-tools/templates/jobs/build-images.yml

@@ -42,6 +42,15 @@ jobs:
       cleanupDocker: true
       customInitSteps: ${{ parameters.customInitSteps }}
   - ${{ parameters.customBuildInitSteps }}
+  - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self
+    parameters:
+      publishConfig: ${{ parameters.publishConfig }}
+      dockerClientOS: ${{ parameters.dockerClientOS }}
+      usesRegistries:
+        - ${{ parameters.publishConfig.BuildRegistry.server }}
+      ${{ if parameters.storageAccountServiceConnection }}:
+        serviceConnections:
+        - name: ${{ parameters.storageAccountServiceConnection.name }}
   - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self
     parameters:
       publicSourceBranch: $(publicSourceBranch)

eng/docker-tools/templates/jobs/copy-base-images.yml

@@ -43,6 +43,11 @@ jobs:
       publishConfig: ${{ parameters.publishConfig }}
       customInitSteps: ${{ parameters.customInitSteps }}
       versionsRepoRef: ${{ parameters.versionsRepoRef }}
+  - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self
+    parameters:
+      publishConfig: ${{ parameters.publishConfig }}
+      usesRegistries:
+        - ${{ parameters.acr.server }}
   - ${{ parameters.customCopyBaseImagesInitSteps }}
   - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self
     parameters:

eng/docker-tools/templates/jobs/post-build.yml

@@ -3,6 +3,7 @@ parameters:
   internalProjectName: null
   publicProjectName: null
   customInitSteps: []
+  publishConfig: null
 
 jobs:
 - job: Build
@@ -18,6 +19,7 @@ jobs:
     parameters:
       dockerClientOS: linux
       customInitSteps: ${{ parameters.customInitSteps }}
+      publishConfig: ${{ parameters.publishConfig }}
   - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self
     parameters:
       targetPath: $(Build.ArtifactStagingDirectory)

eng/docker-tools/templates/jobs/publish.yml

@@ -12,6 +12,9 @@ parameters:
   # When true, overrides the commit SHA in merged image info files to use the current repository commit.
   # This ensures that updated images reference the correct commit in their commitUrl properties.
   overrideImageInfoCommit: false
+  # Service connections not in publishConfig.RegistryAuthentication that need OIDC
+  # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }]
+  additionalServiceConnections: []
 
 jobs:
 - job: Publish
@@ -53,6 +56,14 @@ jobs:
       versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customInitSteps }}
 
+  - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self
+    parameters:
+      publishConfig: ${{ parameters.publishConfig }}
+      usesRegistries:
+        - ${{ parameters.publishConfig.BuildRegistry.server }}
+        - ${{ parameters.publishConfig.PublishRegistry.server }}
+      serviceConnections: ${{ parameters.additionalServiceConnections }}
+
   - template: /eng/docker-tools/templates/steps/retain-build.yml@self
 
   - pwsh: |

eng/docker-tools/templates/jobs/sign-images.yml

@@ -0,0 +1,64 @@
+# Signs container images using ESRP/Notary v2.
+# This job downloads the merged image-info artifact and signs all images listed in it.
+parameters:
+  pool: {}
+  internalProjectName: null
+  publicProjectName: null
+  customInitSteps: []
+  publishConfig: null
+  sourceBuildPipelineRunId: ""
+
+jobs:
+- job: Sign
+  pool: ${{ parameters.pool }}
+  variables:
+    imageInfoDir: $(Build.ArtifactStagingDirectory)/image-info
+  steps:
+
+  # Install MicroBuild signing plugin for ESRP container image signing
+  - template: /eng/docker-tools/templates/steps/init-signing-linux.yml@self
+    parameters:
+      signType: ${{ parameters.publishConfig.Signing.SignType }}
+      envFileVariableName: signingEnvFilePath
+
+  # Setup docker and ImageBuilder
+  - template: /eng/docker-tools/templates/steps/init-common.yml@self
+    parameters:
+      dockerClientOS: linux
+      setupImageBuilder: true
+      customInitSteps: ${{ parameters.customInitSteps }}
+      publishConfig: ${{ parameters.publishConfig }}
+      envFilePath: $(signingEnvFilePath)
+
+  - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self
+    parameters:
+      publishConfig: ${{ parameters.publishConfig }}
+      usesRegistries:
+        - ${{ parameters.publishConfig.BuildRegistry.server }}
+
+  # Download merged image-info artifact from Post_Build stage (or from a previous pipeline run)
+  - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self
+    parameters:
+      targetPath: $(imageInfoDir)
+      artifactName: image-info
+      pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+
+  - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self
+    parameters:
+      displayName: 🔏 Sign Container Images
+      internalProjectName: ${{ parameters.internalProjectName }}
+      args: >-
+        signImages
+        $(artifactsPath)/image-info/image-info.json
+        --registry-override ${{ parameters.publishConfig.BuildRegistry.server }}
+        --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }}
+
+  - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self
+    parameters:
+      displayName: ✅ Verify Container Image Signatures
+      internalProjectName: ${{ parameters.internalProjectName }}
+      args: >-
+        verifySignatures
+        $(artifactsPath)/image-info/image-info.json
+        --registry-override ${{ parameters.publishConfig.BuildRegistry.server }}
+        --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }}

eng/docker-tools/templates/jobs/test-images-linux-client.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix: $[ ${{ parameters.matrix }} ]
   ${{ if eq(parameters.preBuildValidation, 'true') }}:
-    condition: and(succeeded(), ne(variables.testScriptPath, ''))
+    condition: and(succeeded(), ne(variables.preBuildTestScriptPath, ''))
   pool: ${{ parameters.pool }}
   timeoutInMinutes: ${{ parameters.testJobTimeout }}
   steps:

eng/docker-tools/templates/stages/build-and-test.yml

@@ -220,6 +220,33 @@ stages:
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
       customInitSteps: ${{ parameters.customInitSteps }}
+      publishConfig: ${{ parameters.publishConfig }}
+
+################################################################################
+# Sign Images
+################################################################################
+- ${{ if eq(parameters.publishConfig.Signing.Enabled, true) }}:
+  - stage: Sign
+    dependsOn: Post_Build
+    condition: "
+      and(
+        ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'),
+        and(
+          contains(variables['stages'], 'sign'),
+          or(
+            and(
+              succeeded(),
+              contains(variables['stages'], 'build')),
+            not(contains(variables['stages'], 'build')))))"
+    jobs:
+    - template: /eng/docker-tools/templates/jobs/sign-images.yml@self
+      parameters:
+        pool: ${{ parameters.linuxAmd64Pool }}
+        internalProjectName: ${{ parameters.internalProjectName }}
+        publicProjectName: ${{ parameters.publicProjectName }}
+        customInitSteps: ${{ parameters.customInitSteps }}
+        publishConfig: ${{ parameters.publishConfig }}
+        sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
 
 ################################################################################
 # Test Images

eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml

@@ -32,6 +32,10 @@ parameters:
   # Publish parameters
   customPublishInitSteps: []
 
+  # Additional service connections not in publishConfig.RegistryAuthentication
+  # that need OIDC token access (e.g., kusto, marStatus). Shape: [{ name: string }]
+  additionalServiceConnections: []
+
   # Other common parameters
   internalProjectName: null
   publicProjectName: null
@@ -75,5 +79,6 @@ stages:
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
     publishConfig: ${{ parameters.publishConfig }}
+    additionalServiceConnections: ${{ parameters.additionalServiceConnections }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
     versionsRepoRef: ${{ parameters.versionsRepoRef }}

eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml

@@ -38,6 +38,11 @@ parameters:
   type: object
   default: {}
 
+# Enable container image signing
+- name: enableSigning
+  type: boolean
+  default: false
+
 
 stages:
 - template: ${{ parameters.stagesTemplate }}
@@ -103,3 +108,12 @@ stages:
         id: $(test-nonprod.serviceConnection.id)
         clientId: $(test-nonprod.serviceConnection.clientId)
         tenantId: $(testTenant)
+
+      Signing:
+        Enabled: ${{ parameters.enableSigning }}
+        ImageSigningKeyCode: $(microBuildSigningKeyCode.testing)
+        ReferrerSigningKeyCode: $(microBuildSigningKeyCode.testing)
+        # Use signType 'real' even for non-prod to actually sign with the test certificate.
+        # The 'test' signType skips signing entirely on linux; the test keycode provides a non-production certificate.
+        SignType: real
+        TrustStoreName: test

eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml

@@ -38,6 +38,11 @@ parameters:
   type: object
   default: {}
 
+# Enable container image signing
+- name: enableSigning
+  type: boolean
+  default: false
+
 
 stages:
 - template: ${{ parameters.stagesTemplate }}
@@ -103,3 +108,10 @@ stages:
         id: $(test.serviceConnection.id)
         clientId: $(test.serviceConnection.clientId)
         tenantId: $(test.serviceConnection.tenantId)
+
+      Signing:
+        Enabled: ${{ parameters.enableSigning }}
+        ImageSigningKeyCode: $(microBuildSigningKeyCode.containers)
+        ReferrerSigningKeyCode: $(microBuildSigningKeyCode.attestations)
+        SignType: real
+        TrustStoreName: supplychain

eng/docker-tools/templates/stages/dotnet/publish.yml

@@ -13,6 +13,9 @@ parameters:
   sourceBuildPipelineRunId: ''
   versionsRepoRef: null
   overrideImageInfoCommit: false
+  # Service connections not in publishConfig.RegistryAuthentication that need OIDC
+  # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }]
+  additionalServiceConnections: []
 
 stages:
 - template: /eng/docker-tools/templates/stages/publish.yml@self
@@ -22,6 +25,7 @@ stages:
     publishConfig: ${{ parameters.publishConfig }}
     isStandalonePublish: ${{ parameters.isStandalonePublish }}
     customInitSteps: ${{ parameters.customInitSteps }}
+    additionalServiceConnections: ${{ parameters.additionalServiceConnections }}
     sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
     versionsRepoRef: ${{ parameters.versionsRepoRef }}