diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md index 7903acd810..1d8cf40040 100644 --- a/eng/docker-tools/CHANGELOG.md +++ b/eng/docker-tools/CHANGELOG.md @@ -4,6 +4,36 @@ All breaking changes and new features in `eng/docker-tools` will be documented i --- +## 2026-03-12: Service connection OIDC changes + +- Pull request: [#2013](https://github.com/dotnet/docker-tools/pull/2013) +- Issue: [#2012](https://github.com/dotnet/docker-tools/issues/2012) + +`setup-service-connections.yml` has been removed. Azure DevOps no longer +issues OIDC tokens for service connections referenced in a separate stage. +Service connections are now referenced per-job via +`reference-service-connections.yml`. + +**How to update:** + +- Remove any `serviceConnections` parameters passed to `1es-official.yml` or + `1es-unofficial.yml` - they are no longer accepted. +- Remove any calls to `setup-service-connections.yml` from stage templates. +- Non-registry service connections (e.g., kusto, marStatus) should be passed + via `additionalServiceConnections` to the job templates that need them. + +--- + +## 2026-03-04: Pre-build validation gated by `preBuildTestScriptPath` variable + +The `PreBuildValidation` job condition now checks the new `preBuildTestScriptPath` variable instead of `testScriptPath`. +This allows repos to independently control whether pre-build validation runs, without affecting functional tests. + +The new variable defaults to `$(testScriptPath)`, so existing repos that have pre-build tests are not affected. +Repos that do not have pre-build tests can set `preBuildTestScriptPath` to `""` to skip the job entirely. + +--- + ## 2026-02-19: Separate Registry Endpoints from Authentication - Pull request: [#1945](https://github.com/dotnet/docker-tools/pull/1945) diff --git a/eng/docker-tools/DEV-GUIDE.md b/eng/docker-tools/DEV-GUIDE.md index ea0301f2d3..c1d48434b0 100644 --- a/eng/docker-tools/DEV-GUIDE.md +++ b/eng/docker-tools/DEV-GUIDE.md @@ -180,14 +180,16 @@ The `stages` variable is a comma-separated string that controls which pipeline s ```yaml variables: - name: stages - value: "build,test,publish" # Run all stages + value: "build,test,sign,publish" # Run all stages ``` Common patterns: -- `"build"` - Build only, no tests or publishing -- `"build,test"` - Build and test, but don't publish +- `"build"` - Build only, no tests, signing, or publishing +- `"build,test"` - Build and test, but don't sign or publish +- `"build,test,sign"` - Build, test, and sign, but don't publish +- `"sign"` - Sign only (when re-running failed signing from a previous build) - `"publish"` - Publish only (when re-running a failed publish from a previous build) -- `"build,test,publish"` - Full pipeline +- `"build,test,sign,publish"` - Full pipeline **Note:** The `Post_Build` stage is implicitly included whenever `build` is in the stages list. You don't need to specify it separately—it automatically runs after Build to merge image info files and consolidate SBOMs. @@ -372,11 +374,13 @@ Note: For simple retries of failed jobs, use the Azure Pipelines UI "Re-run fail | Scenario | stages | sourceBuildPipelineRunId | |----------|--------|--------------------------| -| Normal full build | `"build,test,publish"` | `$(Build.BuildId)` (default) | +| Normal full build | `"build,test,sign,publish"` | `$(Build.BuildId)` (default) | | Re-run publish after infra fix | `"publish"` | ID of the successful build run | | Re-test after infra fix | `"test"` | ID of the build run to test | +| Re-sign after infra fix | `"sign"` | ID of the build run to sign | | Build only (no publish) | `"build"` | `$(Build.BuildId)` (default) | | Test + publish (skip build) | `"test,publish"` | ID of the build run | +| Sign + publish (skip build/test) | `"sign,publish"` | ID of the build run | **In the Azure DevOps UI:** diff --git a/eng/docker-tools/templates/1es-official.yml b/eng/docker-tools/templates/1es-official.yml index ebf8fcd705..8eb33d656b 100644 --- a/eng/docker-tools/templates/1es-official.yml +++ b/eng/docker-tools/templates/1es-official.yml @@ -17,9 +17,6 @@ parameters: - name: stages type: stageList default: [] -- name: serviceConnections - type: object - default: [] - name: pool type: object default: @@ -62,9 +59,4 @@ extends: tsa: enabled: true stages: - - ${{ if gt(length(parameters.serviceConnections), 0) }}: - - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self - parameters: - pool: ${{ parameters.pool }} - serviceConnections: ${{ parameters.serviceConnections }} - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/1es-unofficial.yml b/eng/docker-tools/templates/1es-unofficial.yml index bc584cd381..c4a849c53a 100644 --- a/eng/docker-tools/templates/1es-unofficial.yml +++ b/eng/docker-tools/templates/1es-unofficial.yml @@ -19,10 +19,6 @@ parameters: - name: stages type: stageList default: [] - # 1ES Pipeline Template parameters -- name: serviceConnections - type: object - default: [] - name: pool type: object default: @@ -71,9 +67,4 @@ extends: tsa: enabled: true stages: - - ${{ if gt(length(parameters.serviceConnections), 0) }}: - - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self - parameters: - pool: ${{ parameters.pool }} - serviceConnections: ${{ parameters.serviceConnections }} - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml index 025d3c8d00..75264104ac 100644 --- a/eng/docker-tools/templates/jobs/build-images.yml +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -42,6 +42,15 @@ jobs: cleanupDocker: true customInitSteps: ${{ parameters.customInitSteps }} - ${{ parameters.customBuildInitSteps }} + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + dockerClientOS: ${{ parameters.dockerClientOS }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + ${{ if parameters.storageAccountServiceConnection }}: + serviceConnections: + - name: ${{ parameters.storageAccountServiceConnection.name }} - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self parameters: publicSourceBranch: $(publicSourceBranch) diff --git a/eng/docker-tools/templates/jobs/copy-base-images.yml b/eng/docker-tools/templates/jobs/copy-base-images.yml index c435b211ab..4a5ed5e88c 100644 --- a/eng/docker-tools/templates/jobs/copy-base-images.yml +++ b/eng/docker-tools/templates/jobs/copy-base-images.yml @@ -43,6 +43,11 @@ jobs: publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customInitSteps }} versionsRepoRef: ${{ parameters.versionsRepoRef }} + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.acr.server }} - ${{ parameters.customCopyBaseImagesInitSteps }} - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self parameters: diff --git a/eng/docker-tools/templates/jobs/post-build.yml b/eng/docker-tools/templates/jobs/post-build.yml index 3dfb84cf98..8159872aad 100644 --- a/eng/docker-tools/templates/jobs/post-build.yml +++ b/eng/docker-tools/templates/jobs/post-build.yml @@ -3,6 +3,7 @@ parameters: internalProjectName: null publicProjectName: null customInitSteps: [] + publishConfig: null jobs: - job: Build @@ -18,6 +19,7 @@ jobs: parameters: dockerClientOS: linux customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self parameters: targetPath: $(Build.ArtifactStagingDirectory) diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml index b86ec1ee2d..92f98b22f5 100644 --- a/eng/docker-tools/templates/jobs/publish.yml +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -12,6 +12,9 @@ parameters: # When true, overrides the commit SHA in merged image info files to use the current repository commit. # This ensures that updated images reference the correct commit in their commitUrl properties. overrideImageInfoCommit: false + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] jobs: - job: Publish @@ -53,6 +56,14 @@ jobs: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + - ${{ parameters.publishConfig.PublishRegistry.server }} + serviceConnections: ${{ parameters.additionalServiceConnections }} + - template: /eng/docker-tools/templates/steps/retain-build.yml@self - pwsh: | diff --git a/eng/docker-tools/templates/jobs/sign-images.yml b/eng/docker-tools/templates/jobs/sign-images.yml new file mode 100644 index 0000000000..fcc43f0967 --- /dev/null +++ b/eng/docker-tools/templates/jobs/sign-images.yml @@ -0,0 +1,64 @@ +# Signs container images using ESRP/Notary v2. +# This job downloads the merged image-info artifact and signs all images listed in it. +parameters: + pool: {} + internalProjectName: null + publicProjectName: null + customInitSteps: [] + publishConfig: null + sourceBuildPipelineRunId: "" + +jobs: +- job: Sign + pool: ${{ parameters.pool }} + variables: + imageInfoDir: $(Build.ArtifactStagingDirectory)/image-info + steps: + + # Install MicroBuild signing plugin for ESRP container image signing + - template: /eng/docker-tools/templates/steps/init-signing-linux.yml@self + parameters: + signType: ${{ parameters.publishConfig.Signing.SignType }} + envFileVariableName: signingEnvFilePath + + # Setup docker and ImageBuilder + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux + setupImageBuilder: true + customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + envFilePath: $(signingEnvFilePath) + + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + + # Download merged image-info artifact from Post_Build stage (or from a previous pipeline run) + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(imageInfoDir) + artifactName: image-info + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: 🔏 Sign Container Images + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + signImages + $(artifactsPath)/image-info/image-info.json + --registry-override ${{ parameters.publishConfig.BuildRegistry.server }} + --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: ✅ Verify Container Image Signatures + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + verifySignatures + $(artifactsPath)/image-info/image-info.json + --registry-override ${{ parameters.publishConfig.BuildRegistry.server }} + --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} diff --git a/eng/docker-tools/templates/jobs/test-images-linux-client.yml b/eng/docker-tools/templates/jobs/test-images-linux-client.yml index 44e9e2fb74..450e067f5a 100644 --- a/eng/docker-tools/templates/jobs/test-images-linux-client.yml +++ b/eng/docker-tools/templates/jobs/test-images-linux-client.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: $[ ${{ parameters.matrix }} ] ${{ if eq(parameters.preBuildValidation, 'true') }}: - condition: and(succeeded(), ne(variables.testScriptPath, '')) + condition: and(succeeded(), ne(variables.preBuildTestScriptPath, '')) pool: ${{ parameters.pool }} timeoutInMinutes: ${{ parameters.testJobTimeout }} steps: diff --git a/eng/docker-tools/templates/stages/build-and-test.yml b/eng/docker-tools/templates/stages/build-and-test.yml index 1e39a998ea..d21e8de909 100644 --- a/eng/docker-tools/templates/stages/build-and-test.yml +++ b/eng/docker-tools/templates/stages/build-and-test.yml @@ -220,6 +220,33 @@ stages: internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + +################################################################################ +# Sign Images +################################################################################ +- ${{ if eq(parameters.publishConfig.Signing.Enabled, true) }}: + - stage: Sign + dependsOn: Post_Build + condition: " + and( + ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), + and( + contains(variables['stages'], 'sign'), + or( + and( + succeeded(), + contains(variables['stages'], 'build')), + not(contains(variables['stages'], 'build')))))" + jobs: + - template: /eng/docker-tools/templates/jobs/sign-images.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} ################################################################################ # Test Images diff --git a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml index 13ca93a337..c590a52874 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml @@ -32,6 +32,10 @@ parameters: # Publish parameters customPublishInitSteps: [] + # Additional service connections not in publishConfig.RegistryAuthentication + # that need OIDC token access (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] + # Other common parameters internalProjectName: null publicProjectName: null @@ -75,5 +79,6 @@ stages: internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} publishConfig: ${{ parameters.publishConfig }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml index 6f3e4995d6..0ce87d4d2d 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml @@ -38,6 +38,11 @@ parameters: type: object default: {} +# Enable container image signing +- name: enableSigning + type: boolean + default: false + stages: - template: ${{ parameters.stagesTemplate }} @@ -103,3 +108,12 @@ stages: id: $(test-nonprod.serviceConnection.id) clientId: $(test-nonprod.serviceConnection.clientId) tenantId: $(testTenant) + + Signing: + Enabled: ${{ parameters.enableSigning }} + ImageSigningKeyCode: $(microBuildSigningKeyCode.testing) + ReferrerSigningKeyCode: $(microBuildSigningKeyCode.testing) + # Use signType 'real' even for non-prod to actually sign with the test certificate. + # The 'test' signType skips signing entirely on linux; the test keycode provides a non-production certificate. + SignType: real + TrustStoreName: test diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml index 24746b3aea..f90e5e8064 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml @@ -38,6 +38,11 @@ parameters: type: object default: {} +# Enable container image signing +- name: enableSigning + type: boolean + default: false + stages: - template: ${{ parameters.stagesTemplate }} @@ -103,3 +108,10 @@ stages: id: $(test.serviceConnection.id) clientId: $(test.serviceConnection.clientId) tenantId: $(test.serviceConnection.tenantId) + + Signing: + Enabled: ${{ parameters.enableSigning }} + ImageSigningKeyCode: $(microBuildSigningKeyCode.containers) + ReferrerSigningKeyCode: $(microBuildSigningKeyCode.attestations) + SignType: real + TrustStoreName: supplychain diff --git a/eng/docker-tools/templates/stages/dotnet/publish.yml b/eng/docker-tools/templates/stages/dotnet/publish.yml index ae0325f010..5cac237b95 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish.yml @@ -13,6 +13,9 @@ parameters: sourceBuildPipelineRunId: '' versionsRepoRef: null overrideImageInfoCommit: false + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] stages: - template: /eng/docker-tools/templates/stages/publish.yml@self @@ -22,6 +25,7 @@ stages: publishConfig: ${{ parameters.publishConfig }} isStandalonePublish: ${{ parameters.isStandalonePublish }} customInitSteps: ${{ parameters.customInitSteps }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/docker-tools/templates/stages/publish.yml b/eng/docker-tools/templates/stages/publish.yml index b72d4fd9e2..7195aaa828 100644 --- a/eng/docker-tools/templates/stages/publish.yml +++ b/eng/docker-tools/templates/stages/publish.yml @@ -25,6 +25,10 @@ parameters: # internally built images still reference public Dockerfiles. overrideImageInfoCommit: false + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] + ################################################################################ # Publish Images ################################################################################ @@ -33,39 +37,36 @@ stages: ${{ if eq(parameters.isStandalonePublish, true) }}: dependsOn: [] ${{ else }}: - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - dependsOn: Test - ${{ else }}: - dependsOn: Post_Build + dependsOn: + - ${{ if eq(parameters.publishConfig.Signing.Enabled, true) }}: + - Sign + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - Test + - ${{ else }}: + - Post_Build + # Run when all of the following are true: + # 1. The pipeline has not been canceled. + # 2. The stages variable includes 'publish'. + # 3. Either signing is not enabled, or the Sign stage succeeded. + # 4. Either the stages variable does not include 'build', or Post_Build succeeded. + # 5. Either the stages variable does not include 'test', or Test succeeded/was skipped. condition: " and( not(canceled()), - and( - contains(variables['stages'], 'publish'), - or( - or( - and( - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - or( - and( - not(contains(variables['stages'], 'build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - and( - not(contains(variables['stages'], 'test')), - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build'))))), - not( - or( - contains(variables['stages'], 'build'), - contains(variables['stages'], 'test'))))))" + contains(variables['stages'], 'publish'), + or( + ne(lower('${{ parameters.publishConfig.Signing.Enabled }}'), 'true'), + in(dependencies.Sign.result, 'Succeeded', 'SucceededWithIssues') + ), + or( + not(contains(variables['stages'], 'build')), + succeeded('Post_Build') + ), + or( + not(contains(variables['stages'], 'test')), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped') + ) + )" jobs: - template: /eng/docker-tools/templates/jobs/publish.yml@self parameters: @@ -80,3 +81,4 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} versionsRepoPath: ${{ parameters.versionsRepoPath }} overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} diff --git a/eng/docker-tools/templates/steps/generate-appsettings.yml b/eng/docker-tools/templates/steps/generate-appsettings.yml index 5eaa5a1429..b1243e7038 100644 --- a/eng/docker-tools/templates/steps/generate-appsettings.yml +++ b/eng/docker-tools/templates/steps/generate-appsettings.yml @@ -1,17 +1,31 @@ # .NET Microsoft.Extensions.Configuration reads appsettings.json from the working directory # where ImageBuilder is run. Place it in the repo root so it will be available at runtime. parameters: +# See: +# - publish-config-prod.yml +# - publish-config-nonprod.yml +# - PublishConfiguration.cs - name: publishConfig type: object +# This should be the path to $(Build.ArtifactStagingDirectory). It is parameterized +# here since it is mounted into the ImageBuilder container at runtime. +- name: artifactStagingDirectory + type: string + default: "" - name: condition type: string default: "true" steps: - powershell: |- + # Escape backslashes for JSON compatibility (Windows paths like D:\a\_work become D:\\a\\_work) + $artifactStagingDirectory = "${{ parameters.artifactStagingDirectory }}" -replace '\\', '\\' $appsettingsJsonContent = @" { - "PublishConfiguration": ${{ convertToJson(parameters.publishConfig) }} + "PublishConfiguration": ${{ convertToJson(parameters.publishConfig) }}, + "BuildConfiguration": { + "ArtifactStagingDirectory": "$artifactStagingDirectory" + } } "@ Set-Content -Path "appsettings.json" -Value $appsettingsJsonContent diff --git a/eng/docker-tools/templates/steps/init-common.yml b/eng/docker-tools/templates/steps/init-common.yml index decad7b935..450ddc583f 100644 --- a/eng/docker-tools/templates/steps/init-common.yml +++ b/eng/docker-tools/templates/steps/init-common.yml @@ -53,6 +53,12 @@ parameters: type: string default: "versions" +# Path to an env file for docker --env-file. +# Passed through to init-imagebuilder.yml. +- name: envFilePath + type: string + default: "" + steps: # Repository Checkout # Multi-repo checkout is used when a versions repository is needed for caching. @@ -197,14 +203,14 @@ steps: TargetFolder: '$(Build.SourcesDirectory)' # Artifacts Path Configuration -# Linux: Uses /artifacts inside the container, which is mounted to the host's -# Build.ArtifactStagingDirectory via docker run -v +# Linux: Uses Build.ArtifactStagingDirectory directly so container and host paths match +# through an identity volume mount # Windows: Uses Build.ArtifactStagingDirectory directly since ImageBuilder runs # as an extracted executable, not in a container - ${{ if eq(parameters.dockerClientOS, 'linux') }}: - script: | - echo "Setting artifactsPath to '/artifacts'" - echo "##vso[task.setvariable variable=artifactsPath]/artifacts" + echo "Setting artifactsPath to '$(Build.ArtifactStagingDirectory)'" + echo "##vso[task.setvariable variable=artifactsPath]$(Build.ArtifactStagingDirectory)" displayName: Define Artifacts Path Variable condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if eq(parameters.dockerClientOS, 'windows') }}: @@ -245,3 +251,4 @@ steps: publishConfig: ${{ parameters.publishConfig }} condition: ${{ parameters.condition }} customInitSteps: ${{ parameters.customInitSteps }} + envFilePath: ${{ parameters.envFilePath }} diff --git a/eng/docker-tools/templates/steps/init-imagebuilder.yml b/eng/docker-tools/templates/steps/init-imagebuilder.yml index 2c3f948571..a5488caa59 100644 --- a/eng/docker-tools/templates/steps/init-imagebuilder.yml +++ b/eng/docker-tools/templates/steps/init-imagebuilder.yml @@ -21,6 +21,13 @@ parameters: type: stepList default: [] +# Path to an env file for docker --env-file. +# When set, --env-file is added to the docker run commands so the container +# receives the environment variables defined in the file. +- name: envFilePath + type: string + default: "" + steps: # Custom ImageBuilder setup (e.g., bootstrap from source) - ${{ if gt(length(parameters.customInitSteps), 0) }}: @@ -60,6 +67,7 @@ steps: - template: /eng/docker-tools/templates/steps/generate-appsettings.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} + artifactStagingDirectory: $(artifactsPath) condition: ${{ parameters.condition }} # On Linux, build the "withrepo" image that includes the repo's source code. @@ -96,18 +104,20 @@ steps: "-v $(Build.ArtifactStagingDirectory):$(artifactsPath)" "-w /repo" "$(imageBuilderDockerRunExtraOptions)" - "$(imageNames.imageBuilder.withrepo)" ) + $envFilePath = "${{ parameters.envFilePath }}" + if ($envFilePath) { + $dockerRunArgs += "--env-file $envFilePath" + } + $authedDockerRunArgs = @( - '-e' - 'SYSTEM_ACCESSTOKEN=$env:SYSTEM_ACCESSTOKEN' - '-e' - 'SYSTEM_OIDCREQUESTURI=$env:SYSTEM_OIDCREQUESTURI' + "-e", 'SYSTEM_ACCESSTOKEN' + "-e", 'SYSTEM_OIDCREQUESTURI' ) - $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs - $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs + $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs + @("$(imageNames.imageBuilder.withrepo)") + $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs + @("$(imageNames.imageBuilder.withrepo)") $runImageBuilderCmd = $($dockerRunCmd -join ' ') $runAuthedImageBuilderCmd = $($authedDockerRunCmd -join ' ') diff --git a/eng/docker-tools/templates/steps/init-signing-linux.yml b/eng/docker-tools/templates/steps/init-signing-linux.yml new file mode 100644 index 0000000000..4513bc18a6 --- /dev/null +++ b/eng/docker-tools/templates/steps/init-signing-linux.yml @@ -0,0 +1,124 @@ +# Installs the MicroBuild signing plugin for ESRP container image signing. +# After installation, MBSIGN_APPFOLDER environment variable points to DDSignFiles.dll location. +parameters: +- name: signType + type: string + default: test + values: + - test + - real + +- name: condition + type: string + default: "true" + +- name: microBuildOutputFolder + type: string + default: $(Agent.TempDirectory)/MicroBuild + +# Name of the pipeline variable to set with the signing env file path. +# When set, a signing env file is written after plugin installation and +# the specified pipeline variable is set to its path via logging directive, +# allowing downstream steps to reference it as $(variableName). +# When empty, no env file is created (non-signing jobs). +- name: envFileVariableName + type: string + default: "" + +steps: +# Install .NET SDK on Linux - needed to download the MicroBuild plugin nupkgs when nuget.exe is unavailable. +# Install to an isolated location so the repo's global.json doesn't interfere. +- task: UseDotNet@2 + displayName: Install .NET SDK for MicroBuild Plugin + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + packageType: sdk + version: 8.0.x + installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet + +# Create a global.json in the MicroBuild folder that pins to the installed SDK. +# This prevents the repo's global.json from causing SDK resolution failures. +- script: | + mkdir -p ${{ parameters.microBuildOutputFolder }} + version=$(dotnet --version) + cat > ${{ parameters.microBuildOutputFolder }}/global.json << EOF + { + "sdk": { + "version": "$version" + } + } + EOF + displayName: Create global.json for MicroBuild + condition: and(succeeded(), ${{ parameters.condition }}) + +- task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild Signing Plugin + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + version: $(MicroBuildPluginVersion) + ${{ if eq(parameters.signType, 'test') }}: + signType: test + ${{ else }}: + signType: real + zipSources: false + feedSource: $(MicroBuildFeedSource) + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc + workingDirectory: ${{ parameters.microBuildOutputFolder }} + env: + TeamName: $(TeamName) + MicroBuildOutputFolderOverride: $(Agent.TempDirectory)/MicroBuild + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +# Configure ImageBuilder docker run options and write env file for signing. +# Sets imageBuilderDockerRunExtraOptions with the plugin volume mount and, +# when envFileVariableName is provided, writes a signing env file and sets +# the named pipeline variable to its path via logging directive. +- ${{ if ne(parameters.envFileVariableName, '') }}: + - task: PowerShell@2 + displayName: Configure ImageBuilder Signing Options + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + # Mount the MicroBuild signing plugin directory (contains DDSignFiles.dll and esrpcli.dll). + $imageBuilderDockerRunExtraOptions = "-v $env:MBSIGN_APPFOLDER`:/microbuild" + Write-Host "MicroBuild signing enabled, mounting $env:MBSIGN_APPFOLDER to /microbuild" + Write-Host "##vso[task.setvariable variable=imageBuilderDockerRunExtraOptions]$imageBuilderDockerRunExtraOptions" + + # Write the signing env file for docker --env-file. + # Docker reads this file on the host before creating the container, + # so no volume mount is needed for the file itself. + $envFilePath = "$(Agent.TempDirectory)/imagebuilder-signing.env" + $envFileContent = @( + # MicroBuild plugin variables for DDSignFiles.dll + "MBSIGN_APPFOLDER=/microbuild" + "VSENGESRPSSL" + "USEESRPCLI" + "MBSIGN_CONNECTEDSERVICE" + + # Container-local temp/workspace paths (host paths aren't accessible inside the container) + "MBSIGNTEMPDIR=/tmp/MicroBuildSign" + "PIPELINE_WORKSPACE=$(Build.ArtifactStagingDirectory)" + "AGENT_TEMPDIRECTORY=/tmp" + + # Azure DevOps pipeline variables for ESRP bearer token auth (ESRPUtils.GetAccountInfo) + "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI" + "BUILD_BUILDID" + "SYSTEM_TEAMPROJECT" + "BUILD_SOURCEBRANCH" + + # Azure DevOps pipeline variables for ESRP CLI federated token (ESRPCliDll.GetFederatedTokenData) + "SYSTEM_JOBID" + "SYSTEM_PLANID" + "SYSTEM_TEAMPROJECTID" + "SYSTEM_HOSTTYPE" + "SYSTEM_COLLECTIONURI" + + # Azure DevOps pipeline variables for DDSignFilesConfiguration + "BUILD_DEFINITIONNAME" + "BUILD_BUILDNUMBER" + ) + + $envFileContent | Set-Content -Path $envFilePath -Encoding utf8NoBOM + Write-Host "##vso[task.setvariable variable=${{ parameters.envFileVariableName }}]$envFilePath" diff --git a/eng/docker-tools/templates/steps/reference-service-connections.yml b/eng/docker-tools/templates/steps/reference-service-connections.yml new file mode 100644 index 0000000000..30f59e4403 --- /dev/null +++ b/eng/docker-tools/templates/steps/reference-service-connections.yml @@ -0,0 +1,68 @@ +# Emit AzureCLI@2 steps that reference service connections in the current +# stage so that Azure DevOps will issue OIDC tokens for them. Azure DevOps +# requires each stage to explicitly reference (via an azureSubscription task +# input) any service connection it needs OIDC tokens for. +# +# This template must be included in every job that authenticates to Azure +# via AzurePipelinesCredential (i.e., any job that uses run-imagebuilder.yml +# with internalProjectName, or run-pwsh-with-auth.yml). +# +# Service connections can be specified in two ways: +# - Via publishConfig + usesRegistries: looks up service connections from +# publishConfig.RegistryAuthentication entries matching the given servers. +# Use this for registry-scoped connections (e.g., BuildRegistry, PublishRegistry). +# - Via serviceConnections: a direct list of { name: string } objects. +# Use this for non-registry connections (e.g., kusto, marStatus, cleanServiceConnection). +# +# Both can be used together in a single call. +parameters: +# Publishing configuration object. Only needed when using the usesRegistries parameter. +- name: publishConfig + type: object + default: {} +# List of registry server URLs to look up in publishConfig.RegistryAuthentication. +# Each matching entry's service connection will be referenced. +- name: usesRegistries + type: object + default: [] +# Direct list of service connections to reference. Shape: [{ name: string }] +- name: serviceConnections + type: object + default: [] +# The OS of the agent running this job. Determines whether to use PowerShell +# Core (pscore, Linux) or Windows PowerShell (ps, Windows) for the AzureCLI task. +- name: dockerClientOS + type: string + default: linux +# The internal Azure DevOps project name. Reference steps are only emitted +# for internal non-PR builds, since public projects don't have these service +# connections. +- name: internalProjectName + type: string + default: internal + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ each serviceConnection in parameters.serviceConnections }}: + - task: AzureCLI@2 + displayName: Reference ${{ serviceConnection.name }} + inputs: + azureSubscription: ${{ serviceConnection.name }} + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: ps + ${{ else }}: + scriptType: pscore + scriptLocation: inlineScript + inlineScript: Write-Host "Service connection referenced for OIDC" + - ${{ each auth in parameters.publishConfig.RegistryAuthentication }}: + - ${{ if containsValue(parameters.usesRegistries, auth.server) }}: + - task: AzureCLI@2 + displayName: Reference ${{ auth.serviceConnection.name }} + inputs: + azureSubscription: ${{ auth.serviceConnection.name }} + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: ps + ${{ else }}: + scriptType: pscore + scriptLocation: inlineScript + inlineScript: Write-Host "Service connection referenced for OIDC" diff --git a/eng/docker-tools/templates/variables/common-paths.yml b/eng/docker-tools/templates/variables/common-paths.yml index d8a5200250..d676441b54 100644 --- a/eng/docker-tools/templates/variables/common-paths.yml +++ b/eng/docker-tools/templates/variables/common-paths.yml @@ -3,3 +3,4 @@ variables: engDockerToolsPath: $(Build.Repository.LocalPath)/$(engDockerToolsRelativePath) engPath: $(Build.Repository.LocalPath)/eng testScriptPath: "" + preBuildTestScriptPath: $(testScriptPath) diff --git a/eng/docker-tools/templates/variables/common.yml b/eng/docker-tools/templates/variables/common.yml index 5eba63719b..680dc12930 100644 --- a/eng/docker-tools/templates/variables/common.yml +++ b/eng/docker-tools/templates/variables/common.yml @@ -6,6 +6,8 @@ variables: value: true - name: publishImageInfo value: true +- name: ingestKustoImageInfo + value: true # CG is disabled by default because projects are built within Dockerfiles and CG step do not scan artifacts # that are built within Dockerfiles. A separate CG pipeline exists for this reason. - name: skipComponentGovernanceDetection @@ -77,3 +79,11 @@ variables: value: 00000000-0000-0000-0000-000000000000 - name: acr-staging.subscription value: 00000000-0000-0000-0000-000000000000 + +# See https://devdiv.visualstudio.com/Engineering/_git/Sign?version=GBmain&path=/src/CertificateMappings.xml +- name: microBuildSigningKeyCode.containers + value: 4512 +- name: microBuildSigningKeyCode.attestations + value: 4571 +- name: microBuildSigningKeyCode.testing + value: 2151 diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index 86b50eb984..293657be58 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2914488 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2919324 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner diff --git a/eng/docker-tools/templates/variables/dotnet/common.yml b/eng/docker-tools/templates/variables/dotnet/common.yml index 1515bd75ca..1a0322208f 100644 --- a/eng/docker-tools/templates/variables/dotnet/common.yml +++ b/eng/docker-tools/templates/variables/dotnet/common.yml @@ -68,4 +68,15 @@ variables: - name: windowsServer2025PoolName value: Docker-2025-${{ variables['System.TeamProject'] }} +## Signing +# Team name required for MicroBuild signing plugin +- name: TeamName + value: DotNetCore +# MicroBuild signing plugin feed source +- name: MicroBuildFeedSource + value: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json +# MicroBuild signing plugin version +- name: MicroBuildPluginVersion + value: latest + - group: DotNet-Docker-Common-2 diff --git a/eng/pipelines/stages/build-and-test.yml b/eng/pipelines/stages/build-and-test.yml index da30d979f5..de05eed741 100644 --- a/eng/pipelines/stages/build-and-test.yml +++ b/eng/pipelines/stages/build-and-test.yml @@ -41,17 +41,6 @@ parameters: # clientId: "" stages: -- ${{ if parameters.isStandaloneBuild }}: - - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - usesRegistries: - - ${{ parameters.publishConfig.InternalMirrorRegistry.server }} - - ${{ parameters.publishConfig.BuildRegistry.server }} - serviceConnections: - - ${{ if parameters.storageAccountServiceConnection }}: - - name: ${{ parameters.storageAccountServiceConnection.name }} - - template: /eng/docker-tools/templates/stages/dotnet/build-and-test.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} diff --git a/eng/pipelines/stages/build-test-publish-repo.yml b/eng/pipelines/stages/build-test-publish-repo.yml index d6790b0d8a..4e9f354fe6 100644 --- a/eng/pipelines/stages/build-test-publish-repo.yml +++ b/eng/pipelines/stages/build-test-publish-repo.yml @@ -33,18 +33,6 @@ parameters: default: [] stages: -- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - usesRegistries: - - ${{ parameters.publishConfig.InternalMirrorRegistry.server }} - - ${{ parameters.publishConfig.BuildRegistry.server }} - - ${{ parameters.publishConfig.PublishRegistry.server }} - serviceConnections: - - ${{ each serviceConnection in parameters.additionalServiceConnections }}: - - name: ${{ serviceConnection.name }} - - template: /eng/pipelines/stages/build-and-test.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} @@ -61,3 +49,4 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} versionsRepoRef: ${{ parameters.versionsRepoRef }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} diff --git a/eng/pipelines/stages/cg-images.yml b/eng/pipelines/stages/cg-images.yml index 3fc01d3c08..960aab9f0b 100644 --- a/eng/pipelines/stages/cg-images.yml +++ b/eng/pipelines/stages/cg-images.yml @@ -34,8 +34,9 @@ stages: arch: ${{ arch.arch }} steps: - - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - template: /eng/docker-tools/templates/steps/init-common.yml@self parameters: + dockerClientOS: linux cleanupDocker: true - script: > diff --git a/eng/pipelines/stages/publish.yml b/eng/pipelines/stages/publish.yml index d80fb09a4c..e21e35461b 100644 --- a/eng/pipelines/stages/publish.yml +++ b/eng/pipelines/stages/publish.yml @@ -15,7 +15,7 @@ parameters: type: boolean default: false # Additional service connections needed by this pipeline that are not included - # in the publishConfig. Only used when `isStandalonePublish` is true. + # in the publishConfig. - name: additionalServiceConnections type: object default: [] @@ -36,16 +36,6 @@ parameters: default: false stages: -- ${{ if parameters.isStandalonePublish }}: - - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - usesRegistries: - - ${{ parameters.publishConfig.PublishRegistry.server }} - serviceConnections: - - ${{ each serviceConnection in parameters.additionalServiceConnections }}: - - name: ${{ serviceConnection.name }} - - template: /eng/docker-tools/templates/stages/dotnet/publish.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} @@ -53,6 +43,7 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} versionsRepoRef: ${{ parameters.versionsRepoRef }} isStandalonePublish: ${{ parameters.isStandalonePublish }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} pool: ${{ parameters.linuxAmd64Pool }} customPublishInitSteps: - ${{ parameters.customPublishInitSteps }} diff --git a/eng/pipelines/steps/set-publish-mcrdocs-args-var.yml b/eng/pipelines/steps/set-publish-mcrdocs-args-var.yml index 74897ab420..79b4a71d3c 100644 --- a/eng/pipelines/steps/set-publish-mcrdocs-args-var.yml +++ b/eng/pipelines/steps/set-publish-mcrdocs-args-var.yml @@ -1,8 +1,8 @@ steps: - powershell: | # When multiple repos are checked out, the repo is in a subdirectory named after the repo. - # The buildRepoName variable is set in init-matrix-build-publish.yml, but only when mutiple - # repos are checked out. + # The buildRepoName variable is set in init-common.yml, but only when multiple repos are + # checked out. $buildRepoName = "$env:BUILDREPONAME" if ($buildRepoName) { $portalDocsRoot = "/repo/$buildRepoName/.portal-docs" diff --git a/eng/pipelines/update-readmes.yml b/eng/pipelines/update-readmes.yml index a5343bf20f..9598a7df36 100644 --- a/eng/pipelines/update-readmes.yml +++ b/eng/pipelines/update-readmes.yml @@ -14,7 +14,9 @@ extends: - job: UpdateReadmes displayName: Update Readmes steps: - - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux - template: /eng/pipelines/steps/set-public-source-branch-var.yml@self - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self parameters: diff --git a/eng/pipelines/variables/samples.yml b/eng/pipelines/variables/samples.yml index e4b0e2acf3..5283ae9a52 100644 --- a/eng/pipelines/variables/samples.yml +++ b/eng/pipelines/variables/samples.yml @@ -1,7 +1,8 @@ variables: - template: /eng/docker-tools/templates/variables/dotnet/build-test-publish.yml@self -# Cannot be readonly because it's currently overridden by eng/docker-tools/templates/steps/init-matrix-build-publish.yml +# Cannot be readonly because it's currently overridden by +# eng/docker-tools/templates/steps/init-common.yml during multi-repo checkout. # `manifest` is overloaded, since it is used both for this definition and also # to refer to the precise path of the manifest file in multi-repo builds (e.g. # when we are using a versions repo)