Implement Windows release builds in Azure Pipelines

.azure-pipelines/windows-release.yml

@@ -0,0 +1,96 @@
+name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+# QUEUE TIME VARIABLES
+# variables:
+#   GitRemote: python
+#   SourceTag:
+#   DoPGO: true
+#   SigningCertificate: 'Python Software Foundation'
+#   SigningDescription: 'Built: $(Build.BuildNumber)'
+#   DoLayout: true
+#   DoMSIX: true
+#   DoNuget: true
+#   DoEmbed: true
+#   DoMSI: true
+#   DoPublish: false
+
+trigger: none
+pr: none
+
+stages:
+- stage: Build
+  displayName: Build binaries
+  jobs:
+  - template: windows-release/stage-build.yml
+
+- stage: Sign
+  displayName: Sign binaries
+  dependsOn: Build
+  jobs:
+  - template: windows-release/stage-sign.yml
+
+- stage: Layout
+  displayName: Generate layouts
+  dependsOn: Sign
+  jobs:
+  - template: windows-release/stage-layout-full.yml
+  - template: windows-release/stage-layout-embed.yml
+  - template: windows-release/stage-layout-nuget.yml
+
+- stage: Pack
+  dependsOn: Layout
+  jobs:
+  - template: windows-release/stage-pack-nuget.yml
+
+- stage: Test
+  dependsOn: Pack
+  jobs:
+  - template: windows-release/stage-test-embed.yml
+  - template: windows-release/stage-test-nuget.yml
+
+- stage: Layout_MSIX
+  displayName: Generate MSIX layouts
+  dependsOn: Sign
+  condition: and(succeeded(), eq(variables['DoMSIX'], 'true'))
+  jobs:
+  - template: windows-release/stage-layout-msix.yml
+
+- stage: Pack_MSIX
+  displayName: Package MSIX
+  dependsOn: Layout_MSIX
+  jobs:
+  - template: windows-release/stage-pack-msix.yml
+
+- stage: Build_MSI
+  displayName: Build MSI installer
+  dependsOn: Sign
+  condition: and(succeeded(), eq(variables['DoMSI'], 'true'))
+  jobs:
+  - template: windows-release/stage-msi.yml
+
+- stage: Test_MSI
+  displayName: Test MSI installer
+  dependsOn: Build_MSI
+  jobs:
+  - template: windows-release/stage-test-msi.yml
+
+- stage: PublishPyDotOrg
+  displayName: Publish to python.org
+  dependsOn: ['Test_MSI', 'Test']
+  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  jobs:
+  - template: windows-release/stage-publish-pythonorg.yml
+
+- stage: PublishNuget
+  displayName: Publish to nuget.org
+  dependsOn: Test
+  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  jobs:
+  - template: windows-release/stage-publish-nugetorg.yml
+
+- stage: PublishStore
+  displayName: Publish to Store
+  dependsOn: Pack_MSIX
+  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  jobs:
+  - template: windows-release/stage-publish-store.yml

.azure-pipelines/windows-release/build-steps.yml

@@ -0,0 +1,83 @@
+parameters:
+  ShouldPGO: false
+
+steps:
+- template: ./checkout.yml
+
+- powershell: |
+    $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+    Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)"
+    Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)"
+    Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)"
+    Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)"
+    Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)"
+    Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)"
+  displayName: 'Extract version numbers'
+
+- ${{ if eq(parameters.ShouldPGO, 'false') }}:
+  - powershell: |
+      $env:SigningCertificate = $null
+      .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration)
+    displayName: 'Run build'
+    env:
+      IncludeUwp: true
+      Py_OutDir: '$(Build.BinariesDirectory)\bin'
+
+- ${{ if eq(parameters.ShouldPGO, 'true') }}:
+  - powershell: |
+      $env:SigningCertificate = $null
+      .\PCbuild\build.bat -v -p $(Platform) --pgo
+    displayName: 'Run build with PGO'
+    env:
+      IncludeUwp: true
+      Py_OutDir: '$(Build.BinariesDirectory)\bin'
+
+- powershell: |
+    $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
+    $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1)
+    if (-not $tool) {
+      throw "SDK is not available"
+    }
+    Write-Host "##vso[task.prependpath]$($tool.Directory)"
+  displayName: 'Add WinSDK tools to path'
+
+- powershell: |
+    $env:SigningCertificate = $null
+    .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default
+    makecat "${env:CAT}.cdf"
+    del "${env:CAT}.cdf"
+    if (-not (Test-Path "${env:CAT}.cat")) {
+      throw "Failed to build catalog file"
+    }
+  displayName: 'Generate catalog'
+  env:
+    CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish binaries'
+  condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])))
+  inputs:
+    PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)'
+    ArtifactName: bin_$(Name)
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish binaries for signing'
+  condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))
+  inputs:
+    PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)'
+    ArtifactName: unsigned_bin_$(Name)
+
+- task: CopyFiles@2
+  displayName: 'Layout Artifact: symbols'
+  inputs:
+    sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch)
+    targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name)
+    flatten: true
+    contents: |
+      **\*.pdb
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish Artifact: symbols'
+  inputs:
+    PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols'
+    ArtifactName: symbols

.azure-pipelines/windows-release/checkout.yml

@@ -0,0 +1,21 @@
+parameters:
+  depth: 3
+
+steps:
+- checkout: none
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git .
+  displayName: 'git clone ($(GitRemote)/$(SourceTag))'
+  condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag']))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) .
+  displayName: 'git clone (<default>/$(SourceTag))'
+  condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag']))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git .
+  displayName: 'git clone ($(GitRemote)/<default>)'
+  condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag'])))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) .
+  displayName: 'git clone'
+  condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag'])))

.azure-pipelines/windows-release/find-sdk.yml

@@ -0,0 +1,17 @@
+# Locate the Windows SDK and add its binaries directory to PATH
+#
+# `toolname` can be overridden to use a different marker file.
+
+parameters:
+  toolname: signtool.exe
+
+steps:
+  - powershell: |
+      $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
+      $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1)
+      if (-not $tool) {
+          throw "SDK is not available"
+      }
+      Write-Host "##vso[task.prependpath]$($tool.Directory)"
+      Write-Host "Adding $($tool.Directory) to PATH"
+    displayName: 'Add WinSDK tools to path'

.azure-pipelines/windows-release/layout-command.yml

@@ -0,0 +1,20 @@
+steps:
+- powershell: >
+    Write-Host (
+    '##vso[task.setvariable variable=LayoutCmd]&
+    "{0}"
+    "{1}\PC\layout"
+    -vv
+    --source "{1}"
+    --build "{2}"
+    --temp "{3}"
+    --include-cat "{2}\python.cat"
+    --doc-build "{4}"'
+    -f (
+    "$(PYTHON)",
+    "$(Build.SourcesDirectory)",
+    (Split-Path -Parent "$(PYTHON)"),
+    "$(Build.BinariesDirectory)\layout-temp",
+    "$(Build.BinariesDirectory)\doc"
+    ))
+  displayName: 'Set LayoutCmd'

.azure-pipelines/windows-release/mingw-lib.yml

@@ -0,0 +1,13 @@
+parameters:
+  DllToolOpt: -m i386:x86-64
+  #DllToolOpt: -m i386 --as-flags=--32
+
+steps:
+- powershell: |
+    git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils"
+    gci "bin\$(Arch)\python*.dll" | %{
+      & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def
+      & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }}
+    }
+  displayName: 'Generate MinGW import library'
+  workingDirectory: $(Build.BinariesDirectory)

.azure-pipelines/windows-release/msi-steps.yml

@@ -0,0 +1,142 @@
+steps:
+  - template: ./checkout.yml
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: doc'
+    inputs:
+      artifactName: doc
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: CopyFiles@2
+    displayName: 'Merge documentation files'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\doc
+      targetFolder: $(Build.SourcesDirectory)\Doc\build
+      contents: |
+        htmlhelp\*.chm
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_win32'
+    inputs:
+      artifactName: bin_win32
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_win32_d'
+    inputs:
+      artifactName: bin_win32_d
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: CopyFiles@2
+    displayName: 'Merge win32 debug files'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\bin_win32_d
+      targetFolder: $(Build.BinariesDirectory)\bin_win32
+      contents: |
+        **\*_d.*
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_amd64'
+    inputs:
+      artifactName: bin_amd64
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_amd64_d'
+    inputs:
+      artifactName: bin_amd64_d
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: CopyFiles@2
+    displayName: 'Merge amd64 debug files'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\bin_amd64_d
+      targetFolder: $(Build.BinariesDirectory)\bin_amd64
+      contents: |
+        **\*_d.*
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: tcltk_lib_win32'
+    inputs:
+      artifactName: tcltk_lib_win32
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: tcltk_lib_amd64'
+    inputs:
+      artifactName: tcltk_lib_amd64
+      downloadPath: $(Build.BinariesDirectory)
+
+  - script: |
+      ren bin_win32 win32
+      ren bin_amd64 amd64
+    displayName: 'Correct artifact directory names'
+    workingDirectory: $(Build.BinariesDirectory)
+
+  - script: |
+      call Tools\msi\get_externals.bat
+      call PCbuild\find_python.bat
+      echo ##vso[task.setvariable variable=PYTHON]%PYTHON%
+      call PCbuild/find_msbuild.bat
+      echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD%
+    displayName: 'Get external dependencies'
+
+  - script: |
+      %PYTHON% -m pip install blurb
+      %PYTHON% -m blurb merge -f Misc\NEWS
+    displayName: 'Merge NEWS file'
+
+  - script: |
+      %MSBUILD% Tools\msi\launcher\launcher.wixproj
+    displayName: 'Build launcher installer'
+    env:
+      Platform: x86
+      Py_OutDir: $(Build.BinariesDirectory)
+
+  - script: |
+      %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true /p:BuildForRelease=true
+      %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false /p:BuildForRelease=true
+    displayName: 'Build win32 installer'
+    env:
+      Platform: x86
+      Py_OutDir: $(Build.BinariesDirectory)
+      PYTHON: $(Build.BinariesDirectory)\win32\python.exe
+      PYTHONHOME: $(Build.SourcesDirectory)
+      TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32
+
+  - script: |
+      %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true /p:BuildForRelease=true
+      %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false /p:BuildForRelease=true
+    displayName: 'Build amd64 installer'
+    env:
+      Platform: x64
+      Py_OutDir: $(Build.BinariesDirectory)
+      PYTHON: $(Build.BinariesDirectory)\amd64\python.exe
+      PYTHONHOME: $(Build.SourcesDirectory)
+      TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64
+
+  - task: CopyFiles@2
+    displayName: 'Assemble artifact: msi (1/2)'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\win32\en-us
+      targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32
+      contents: |
+        *.msi
+        *.cab
+        *.exe
+
+  - task: CopyFiles@2
+    displayName: 'Assemble artifact: msi (2/2)'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\amd64\en-us
+      targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64
+      contents: |
+        *.msi
+        *.cab
+        *.exe
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish MSI'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\msi'
+      ArtifactName: msi