Add scripts for creating artifacts (#8602)

* add remote build to the pipeline

* add a separte compile step

* fix darwin build

* add linux container and fix docker creation

* fix distro

* remove system install and add xvfb start

* distro

* add logic to only run tests on hosted machine

* fix yml

* fix yml

* add linux docker container

* fix docker file

* fixdocker

* fix darwin

* fix linux build

* add cache salt to npm cache

* intentially ignore kerberos binaries

* disable docker for now

* remove vsix from win32

* fix linxu and win32

* fix linux and win32

* fix linux and win32

* fix linux

* maybe fix win32

* fix linux

* fix linux image; disable server package for noe

* fix minimatch for win32 test

* fix linux build

* add back in docker

* fix test

* use tmp directory insteado workspace

* change name of docker image

* try a differnt folder

* fix download

* add a git clean step

* bump cache

* fix issues with builidng

* readd windows build, revert signing changes

* simplify win32 server packaage

* some more optimizations

* use decompress task

* add back in install for windows test

* fix linux build

* add integration test to bat file

* make platform the same

* add copy extension to windows test

* revert tests back

* fix vsix drop aquasition

* inital changes

* fix download

* fix dependent on for release

* just downlaod everything which makes it easier

* setup pipeline artifacts

* more clean up

* fix linux

* add logic to install extensions for integration tests

* fix extension install

* fix build failures

* fix some issues

* fix darwin drop

* change linux build copy to js

* fix darwin archive

* fix copy artifacts and use it for windows

* use for darinw

* fix darwin

* ad dep on linux

* fix win32

* fix darwin

* fix copy artifacts

* mkdir p darwin

* fix copy

* add error handler

* add more binaries

* add more binaries

* fix archive path on linux

* add more options to integration extension install

* add more binaries

* add verbose to installer copy

* fix ip rate issues

* fix bat file for including extensions

* move echo

* add windows test condition

* use powershell cmdlet rather than cp

* remove verbose

* remove compiling of extensions

* fix pipelines

* update docker location

* fix copy item

* fix signing on win32

* fix locations

* move back to using cp

* ensure the docker folder exists

* test a createdrop script on darwin

* wip

* fix copy

* add drop for linux

* fix builds

* fix drop

* fix docker on linx

* fix darwin

* lets try this again

* fix linux drop

* i guess try the copy files task

* add create drop for win32

* ensure windows drop location exists

* fix extension install

* just use mkdir

* add better logic for installing extensions

* ignore errors?

* try force

* testing

* ok this should work

* use production cli

* fix liveshare vscodeignore

* fix integration test script

* revert changes to integration tests to fix them

* try newitem

* remove exec

* explicitly clear last exit code

* fix test build

* revert publish scripts

* add version json

* fix tests

* add back sources creation

* this is stupid

* fix clean positioning

* add version information to drop

* fix locations of artifacts in publush scripts

build/azure-pipelines/win32/createDrop.ps1

@@ -0,0 +1,20 @@
+. build/azure-pipelines/win32/exec.ps1
+$ErrorActionPreference = "Stop"
+
+$Arch = "x64"
+
+$Repo = "$(pwd)"
+$Root = "$Repo\.."
+$LegacyServer = "$Root\azuredatastudio-reh-win32-$Arch"
+$ServerName = "azuredatastudio-server-win32-$Arch"
+$Server = "$Root\$ServerName"
+$ServerZipLocation = "$Repo\.build\win32-$Arch\server"
+$ServerZip = "$ServerZipLocation\azuredatastudio-server-win32-$Arch.zip"
+
+# Create server archive
+New-Item $ServerZipLocation -ItemType Directory # this will throw even when success for we don't want to exec this
+$global:LASTEXITCODE = 0
+exec { Rename-Item -Path $LegacyServer -NewName $ServerName } "Rename Item"
+exec { .\node_modules\7zip\7zip-lite\7z.exe a -tzip $ServerZip $Server -r } "Zip Server"
+
+exec { node build/azure-pipelines/common/copyArtifacts.js } "Copy Artifacts"

build/azure-pipelines/win32/sql-product-build-win32.yml

@@ -105,19 +105,6 @@ steps:
     env:
       VSCODE_MIXIN_PASSWORD: $(github-distro-mixin-password)
 
-  - task: ArchiveFiles@2 # WHY
-    displayName: 'Archive build scripts source'
-    inputs:
-      rootFolderOrFile: '$(Build.SourcesDirectory)/build'
-      archiveType: tar
-      archiveFile: '$(Build.BinariesDirectory)/source.tar.gz'
-
-  - task: PublishBuildArtifacts@1 # WHY
-    displayName: 'Publish Artifact: build scripts source'
-    inputs:
-      PathtoPublish: '$(Build.BinariesDirectory)/source.tar.gz'
-      ArtifactName: source
-
   - powershell: |
       . build/azure-pipelines/win32/exec.ps1
       $ErrorActionPreference = "Stop"
@@ -189,39 +176,11 @@ steps:
       exec { yarn gulp "vscode-win32-x64-archive" }
     displayName: Archive & User & System setup
 
-  - task: CopyFiles@2
-    displayName: 'Copy Archive to: $(Build.ArtifactStagingDirectory)'
-    inputs:
-      SourceFolder: '$(Build.SourcesDirectory)/.build/win32-x64/archive/'
-      TargetFolder: '$(Build.ArtifactStagingDirectory)' # our release scripts expect the archive to be in the root
-
-  - task: CopyFiles@2
-    displayName: 'Copy User Installer to: $(Build.ArtifactStagingDirectory)'
-    inputs:
-      SourceFolder: '$(Build.SourcesDirectory)/.build/win32-x64/user-setup/'
-      TargetFolder: '$(Build.ArtifactStagingDirectory)/user-setup/'
-
-  - task: CopyFiles@2
-    displayName: 'Copy System Install to: $(Build.ArtifactStagingDirectory)'
-    inputs:
-      SourceFolder: '$(Build.SourcesDirectory)/.build/win32-x64/system-setup/'
-      TargetFolder: '$(Build.ArtifactStagingDirectory)/' # our release scripts except system exe to be in root and user setup to be under /user-setup
-
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      $Repo = "$(pwd)"
-      $Root = "$Repo\.."
-      $LegacyServer = "$Root\azuredatastudio-reh-win32-x64"
-      $ServerZip = "$(Build.ArtifactStagingDirectory)\azuredatastudio-server-win32-x64.zip"
-      exec { .\node_modules\7zip\7zip-lite\7z.exe a -tzip $ServerZip $LegacyServer -r }
-    displayName: 'Package server'
-
   - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
     displayName: 'Sign installers'
     inputs:
       ConnectedServiceName: 'Code Signing'
-      FolderPath: '$(Build.ArtifactStagingDirectory)'
+      FolderPath: '.build'
       Pattern: '*.exe'
       signConfigType: inlineSignParams
       inlineOperation: |
@@ -272,26 +231,24 @@ steps:
       MaxRetryAttempts: 20
     condition: and(succeeded(), eq(variables['signed'], true))
 
-  - powershell: | # WHY!
-      $PackageJson = Get-Content -Raw -Path "$(Build.SourcesDirectory)\package.json" | ConvertFrom-Json
+  - task: ArchiveFiles@2
+    displayName: 'Archive build scripts source'
+    inputs:
+      rootFolderOrFile: '$(Build.SourcesDirectory)/build'
+      archiveType: tar
+      archiveFile: '$(Build.BinariesDirectory)/source.tar.gz'
 
-      $jsonResult = @{
-      version = $PackageJson.version
-      quality = $env:VSCODE_QUALITY
-      commit = "$(git rev-parse HEAD)"
-      }
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: build scripts source'
+    inputs:
+      PathtoPublish: '$(Build.BinariesDirectory)/source.tar.gz'
+      ArtifactName: source
 
-      $jsonResult | ConvertTo-Json | Out-File "$(Build.ArtifactStagingDirectory)\version.json"
-    displayName: 'Create version.json'
-
-  - powershell: | # WHY
-      Get-ChildItem "." |
-      ForEach-Object {
-          certutil.exe -hashfile $_.FullName SHA256 >> sha256hashes.txt
-      }
-    workingDirectory: '$(Build.ArtifactStagingDirectory)'
-    displayName: 'Get SHA256 Hashes'
-    continueOnError: true
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      .\build\azure-pipelines\win32\createDrop.ps1
+    displayName: Create Drop
 
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: drop'
@@ -319,11 +276,3 @@ steps:
     displayName: 'Component Detection'
     inputs:
       failOnAlert: true
-
-  - powershell: 'Write-Host "##vso[build.addbuildtag]Scheduled" '
-    displayName: 'Tag build if scheduled'
-    condition: and(in(variables['Agent.JobStatus'], 'Succeeded'), eq(variables['Build.Reason'], 'Schedule'))
-
-  - powershell: 'Write-Host "##vso[build.addbuildtag]PerfTestCandidate" '
-    displayName: 'Tag build for PerfTestCandidate if needed'
-    condition: and(in(variables['Agent.JobStatus'], 'Succeeded'), eq(variables['VSCODE_QUALITY'], 'insider'))

build/azure-pipelines/win32/sql-product-test-win32.yml

@@ -1,10 +1,4 @@
 steps:
-- task: DownloadPipelineArtifact@2
-  inputs:
-    buildType: 'current'
-    targetPath: '$(Agent.TempDirectory)'
-    artifactName: drop
-
 - task: NodeTool@0
   inputs:
     versionSpec: "10.15.1"
@@ -20,6 +14,12 @@ steps:
     exec { git clean -fxd }
   displayName: Clean repo
 
+- task: DownloadPipelineArtifact@2
+  inputs:
+    buildType: 'current'
+    targetPath: '$(Build.SourcesDirectory)\.build'
+    artifactName: drop
+
 - powershell: |
     . build/azure-pipelines/win32/exec.ps1
     $ErrorActionPreference = "Stop"
@@ -30,7 +30,7 @@ steps:
 - powershell: |
     . build/azure-pipelines/win32/exec.ps1
     $ErrorActionPreference = "Stop"
-    exec { .\node_modules\7zip\7zip-lite\7z.exe x $(Agent.TempDirectory)\azuredatastudio-win32-x64.zip -o$(Agent.TempDirectory)\azuredatastudio-win32-x64 }
+    exec { .\node_modules\7zip\7zip-lite\7z.exe x $(Build.SourcesDirectory)\.build\win32-x64/archive/azuredatastudio-win32-x64.zip -o$(Agent.TempDirectory)\azuredatastudio-win32-x64 }
   displayName: Unzip artifact
 
 - task: AzureKeyVault@1

build/azure-pipelines/win32/sql-publish.ps1

@@ -9,11 +9,11 @@ $env:AZURE_STORAGE_ACCESS_KEY_2 = $storageKey
 $env:AZURE_DOCUMENTDB_MASTERKEY = $documentDbKey
 
 $ExeName = "AzureDataStudioSetup.exe"
-$SystemExe = "$artifactsDir\$ExeName"
-$UserExe = "$artifactsDir\user-setup\$ExeName"
+$SystemExe = "$artifactsDir\win32-x64\system-setup\$ExeName"
+$UserExe = "$artifactsDir\win32-x64\user-setup\$ExeName"
 $UserExeName = "AzureDataStudioUserSetup.exe"
 $ZipName = "azuredatastudio-win32-x64.zip"
-$Zip = "$artifactsDir\$ZipName"
+$Zip = "$artifactsDir\win32-x64\archive\$ZipName"
 
 $VersionJson = Get-Content -Raw -Path "$artifactsDir\version.json" | ConvertFrom-Json
 $Version = $VersionJson.version