Add scripts for creating artifacts (#8602)

* add remote build to the pipeline

* add a separte compile step

* fix darwin build

* add linux container and fix docker creation

* fix distro

* remove system install and add xvfb start

* distro

* add logic to only run tests on hosted machine

* fix yml

* fix yml

* add linux docker container

* fix docker file

* fixdocker

* fix darwin

* fix linux build

* add cache salt to npm cache

* intentially ignore kerberos binaries

* disable docker for now

* remove vsix from win32

* fix linxu and win32

* fix linux and win32

* fix linux and win32

* fix linux

* maybe fix win32

* fix linux

* fix linux image; disable server package for noe

* fix minimatch for win32 test

* fix linux build

* add back in docker

* fix test

* use tmp directory insteado workspace

* change name of docker image

* try a differnt folder

* fix download

* add a git clean step

* bump cache

* fix issues with builidng

* readd windows build, revert signing changes

* simplify win32 server packaage

* some more optimizations

* use decompress task

* add back in install for windows test

* fix linux build

* add integration test to bat file

* make platform the same

* add copy extension to windows test

* revert tests back

* fix vsix drop aquasition

* inital changes

* fix download

* fix dependent on for release

* just downlaod everything which makes it easier

* setup pipeline artifacts

* more clean up

* fix linux

* add logic to install extensions for integration tests

* fix extension install

* fix build failures

* fix some issues

* fix darwin drop

* change linux build copy to js

* fix darwin archive

* fix copy artifacts and use it for windows

* use for darinw

* fix darwin

* ad dep on linux

* fix win32

* fix darwin

* fix copy artifacts

* mkdir p darwin

* fix copy

* add error handler

* add more binaries

* add more binaries

* fix archive path on linux

* add more options to integration extension install

* add more binaries

* add verbose to installer copy

* fix ip rate issues

* fix bat file for including extensions

* move echo

* add windows test condition

* use powershell cmdlet rather than cp

* remove verbose

* remove compiling of extensions

* fix pipelines

* update docker location

* fix copy item

* fix signing on win32

* fix locations

* move back to using cp

* ensure the docker folder exists

* test a createdrop script on darwin

* wip

* fix copy

* add drop for linux

* fix builds

* fix drop

* fix docker on linx

* fix darwin

* lets try this again

* fix linux drop

* i guess try the copy files task

* add create drop for win32

* ensure windows drop location exists

* fix extension install

* just use mkdir

* add better logic for installing extensions

* ignore errors?

* try force

* testing

* ok this should work

* use production cli

* fix liveshare vscodeignore

* fix integration test script

* revert changes to integration tests to fix them

* try newitem

* remove exec

* explicitly clear last exit code

* fix test build

* revert publish scripts

* add version json

* fix tests

* add back sources creation

* this is stupid

* fix clean positioning

* add version information to drop

* fix locations of artifacts in publush scripts

build/azure-pipelines/darwin/createDrop.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+REPO="$(pwd)"
+
+# ensure drop directories exist
+mkdir -p $REPO/.build/darwin/{archive,server}
+
+# remove pkg from archive
+zip -d $REPO/.build/darwin/archive/azuredatastudio-darwin.zip "*.pkg"
+
+# package Remote Extension Host
+pushd .. && mv azuredatastudio-reh-darwin azuredatastudio-server-darwin && zip -Xry $REPO/.build/darwin/server/azuredatastudio-server-darwin.zip azuredatastudio-server-darwin && popd
+
+node build/azure-pipelines/common/copyArtifacts.js

build/azure-pipelines/darwin/sql-product-build-darwin.yml

@@ -99,38 +99,23 @@ steps:
     env:
       VSCODE_MIXIN_PASSWORD: $(github-distro-mixin-password)
 
-  - task: ArchiveFiles@2 # WHY ARE WE DOING THIS?
-    displayName: 'Archive build scripts source'
-    inputs:
-      rootFolderOrFile: '$(Build.SourcesDirectory)/build'
-      archiveType: tar
-      archiveFile: '$(Build.BinariesDirectory)/source.tar.gz'
-
-  - task: PublishBuildArtifacts@1 # WHY ARE WE DOING THIS?
-    displayName: 'Publish Artifact: build scripts source'
-    inputs:
-      PathtoPublish: '$(Build.BinariesDirectory)/source.tar.gz'
-      ArtifactName: source
+  - script: |
+      set -e
+      ./scripts/test.sh --build --coverage --reporter mocha-junit-reporter
+    displayName: Run unit tests
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
 
   - script: |
       set -e
-      cd $(Build.SourcesDirectory)/../azuredatastudio-darwin
-      zip -r -y $(Build.ArtifactStagingDirectory)/azuredatastudio-darwin.zip "Azure Data Studio.app"
-    displayName: 'Stable - Run zip'
-    condition: and(succeeded(), eq(variables['VSCODE_QUALITY'], 'stable'))
-
-  - script: |
-      set -e
-      cd $(Build.SourcesDirectory)/../azuredatastudio-darwin
-      zip -r -y $(Build.ArtifactStagingDirectory)/azuredatastudio-darwin.zip "Azure Data Studio - Insiders.app"
-    displayName: 'Insiders - Run zip'
-    condition: and(succeeded(), eq(variables['VSCODE_QUALITY'], 'insider'))
+      mkdir -p .build/darwin/archive
+      pushd ../azuredatastudio-darwin && zip -r -X -y $(Build.SourcesDirectory)/.build/darwin/archive/azuredatastudio-darwin.zip * && popd
+    displayName: 'Archive'
 
   - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
     displayName: 'ESRP CodeSigning'
     inputs:
       ConnectedServiceName: 'Code Signing'
-      FolderPath: '$(Build.ArtifactStagingDirectory)'
+      FolderPath: '$(Build.SourcesDirectory)/.build/darwin/archive'
       Pattern: 'azuredatastudio-darwin.zip'
       signConfigType: inlineSignParams
       inlineOperation: |
@@ -145,48 +130,10 @@ steps:
         ]
       SessionTimeout: 20
 
-  - script: | # WHY ARE WE DOING THIS?
-      set -e
-      zip -d $(Build.ArtifactStagingDirectory)/azuredatastudio-darwin.zip "Azure Data Studio.pkg"
-    displayName: 'Stable - Remove .pkg from signed zip'
-    condition: and(succeeded(), eq(variables['VSCODE_QUALITY'], 'stable'))
-
-  - script: | # WHY ARE WE DOING THIS?
-      set -e
-      zip -d $(Build.ArtifactStagingDirectory)/azuredatastudio-darwin.zip "Azure Data Studio - Insiders.pkg"
-    displayName: 'Insiders - Remove .pkg from signed zip'
-    condition: and(succeeded(), eq(variables['VSCODE_QUALITY'], 'insider'))
-
   - script: |
       set -e
-      ./scripts/test.sh --build --coverage --reporter mocha-junit-reporter
-    displayName: Run unit tests
-    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
-    timeoutInMinutes: 20
-    continueOnError: true
-
-  - script: |
-      pushd .. && mv azuredatastudio-reh-darwin azuredatastudio-server-darwin && zip -Xry $(Build.ArtifactStagingDirectory)/azuredatastudio-server-darwin.zip azuredatastudio-server-darwin && popd
-    displayName: 'Package server'
-
-  - script: | # WHY ARE WE DOING THIS?
-      set -e
-      PACKAGEJSON=`ls $(Build.SourcesDirectory)/package.json`
-      VERSION=`node -p "require(\"$PACKAGEJSON\").version"`
-      COMMIT_ID=`git rev-parse HEAD`
-
-      echo -e "{  \"version\": \"$VERSION\", \"quality\": \"$VSCODE_QUALITY\", \"commit\": \"$COMMIT_ID\" }" > "$(Build.ArtifactStagingDirectory)/version.json"
-    displayName: 'Create version.json'
-
-  - script: | # WHY ARE WE DOING THIS?
-      set -e
-      for f in *
-      do
-      shasum -a 256 "$f" >> sha256hashes.txt
-      done
-    workingDirectory: '$(Build.ArtifactStagingDirectory)'
-    displayName: 'Get SHA256 Hashes'
-    continueOnError: true
+      ./build/azure-pipelines/darwin/createDrop.sh
+    displayName: Create Drop
 
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: drop'
@@ -220,7 +167,3 @@ steps:
     displayName: 'Component Detection'
     inputs:
       failOnAlert: true
-
-  - script: 'echo "##vso[build.addbuildtag]Scheduled" '
-    displayName: 'Add scheduled tag if needed'
-    condition: and(in(variables['Agent.JobStatus'], 'Succeeded'), eq(variables['Build.Reason'], 'Schedule'))

build/azure-pipelines/darwin/sql-publish.ps1

@@ -14,6 +14,6 @@ $Quality = $VersionJson.quality
 $CommitId = $VersionJson.commit
 
 $ZipName = "azuredatastudio-darwin.zip"
-$Zip = "$artifactsDir\$ZipName"
+$Zip = "$artifactsDir\darwin\archive\$ZipName"
 
 node $sourcesDir\build\azure-pipelines\common\publish.js $Quality darwin archive $ZipName $Version true $Zip $CommitId