Updating build pipelines to Cache@2 and switching to artifacts for compiled file sharing in pipeline jobs. (#14989)

* switiching product compile node cache task to newer version

* moving new changes from product-compile to sql-product-compile

* changing to yarn.lock as cache key

* Adding compilation cache

* changing keypath to key

* letting find command do the heavy lifting

* removing old save cache task

* reverting compilation cache to old task

* Creating a js to list compiled files
switching to cache 2 for compiled files
Creating a js file to compute yarn cache

* removed unused input targetFolder from pipeline cache task

* removed save cache

* Fixing compute node modules file

* Adding compiled computenodemodules

* Fixing checked variables on product compile
Updating all pipeline jobs to cache 2
Using tar for windows pipeline. Hoping it works

* Fixing indentation in web job

* Fixing different indentation in web job

* Generating sha keys for compilation cache to be cross plat

* trying deterministic key for compilation cache

* Fixing md5 command

* Trying another method of generating compilation cache

* testing with a hardcoded string

* Changing to a better hardcoded string

* Remove redundant make dir

* Fixing mkdir command in windows and trying new string key

* fixing $$ in sql product compile

* Removin redundant mkdir

* Trying source version var

* Fixing compilation key

* chaning script to powershell

* Adding artifacts to store compiled files
switching to 7zip for windows node cache

* Adding missing step key in web build

* Building not found directories

* Making correct directory

* Switching to vscode's computeNodeModuleCache

* Fixing formatting and making it look more like vscode's pipeline

* Adding back compiled comput cache key

* Fixing cache file

* Fixing copyright message
Adding sql header to custom node cache generator
Updating cache salt to force a cache miss

* Using glob instead of custom method to find all yarn.lock files
Fixing some other pipeline errors.

* Removing unnecessary variable checks.

* Added back VSCODE_STEP_ON_IT check
Moving drop artifacts before compiled files to keep it drop folder free  from compiled files

* Changing task name from cache flags to cache key

* Removing glob from compute node module cache
Fixing copyright message

* checking in updated js

build/azure-pipelines/sql-product-compile.yml

@@ -1,19 +1,4 @@
 steps:
-- script: |
-    mkdir -p .build
-    echo -n $BUILD_SOURCEVERSION > .build/commit
-    echo -n $VSCODE_QUALITY > .build/quality
-  displayName: Prepare cache flag
-
-- task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-  displayName: Restore Cache - Compiled Files
-  inputs:
-    keyfile: 'build/.cachesalt, .build/commit, .build/quality'
-    targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
-    vstsFeed: 'BuildCache'
-    platformIndependent: true
-    alias: 'Compilation'
-
 - task: NodeTool@0
   inputs:
     versionSpec: "12.13.0"
@@ -27,7 +12,6 @@ steps:
   inputs:
     azureSubscription: 'ClientToolsInfra_670062 (88d5392f-a34f-4769-b405-f597fc533613)'
     KeyVaultName: ado-secrets
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -40,7 +24,6 @@ steps:
     git config user.email "sqltools@service.microsoft.com"
     git config user.name "AzureDataStudio"
   displayName: Prepare tooling
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -48,34 +31,44 @@ steps:
     git fetch distro
     git merge $(node -p "require('./package.json').distro")
   displayName: Merge distro
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
-- task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-  displayName: Restore Cache - Node Modules
+- script: |
+    mkdir -p .build
+    node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash
+  displayName: Prepare yarn cache key
+
+- task: Cache@2
   inputs:
-    keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-    targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-    vstsFeed: 'BuildCache'
+    key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash'
+    path: .build/node_modules_cache
+    cacheHitVar: NODE_MODULES_RESTORED
+  displayName: Restore Cache - Node Modules
+
+- script: |
+    set -e
+    tar -xzf .build/node_modules_cache/cache.tgz
+  condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
+  displayName: Extract node_modules archive
 
 - script: |
     set -e
     CHILD_CONCURRENCY=1 yarn --frozen-lockfile
   displayName: Install dependencies
-  condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+  condition: and(succeeded(), ne(variables['NODE_MODULES_RESTORED'], 'true'))
 
-- task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-  displayName: Save Cache - Node Modules
-  inputs:
-    keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-    targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-    vstsFeed: 'BuildCache'
-  condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+- script: |
+    set -e
+    node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt
+    mkdir -p .build/node_modules_cache
+    tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt
+  condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
+  displayName: Create node_modules archive
 
 - script: |
     set -e
     yarn postinstall
   displayName: Run postinstall scripts
-  condition: and(succeeded(), eq(variables['CacheRestored'], 'true'))
+  condition: and(succeeded(), eq(variables['NODE_MODULES_RESTORED'], 'true'))
 
 # Mixin must run before optimize, because the CSS loader will
 # inline small SVGs
@@ -83,7 +76,6 @@ steps:
     set -e
     node build/azure-pipelines/mixin
   displayName: Mix in quality
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -92,7 +84,7 @@ steps:
     yarn strict-vscode
     yarn valid-layers-check
   displayName: Run hygiene, eslint
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
 - script: |
     set -e
@@ -102,7 +94,6 @@ steps:
     yarn gulp vscode-reh-linux-x64-min
     yarn gulp vscode-reh-web-linux-x64-min
   displayName: Compile
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -110,7 +101,6 @@ steps:
     AZURE_STORAGE_ACCESS_KEY="$(sourcemap-storage-key)" \
       node build/azure-pipelines/upload-sourcemaps
   displayName: Upload sourcemaps
-  condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -125,12 +115,13 @@ steps:
 - task: PublishBuildArtifacts@1
   displayName: 'Publish Artifact: drop'
 
-- task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-  displayName: Save Cache - Compiled Files
+- script: |
+    set -e
+    tar -czf $(Build.ArtifactStagingDirectory)/compilation.tar.gz .build out-*
+  displayName: Compress compilation artifact
+
+- task: PublishPipelineArtifact@1
   inputs:
-    keyfile: 'build/.cachesalt, .build/commit, .build/quality'
-    targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
-    vstsFeed: 'BuildCache'
-    platformIndependent: true
-    alias: 'Compilation'
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
+    targetPath: $(Build.ArtifactStagingDirectory)/compilation.tar.gz
+    artifactName: Compilation
+  displayName: Publish compilation artifact