Updating build pipelines to Cache@2 and switching to artifacts for compiled file sharing in pipeline jobs. (#14989)

* switiching product compile node cache task to newer version

* moving new changes from product-compile to sql-product-compile

* changing to yarn.lock as cache key

* Adding compilation cache

* changing keypath to key

* letting find command do the heavy lifting

* removing old save cache task

* reverting compilation cache to old task

* Creating a js to list compiled files
switching to cache 2 for compiled files
Creating a js file to compute yarn cache

* removed unused input targetFolder from pipeline cache task

* removed save cache

* Fixing compute node modules file

* Adding compiled computenodemodules

* Fixing checked variables on product compile
Updating all pipeline jobs to cache 2
Using tar for windows pipeline. Hoping it works

* Fixing indentation in web job

* Fixing different indentation in web job

* Generating sha keys for compilation cache to be cross plat

* trying deterministic key for compilation cache

* Fixing md5 command

* Trying another method of generating compilation cache

* testing with a hardcoded string

* Changing to a better hardcoded string

* Remove redundant make dir

* Fixing mkdir command in windows and trying new string key

* fixing $$ in sql product compile

* Removin redundant mkdir

* Trying source version var

* Fixing compilation key

* chaning script to powershell

* Adding artifacts to store compiled files
switching to 7zip for windows node cache

* Adding missing step key in web build

* Building not found directories

* Making correct directory

* Switching to vscode's computeNodeModuleCache

* Fixing formatting and making it look more like vscode's pipeline

* Adding back compiled comput cache key

* Fixing cache file

* Fixing copyright message
Adding sql header to custom node cache generator
Updating cache salt to force a cache miss

* Using glob instead of custom method to find all yarn.lock files
Fixing some other pipeline errors.

* Removing unnecessary variable checks.

* Added back VSCODE_STEP_ON_IT check
Moving drop artifacts before compiled files to keep it drop folder free  from compiled files

* Changing task name from cache flags to cache key

* Removing glob from compute node module cache
Fixing copyright message

* checking in updated js

build/.cachesalt

@@ -1 +1 @@
-2020-04-29T05:20:58.491Z
+2021-04-07T00:04:17.775Z

build/azure-pipelines/common/listNodeModules.ts

@@ -0,0 +1,46 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+if (process.argv.length !== 3) {
+	console.error('Usage: node listNodeModules.js OUTPUT_FILE');
+	process.exit(-1);
+}
+
+const ROOT = path.join(__dirname, '../../../');
+
+function findNodeModulesFiles(location: string, inNodeModules: boolean, result: string[]) {
+	const entries = fs.readdirSync(path.join(ROOT, location));
+	for (const entry of entries) {
+		const entryPath = `${location}/${entry}`;
+
+		if (/(^\/out)|(^\/src$)|(^\/.git$)|(^\/.build$)/.test(entryPath)) {
+			continue;
+		}
+
+		let stat: fs.Stats;
+		try {
+			stat = fs.statSync(path.join(ROOT, entryPath));
+		} catch (err) {
+			continue;
+		}
+
+		if (stat.isDirectory()) {
+			findNodeModulesFiles(entryPath, inNodeModules || (entry === 'node_modules'), result);
+		} else {
+			if (inNodeModules) {
+				result.push(entryPath.substr(1));
+			}
+		}
+	}
+}
+
+const result: string[] = [];
+findNodeModulesFiles('', false, result);
+fs.writeFileSync(process.argv[2], result.join('\n') + '\n');

build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js

@@ -0,0 +1,47 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+const ROOT = path.join(__dirname, '../../../');
+function findFiles(location, pattern, result) {
+    const entries = fs.readdirSync(path.join(ROOT, location));
+    for (const entry of entries) {
+        const entryPath = `${location}/${entry}`;
+        let stat;
+        try {
+            stat = fs.statSync(path.join(ROOT, entryPath));
+        }
+        catch (err) {
+            continue;
+        }
+        if (stat.isDirectory()) {
+            findFiles(entryPath, pattern, result);
+        }
+        else {
+            if (stat.isFile() && entry.endsWith(pattern)) {
+                result.push(path.join(ROOT, entryPath));
+            }
+        }
+    }
+}
+const shasum = crypto.createHash('sha1');
+/**
+ * Creating a sha hash of all the files that can cause packages to change/redownload.
+ */
+shasum.update(fs.readFileSync(path.join(ROOT, 'build/.cachesalt')));
+shasum.update(fs.readFileSync(path.join(ROOT, '.yarnrc')));
+shasum.update(fs.readFileSync(path.join(ROOT, 'remote/.yarnrc')));
+// Adding all yarn.lock files into sha sum.
+const result = [];
+findFiles('', 'yarn.lock', result);
+result.forEach(f => shasum.update(fs.readFileSync(f)));
+// Add any other command line arguments
+for (let i = 2; i < process.argv.length; i++) {
+    shasum.update(process.argv[i]);
+}
+process.stdout.write(shasum.digest('hex'));

build/azure-pipelines/common/sql-computeNodeModulesCacheKey.ts

@@ -0,0 +1,54 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+
+const ROOT = path.join(__dirname, '../../../');
+
+function findFiles(location: string, pattern: string, result: string[]) {
+	const entries = fs.readdirSync(path.join(ROOT, location));
+
+	for (const entry of entries) {
+		const entryPath = `${location}/${entry}`;
+		let stat: fs.Stats;
+		try {
+			stat = fs.statSync(path.join(ROOT, entryPath));
+		} catch (err) {
+			continue;
+		}
+		if (stat.isDirectory()) {
+			findFiles(entryPath, pattern, result);
+		} else {
+			if (stat.isFile() && entry.endsWith(pattern)) {
+				result.push(path.join(ROOT, entryPath));
+			}
+		}
+	}
+}
+
+const shasum = crypto.createHash('sha1');
+
+/**
+ * Creating a sha hash of all the files that can cause packages to change/redownload.
+ */
+shasum.update(fs.readFileSync(path.join(ROOT, 'build/.cachesalt')));
+shasum.update(fs.readFileSync(path.join(ROOT, '.yarnrc')));
+shasum.update(fs.readFileSync(path.join(ROOT, 'remote/.yarnrc')));
+
+// Adding all yarn.lock files into sha sum.
+const result: string[] = [];
+findFiles('', 'yarn.lock', result);
+result.forEach(f => shasum.update(fs.readFileSync(f)));
+
+// Add any other command line arguments
+for (let i = 2; i < process.argv.length; i++) {
+	shasum.update(process.argv[i]);
+}
+
+process.stdout.write(shasum.digest('hex'));

build/azure-pipelines/darwin/sql-product-build-darwin.yml

@@ -5,26 +5,16 @@ steps:
       certSecureFile: 'osx_signing_key.p12'
     condition: eq(variables['signed'], true)
 
-  - script: |
-      mkdir -p .build
-      echo -n $BUILD_SOURCEVERSION > .build/commit
-      echo -n $VSCODE_QUALITY > .build/quality
-    displayName: Prepare cache flag
-
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-    displayName: Restore Cache - Compiled Files
+  - task: DownloadPipelineArtifact@2
     inputs:
-      keyfile: 'build/.cachesalt, .build/commit, .build/quality'
-      targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
-      vstsFeed: 'BuildCache'
-      platformIndependent: true
-      alias: 'Compilation'
+      artifact: Compilation
+      path: $(Build.ArtifactStagingDirectory)
+    displayName: Download compilation output
 
   - script: |
       set -e
-      exit 1
-    displayName: Check RestoreCache
-    condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
+      tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz
+    displayName: Extract compilation output
 
   - task: NodeTool@0
     inputs:
@@ -61,12 +51,23 @@ steps:
       git merge $(node -p "require('./package.json').distro")
     displayName: Merge distro
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
+  - script: |
+      mkdir -p .build
+      node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash
+    displayName: Prepare yarn cache key
+
+  - task: Cache@2
     displayName: Restore Cache - Node Modules
     inputs:
-      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-      targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-      vstsFeed: 'BuildCache'
+      key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash'
+      path: .build/node_modules_cache
+      cacheHitVar: NODE_MODULES_RESTORED
+
+  - script: |
+      set -e
+      tar -xzf .build/node_modules_cache/cache.tgz
+    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Extract node_modules archive
 
   - script: |
       set -e
@@ -74,21 +75,21 @@ steps:
     displayName: Install dependencies
     env:
       GITHUB_TOKEN: $(github-distro-mixin-password)
-    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-    displayName: Save Cache - Node Modules
-    inputs:
-      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-      targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-      vstsFeed: 'BuildCache'
-    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+  - script: |
+      set -e
+      node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt
+      mkdir -p .build/node_modules_cache
+      tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Create node_modules archive
 
   - script: |
       set -e
       yarn postinstall
     displayName: Run postinstall scripts
-    condition: and(succeeded(), eq(variables['CacheRestored'], 'true'))
+    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
 
   - script: |
       set -e

build/azure-pipelines/linux/sql-product-build-linux.yml

@@ -2,27 +2,6 @@ parameters:
   extensionsToUnitTest: []
 
 steps:
-  - script: |
-      mkdir -p .build
-      echo -n $BUILD_SOURCEVERSION > .build/commit
-      echo -n $VSCODE_QUALITY > .build/quality
-    displayName: Prepare cache flag
-
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-    displayName: Restore Cache - Compiled Files
-    inputs:
-      keyfile: 'build/.cachesalt, .build/commit, .build/quality'
-      targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
-      vstsFeed: 'BuildCache'
-      platformIndependent: true
-      alias: 'Compilation'
-
-  - script: |
-      set -e
-      exit 1
-    displayName: Check RestoreCache
-    condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
-
   - task: NodeTool@0
     inputs:
       versionSpec: "12.13.0"
@@ -38,6 +17,17 @@ steps:
       KeyVaultName: ado-secrets
       SecretsFilter: 'github-distro-mixin-password'
 
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifact: Compilation
+      path: $(Build.ArtifactStagingDirectory)
+    displayName: Download compilation output
+
+  - script: |
+      set -e
+      tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz
+    displayName: Extract compilation output
+
   - script: |
       set -e
       cat << EOF > ~/.netrc
@@ -57,12 +47,23 @@ steps:
       git merge $(node -p "require('./package.json').distro")
     displayName: Merge distro
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
+  - script: |
+      mkdir -p .build
+      node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash
+    displayName: Prepare yarn cache key
+
+  - task: Cache@2
     displayName: Restore Cache - Node Modules
     inputs:
-      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-      targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-      vstsFeed: 'BuildCache'
+      key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash'
+      path: .build/node_modules_cache
+      cacheHitVar: NODE_MODULES_RESTORED
+
+  - script: |
+      set -e
+      tar -xzf .build/node_modules_cache/cache.tgz
+    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Extract node_modules archive
 
   - script: |
       set -e
@@ -70,21 +71,21 @@ steps:
     displayName: Install dependencies
     env:
       GITHUB_TOKEN: $(github-distro-mixin-password)
-    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-    displayName: Save Cache - Node Modules
-    inputs:
-      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-      targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-      vstsFeed: 'BuildCache'
-    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+  - script: |
+      set -e
+      node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt
+      mkdir -p .build/node_modules_cache
+      tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Create node_modules archive
 
   - script: |
       set -e
       yarn postinstall
     displayName: Run postinstall scripts
-    condition: and(succeeded(), eq(variables['CacheRestored'], 'true'))
+    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
 
   - script: |
       set -e

build/azure-pipelines/sql-product-compile.yml

@@ -1,19 +1,4 @@
 steps:
-- script: |
-    mkdir -p .build
-    echo -n $BUILD_SOURCEVERSION > .build/commit
-    echo -n $VSCODE_QUALITY > .build/quality
-  displayName: Prepare cache flag
-
-- task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-  displayName: Restore Cache - Compiled Files
-  inputs:
-    keyfile: 'build/.cachesalt, .build/commit, .build/quality'
-    targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
-    vstsFeed: 'BuildCache'
-    platformIndependent: true
-    alias: 'Compilation'
-
 - task: NodeTool@0
   inputs:
     versionSpec: "12.13.0"
@@ -27,7 +12,6 @@ steps:
   inputs:
     azureSubscription: 'ClientToolsInfra_670062 (88d5392f-a34f-4769-b405-f597fc533613)'
     KeyVaultName: ado-secrets
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -40,7 +24,6 @@ steps:
     git config user.email "sqltools@service.microsoft.com"
     git config user.name "AzureDataStudio"
   displayName: Prepare tooling
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -48,34 +31,44 @@ steps:
     git fetch distro
     git merge $(node -p "require('./package.json').distro")
   displayName: Merge distro
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
-- task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-  displayName: Restore Cache - Node Modules
+- script: |
+    mkdir -p .build
+    node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash
+  displayName: Prepare yarn cache key
+
+- task: Cache@2
   inputs:
-    keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-    targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-    vstsFeed: 'BuildCache'
+    key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash'
+    path: .build/node_modules_cache
+    cacheHitVar: NODE_MODULES_RESTORED
+  displayName: Restore Cache - Node Modules
+
+- script: |
+    set -e
+    tar -xzf .build/node_modules_cache/cache.tgz
+  condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
+  displayName: Extract node_modules archive
 
 - script: |
     set -e
     CHILD_CONCURRENCY=1 yarn --frozen-lockfile
   displayName: Install dependencies
-  condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+  condition: and(succeeded(), ne(variables['NODE_MODULES_RESTORED'], 'true'))
 
-- task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-  displayName: Save Cache - Node Modules
-  inputs:
-    keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-    targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-    vstsFeed: 'BuildCache'
-  condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+- script: |
+    set -e
+    node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt
+    mkdir -p .build/node_modules_cache
+    tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt
+  condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
+  displayName: Create node_modules archive
 
 - script: |
     set -e
     yarn postinstall
   displayName: Run postinstall scripts
-  condition: and(succeeded(), eq(variables['CacheRestored'], 'true'))
+  condition: and(succeeded(), eq(variables['NODE_MODULES_RESTORED'], 'true'))
 
 # Mixin must run before optimize, because the CSS loader will
 # inline small SVGs
@@ -83,7 +76,6 @@ steps:
     set -e
     node build/azure-pipelines/mixin
   displayName: Mix in quality
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -92,7 +84,7 @@ steps:
     yarn strict-vscode
     yarn valid-layers-check
   displayName: Run hygiene, eslint
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
 - script: |
     set -e
@@ -102,7 +94,6 @@ steps:
     yarn gulp vscode-reh-linux-x64-min
     yarn gulp vscode-reh-web-linux-x64-min
   displayName: Compile
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -110,7 +101,6 @@ steps:
     AZURE_STORAGE_ACCESS_KEY="$(sourcemap-storage-key)" \
       node build/azure-pipelines/upload-sourcemaps
   displayName: Upload sourcemaps
-  condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true'))
 
 - script: |
     set -e
@@ -125,12 +115,13 @@ steps:
 - task: PublishBuildArtifacts@1
   displayName: 'Publish Artifact: drop'
 
-- task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-  displayName: Save Cache - Compiled Files
+- script: |
+    set -e
+    tar -czf $(Build.ArtifactStagingDirectory)/compilation.tar.gz .build out-*
+  displayName: Compress compilation artifact
+
+- task: PublishPipelineArtifact@1
   inputs:
-    keyfile: 'build/.cachesalt, .build/commit, .build/quality'
-    targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
-    vstsFeed: 'BuildCache'
-    platformIndependent: true
-    alias: 'Compilation'
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
+    targetPath: $(Build.ArtifactStagingDirectory)/compilation.tar.gz
+    artifactName: Compilation
+  displayName: Publish compilation artifact

build/azure-pipelines/web/sql-product-build-web.yml

@@ -1,25 +1,5 @@
+
 steps:
-- script: |
-    mkdir -p .build
-    echo -n $BUILD_SOURCEVERSION > .build/commit
-    echo -n $VSCODE_QUALITY > .build/quality
-  displayName: Prepare cache flag
-
-- task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-  displayName: Restore Cache - Compiled Files
-  inputs:
-    keyfile: 'build/.cachesalt, .build/commit, .build/quality'
-    targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
-    vstsFeed: 'BuildCache'
-    platformIndependent: true
-    alias: 'Compilation'
-
-- script: |
-    set -e
-    exit 1
-  displayName: Check RestoreCache
-  condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
-
 - task: NodeTool@0
   inputs:
     versionSpec: "12.13.0"
@@ -35,6 +15,17 @@ steps:
     KeyVaultName: ado-secrets
     SecretsFilter: 'github-distro-mixin-password'
 
+- task: DownloadPipelineArtifact@2
+  inputs:
+    artifact: Compilation
+    path: $(Build.ArtifactStagingDirectory)
+  displayName: Download compilation output
+
+- script: |
+    set -e
+    tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz
+  displayName: Extract compilation output
+
 - script: |
     set -e
     cat << EOF > ~/.netrc

build/azure-pipelines/win32/sql-product-build-win32.yml

@@ -1,25 +1,4 @@
 steps:
-  - powershell: |
-      mkdir .build -ea 0
-      "$env:BUILD_SOURCEVERSION" | Out-File -Encoding ascii -NoNewLine .build\commit
-      "$env:VSCODE_QUALITY" | Out-File -Encoding ascii -NoNewLine .build\quality
-    displayName: Prepare cache flag
-
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-    displayName: Restore Cache - Compiled Files
-    inputs:
-      keyfile: 'build/.cachesalt, .build/commit, .build/quality'
-      targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
-      vstsFeed: 'BuildCache'
-      platformIndependent: true
-      alias: 'Compilation'
-
-  - powershell: |
-      $ErrorActionPreference = "Stop"
-      exit 1
-    displayName: Check RestoreCache
-    condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
-
   - task: NodeTool@0
     inputs:
       versionSpec: "12.13.0"
@@ -34,11 +13,23 @@ steps:
       addToPath: true
 
   - task: AzureKeyVault@1
-    displayName: 'Azure Key Vault: Get Secrets'
     inputs:
       azureSubscription: 'ClientToolsInfra_670062 (88d5392f-a34f-4769-b405-f597fc533613)'
       KeyVaultName: ado-secrets
       SecretsFilter: 'github-distro-mixin-password'
+    displayName: 'Azure Key Vault: Get Secrets'
+
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifact: Compilation
+      path: $(Build.ArtifactStagingDirectory)
+    displayName: Download compilation output
+
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      exec { tar --force-local -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz }
+    displayName: Extract compilation output
 
   - powershell: |
       . build/azure-pipelines/win32/exec.ps1
@@ -55,37 +46,51 @@ steps:
       git merge $(node -p "require('./package.json').distro")
     displayName: Merge distro
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-    displayName: Restore Cache - Node Modules
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      exec { node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash }
+    displayName: Prepare yarn cache key
+
+  - task: Cache@2
     inputs:
-      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-      targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-      vstsFeed: 'BuildCache'
+      key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash'
+      path: .build/node_modules_cache
+      cacheHitVar: NODE_MODULES_RESTORED
+    displayName: Restore Cache - Node Modules
+
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      exec { 7z.exe x .build/node_modules_cache/cache.7z -aos }
+    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Extract node_modules archive
 
   - powershell: |
       . build/azure-pipelines/win32/exec.ps1
       $ErrorActionPreference = "Stop"
       $env:CHILD_CONCURRENCY="1"
       exec { yarn --frozen-lockfile }
-    displayName: Install dependencies
     env:
       GITHUB_TOKEN: $(github-distro-mixin-password)
-    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Install dependencies
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-    displayName: Save Cache - Node Modules
-    inputs:
-      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-      targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-      vstsFeed: 'BuildCache'
-    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      exec { node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt }
+      exec { mkdir -Force .build/node_modules_cache }
+      exec { 7z.exe a .build/node_modules_cache/cache.7z -mx3 `@.build/node_modules_list.txt }
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Create node_modules archive
 
   - powershell: |
       . build/azure-pipelines/win32/exec.ps1
       $ErrorActionPreference = "Stop"
       exec { yarn postinstall }
     displayName: Run postinstall scripts
-    condition: and(succeeded(), eq(variables['CacheRestored'], 'true'))
+    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
 
   - powershell: |
       . build/azure-pipelines/win32/exec.ps1