From a11beb71dabc84978cdc878b9fd7dade2cf2f630 Mon Sep 17 00:00:00 2001 From: Aasim Khan Date: Fri, 9 Apr 2021 08:06:39 -0700 Subject: [PATCH] Updating build pipelines to Cache@2 and switching to artifacts for compiled file sharing in pipeline jobs. (#14989) * switiching product compile node cache task to newer version * moving new changes from product-compile to sql-product-compile * changing to yarn.lock as cache key * Adding compilation cache * changing keypath to key * letting find command do the heavy lifting * removing old save cache task * reverting compilation cache to old task * Creating a js to list compiled files switching to cache 2 for compiled files Creating a js file to compute yarn cache * removed unused input targetFolder from pipeline cache task * removed save cache * Fixing compute node modules file * Adding compiled computenodemodules * Fixing checked variables on product compile Updating all pipeline jobs to cache 2 Using tar for windows pipeline. Hoping it works * Fixing indentation in web job * Fixing different indentation in web job * Generating sha keys for compilation cache to be cross plat * trying deterministic key for compilation cache * Fixing md5 command * Trying another method of generating compilation cache * testing with a hardcoded string * Changing to a better hardcoded string * Remove redundant make dir * Fixing mkdir command in windows and trying new string key * fixing $$ in sql product compile * Removin redundant mkdir * Trying source version var * Fixing compilation key * chaning script to powershell * Adding artifacts to store compiled files switching to 7zip for windows node cache * Adding missing step key in web build * Building not found directories * Making correct directory * Switching to vscode's computeNodeModuleCache * Fixing formatting and making it look more like vscode's pipeline * Adding back compiled comput cache key * Fixing cache file * Fixing copyright message Adding sql header to custom node cache generator Updating cache salt to force a cache miss * Using glob instead of custom method to find all yarn.lock files Fixing some other pipeline errors. * Removing unnecessary variable checks. * Added back VSCODE_STEP_ON_IT check Moving drop artifacts before compiled files to keep it drop folder free from compiled files * Changing task name from cache flags to cache key * Removing glob from compute node module cache Fixing copyright message * checking in updated js --- build/.cachesalt | 2 +- .../azure-pipelines/common/listNodeModules.ts | 46 +++++++++++ .../common/sql-computeNodeModulesCacheKey.js | 47 +++++++++++ .../common/sql-computeNodeModulesCacheKey.ts | 54 +++++++++++++ .../darwin/sql-product-build-darwin.yml | 59 +++++++------- .../linux/sql-product-build-linux.yml | 69 ++++++++-------- build/azure-pipelines/sql-product-compile.yml | 79 ++++++++----------- .../web/sql-product-build-web.yml | 33 +++----- .../win32/sql-product-build-win32.yml | 79 ++++++++++--------- 9 files changed, 302 insertions(+), 166 deletions(-) create mode 100644 build/azure-pipelines/common/listNodeModules.ts create mode 100644 build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js create mode 100644 build/azure-pipelines/common/sql-computeNodeModulesCacheKey.ts diff --git a/build/.cachesalt b/build/.cachesalt index 79cdde85d1..5794f86823 100644 --- a/build/.cachesalt +++ b/build/.cachesalt @@ -1 +1 @@ -2020-04-29T05:20:58.491Z +2021-04-07T00:04:17.775Z diff --git a/build/azure-pipelines/common/listNodeModules.ts b/build/azure-pipelines/common/listNodeModules.ts new file mode 100644 index 0000000000..c96df52179 --- /dev/null +++ b/build/azure-pipelines/common/listNodeModules.ts @@ -0,0 +1,46 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import * as fs from 'fs'; +import * as path from 'path'; + +if (process.argv.length !== 3) { + console.error('Usage: node listNodeModules.js OUTPUT_FILE'); + process.exit(-1); +} + +const ROOT = path.join(__dirname, '../../../'); + +function findNodeModulesFiles(location: string, inNodeModules: boolean, result: string[]) { + const entries = fs.readdirSync(path.join(ROOT, location)); + for (const entry of entries) { + const entryPath = `${location}/${entry}`; + + if (/(^\/out)|(^\/src$)|(^\/.git$)|(^\/.build$)/.test(entryPath)) { + continue; + } + + let stat: fs.Stats; + try { + stat = fs.statSync(path.join(ROOT, entryPath)); + } catch (err) { + continue; + } + + if (stat.isDirectory()) { + findNodeModulesFiles(entryPath, inNodeModules || (entry === 'node_modules'), result); + } else { + if (inNodeModules) { + result.push(entryPath.substr(1)); + } + } + } +} + +const result: string[] = []; +findNodeModulesFiles('', false, result); +fs.writeFileSync(process.argv[2], result.join('\n') + '\n'); diff --git a/build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js b/build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js new file mode 100644 index 0000000000..b8b1f6d5af --- /dev/null +++ b/build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js @@ -0,0 +1,47 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; +Object.defineProperty(exports, "__esModule", { value: true }); +const fs = require("fs"); +const path = require("path"); +const crypto = require("crypto"); +const ROOT = path.join(__dirname, '../../../'); +function findFiles(location, pattern, result) { + const entries = fs.readdirSync(path.join(ROOT, location)); + for (const entry of entries) { + const entryPath = `${location}/${entry}`; + let stat; + try { + stat = fs.statSync(path.join(ROOT, entryPath)); + } + catch (err) { + continue; + } + if (stat.isDirectory()) { + findFiles(entryPath, pattern, result); + } + else { + if (stat.isFile() && entry.endsWith(pattern)) { + result.push(path.join(ROOT, entryPath)); + } + } + } +} +const shasum = crypto.createHash('sha1'); +/** + * Creating a sha hash of all the files that can cause packages to change/redownload. + */ +shasum.update(fs.readFileSync(path.join(ROOT, 'build/.cachesalt'))); +shasum.update(fs.readFileSync(path.join(ROOT, '.yarnrc'))); +shasum.update(fs.readFileSync(path.join(ROOT, 'remote/.yarnrc'))); +// Adding all yarn.lock files into sha sum. +const result = []; +findFiles('', 'yarn.lock', result); +result.forEach(f => shasum.update(fs.readFileSync(f))); +// Add any other command line arguments +for (let i = 2; i < process.argv.length; i++) { + shasum.update(process.argv[i]); +} +process.stdout.write(shasum.digest('hex')); diff --git a/build/azure-pipelines/common/sql-computeNodeModulesCacheKey.ts b/build/azure-pipelines/common/sql-computeNodeModulesCacheKey.ts new file mode 100644 index 0000000000..ce4819af92 --- /dev/null +++ b/build/azure-pipelines/common/sql-computeNodeModulesCacheKey.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import * as fs from 'fs'; +import * as path from 'path'; +import * as crypto from 'crypto'; + +const ROOT = path.join(__dirname, '../../../'); + +function findFiles(location: string, pattern: string, result: string[]) { + const entries = fs.readdirSync(path.join(ROOT, location)); + + for (const entry of entries) { + const entryPath = `${location}/${entry}`; + let stat: fs.Stats; + try { + stat = fs.statSync(path.join(ROOT, entryPath)); + } catch (err) { + continue; + } + if (stat.isDirectory()) { + findFiles(entryPath, pattern, result); + } else { + if (stat.isFile() && entry.endsWith(pattern)) { + result.push(path.join(ROOT, entryPath)); + } + } + } +} + +const shasum = crypto.createHash('sha1'); + +/** + * Creating a sha hash of all the files that can cause packages to change/redownload. + */ +shasum.update(fs.readFileSync(path.join(ROOT, 'build/.cachesalt'))); +shasum.update(fs.readFileSync(path.join(ROOT, '.yarnrc'))); +shasum.update(fs.readFileSync(path.join(ROOT, 'remote/.yarnrc'))); + +// Adding all yarn.lock files into sha sum. +const result: string[] = []; +findFiles('', 'yarn.lock', result); +result.forEach(f => shasum.update(fs.readFileSync(f))); + +// Add any other command line arguments +for (let i = 2; i < process.argv.length; i++) { + shasum.update(process.argv[i]); +} + +process.stdout.write(shasum.digest('hex')); diff --git a/build/azure-pipelines/darwin/sql-product-build-darwin.yml b/build/azure-pipelines/darwin/sql-product-build-darwin.yml index 2e07575149..f2be593bac 100644 --- a/build/azure-pipelines/darwin/sql-product-build-darwin.yml +++ b/build/azure-pipelines/darwin/sql-product-build-darwin.yml @@ -5,26 +5,16 @@ steps: certSecureFile: 'osx_signing_key.p12' condition: eq(variables['signed'], true) - - script: | - mkdir -p .build - echo -n $BUILD_SOURCEVERSION > .build/commit - echo -n $VSCODE_QUALITY > .build/quality - displayName: Prepare cache flag - - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: Restore Cache - Compiled Files + - task: DownloadPipelineArtifact@2 inputs: - keyfile: 'build/.cachesalt, .build/commit, .build/quality' - targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min' - vstsFeed: 'BuildCache' - platformIndependent: true - alias: 'Compilation' + artifact: Compilation + path: $(Build.ArtifactStagingDirectory) + displayName: Download compilation output - script: | set -e - exit 1 - displayName: Check RestoreCache - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) + tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz + displayName: Extract compilation output - task: NodeTool@0 inputs: @@ -61,12 +51,23 @@ steps: git merge $(node -p "require('./package.json').distro") displayName: Merge distro - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - script: | + mkdir -p .build + node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash + displayName: Prepare yarn cache key + + - task: Cache@2 displayName: Restore Cache - Node Modules inputs: - keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock' - targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules' - vstsFeed: 'BuildCache' + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + + - script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e @@ -74,21 +75,21 @@ steps: displayName: Install dependencies env: GITHUB_TOKEN: $(github-distro-mixin-password) - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - displayName: Save Cache - Node Modules - inputs: - keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock' - targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules' - vstsFeed: 'BuildCache' - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - script: | set -e yarn postinstall displayName: Run postinstall scripts - condition: and(succeeded(), eq(variables['CacheRestored'], 'true')) + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) - script: | set -e diff --git a/build/azure-pipelines/linux/sql-product-build-linux.yml b/build/azure-pipelines/linux/sql-product-build-linux.yml index 0a15bdb67d..87050f5e68 100644 --- a/build/azure-pipelines/linux/sql-product-build-linux.yml +++ b/build/azure-pipelines/linux/sql-product-build-linux.yml @@ -2,27 +2,6 @@ parameters: extensionsToUnitTest: [] steps: - - script: | - mkdir -p .build - echo -n $BUILD_SOURCEVERSION > .build/commit - echo -n $VSCODE_QUALITY > .build/quality - displayName: Prepare cache flag - - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: Restore Cache - Compiled Files - inputs: - keyfile: 'build/.cachesalt, .build/commit, .build/quality' - targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min' - vstsFeed: 'BuildCache' - platformIndependent: true - alias: 'Compilation' - - - script: | - set -e - exit 1 - displayName: Check RestoreCache - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) - - task: NodeTool@0 inputs: versionSpec: "12.13.0" @@ -38,6 +17,17 @@ steps: KeyVaultName: ado-secrets SecretsFilter: 'github-distro-mixin-password' + - task: DownloadPipelineArtifact@2 + inputs: + artifact: Compilation + path: $(Build.ArtifactStagingDirectory) + displayName: Download compilation output + + - script: | + set -e + tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz + displayName: Extract compilation output + - script: | set -e cat << EOF > ~/.netrc @@ -57,12 +47,23 @@ steps: git merge $(node -p "require('./package.json').distro") displayName: Merge distro - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + - script: | + mkdir -p .build + node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash + displayName: Prepare yarn cache key + + - task: Cache@2 displayName: Restore Cache - Node Modules inputs: - keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock' - targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules' - vstsFeed: 'BuildCache' + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + + - script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e @@ -70,21 +71,21 @@ steps: displayName: Install dependencies env: GITHUB_TOKEN: $(github-distro-mixin-password) - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - displayName: Save Cache - Node Modules - inputs: - keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock' - targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules' - vstsFeed: 'BuildCache' - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - script: | set -e yarn postinstall displayName: Run postinstall scripts - condition: and(succeeded(), eq(variables['CacheRestored'], 'true')) + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) - script: | set -e diff --git a/build/azure-pipelines/sql-product-compile.yml b/build/azure-pipelines/sql-product-compile.yml index 09613597bd..a612e29c5a 100644 --- a/build/azure-pipelines/sql-product-compile.yml +++ b/build/azure-pipelines/sql-product-compile.yml @@ -1,19 +1,4 @@ steps: -- script: | - mkdir -p .build - echo -n $BUILD_SOURCEVERSION > .build/commit - echo -n $VSCODE_QUALITY > .build/quality - displayName: Prepare cache flag - -- task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: Restore Cache - Compiled Files - inputs: - keyfile: 'build/.cachesalt, .build/commit, .build/quality' - targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min' - vstsFeed: 'BuildCache' - platformIndependent: true - alias: 'Compilation' - - task: NodeTool@0 inputs: versionSpec: "12.13.0" @@ -27,7 +12,6 @@ steps: inputs: azureSubscription: 'ClientToolsInfra_670062 (88d5392f-a34f-4769-b405-f597fc533613)' KeyVaultName: ado-secrets - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) - script: | set -e @@ -40,7 +24,6 @@ steps: git config user.email "sqltools@service.microsoft.com" git config user.name "AzureDataStudio" displayName: Prepare tooling - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) - script: | set -e @@ -48,34 +31,44 @@ steps: git fetch distro git merge $(node -p "require('./package.json').distro") displayName: Merge distro - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) -- task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: Restore Cache - Node Modules +- script: | + mkdir -p .build + node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash + displayName: Prepare yarn cache key + +- task: Cache@2 inputs: - keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock' - targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules' - vstsFeed: 'BuildCache' + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Restore Cache - Node Modules + +- script: | + set -e + tar -xzf .build/node_modules_cache/cache.tgz + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - script: | set -e CHILD_CONCURRENCY=1 yarn --frozen-lockfile displayName: Install dependencies - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables['NODE_MODULES_RESTORED'], 'true')) -- task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - displayName: Save Cache - Node Modules - inputs: - keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock' - targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules' - vstsFeed: 'BuildCache' - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) +- script: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - script: | set -e yarn postinstall displayName: Run postinstall scripts - condition: and(succeeded(), eq(variables['CacheRestored'], 'true')) + condition: and(succeeded(), eq(variables['NODE_MODULES_RESTORED'], 'true')) # Mixin must run before optimize, because the CSS loader will # inline small SVGs @@ -83,7 +76,6 @@ steps: set -e node build/azure-pipelines/mixin displayName: Mix in quality - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) - script: | set -e @@ -92,7 +84,7 @@ steps: yarn strict-vscode yarn valid-layers-check displayName: Run hygiene, eslint - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) + condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - script: | set -e @@ -102,7 +94,6 @@ steps: yarn gulp vscode-reh-linux-x64-min yarn gulp vscode-reh-web-linux-x64-min displayName: Compile - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) - script: | set -e @@ -110,7 +101,6 @@ steps: AZURE_STORAGE_ACCESS_KEY="$(sourcemap-storage-key)" \ node build/azure-pipelines/upload-sourcemaps displayName: Upload sourcemaps - condition: and(succeeded(), ne(variables['CacheExists-Compilation'], 'true')) - script: | set -e @@ -125,12 +115,13 @@ steps: - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: drop' -- task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - displayName: Save Cache - Compiled Files +- script: | + set -e + tar -czf $(Build.ArtifactStagingDirectory)/compilation.tar.gz .build out-* + displayName: Compress compilation artifact + +- task: PublishPipelineArtifact@1 inputs: - keyfile: 'build/.cachesalt, .build/commit, .build/quality' - targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min' - vstsFeed: 'BuildCache' - platformIndependent: true - alias: 'Compilation' - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) + targetPath: $(Build.ArtifactStagingDirectory)/compilation.tar.gz + artifactName: Compilation + displayName: Publish compilation artifact diff --git a/build/azure-pipelines/web/sql-product-build-web.yml b/build/azure-pipelines/web/sql-product-build-web.yml index bec0649d1c..c396d8307c 100644 --- a/build/azure-pipelines/web/sql-product-build-web.yml +++ b/build/azure-pipelines/web/sql-product-build-web.yml @@ -1,25 +1,5 @@ + steps: -- script: | - mkdir -p .build - echo -n $BUILD_SOURCEVERSION > .build/commit - echo -n $VSCODE_QUALITY > .build/quality - displayName: Prepare cache flag - -- task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: Restore Cache - Compiled Files - inputs: - keyfile: 'build/.cachesalt, .build/commit, .build/quality' - targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min' - vstsFeed: 'BuildCache' - platformIndependent: true - alias: 'Compilation' - -- script: | - set -e - exit 1 - displayName: Check RestoreCache - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) - - task: NodeTool@0 inputs: versionSpec: "12.13.0" @@ -35,6 +15,17 @@ steps: KeyVaultName: ado-secrets SecretsFilter: 'github-distro-mixin-password' +- task: DownloadPipelineArtifact@2 + inputs: + artifact: Compilation + path: $(Build.ArtifactStagingDirectory) + displayName: Download compilation output + +- script: | + set -e + tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz + displayName: Extract compilation output + - script: | set -e cat << EOF > ~/.netrc diff --git a/build/azure-pipelines/win32/sql-product-build-win32.yml b/build/azure-pipelines/win32/sql-product-build-win32.yml index 0243e8d1f1..9fbe509866 100644 --- a/build/azure-pipelines/win32/sql-product-build-win32.yml +++ b/build/azure-pipelines/win32/sql-product-build-win32.yml @@ -1,25 +1,4 @@ steps: - - powershell: | - mkdir .build -ea 0 - "$env:BUILD_SOURCEVERSION" | Out-File -Encoding ascii -NoNewLine .build\commit - "$env:VSCODE_QUALITY" | Out-File -Encoding ascii -NoNewLine .build\quality - displayName: Prepare cache flag - - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: Restore Cache - Compiled Files - inputs: - keyfile: 'build/.cachesalt, .build/commit, .build/quality' - targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min' - vstsFeed: 'BuildCache' - platformIndependent: true - alias: 'Compilation' - - - powershell: | - $ErrorActionPreference = "Stop" - exit 1 - displayName: Check RestoreCache - condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true')) - - task: NodeTool@0 inputs: versionSpec: "12.13.0" @@ -34,11 +13,23 @@ steps: addToPath: true - task: AzureKeyVault@1 - displayName: 'Azure Key Vault: Get Secrets' inputs: azureSubscription: 'ClientToolsInfra_670062 (88d5392f-a34f-4769-b405-f597fc533613)' KeyVaultName: ado-secrets SecretsFilter: 'github-distro-mixin-password' + displayName: 'Azure Key Vault: Get Secrets' + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: Compilation + path: $(Build.ArtifactStagingDirectory) + displayName: Download compilation output + + - powershell: | + . build/azure-pipelines/win32/exec.ps1 + $ErrorActionPreference = "Stop" + exec { tar --force-local -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz } + displayName: Extract compilation output - powershell: | . build/azure-pipelines/win32/exec.ps1 @@ -55,37 +46,51 @@ steps: git merge $(node -p "require('./package.json').distro") displayName: Merge distro - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: Restore Cache - Node Modules + - powershell: | + . build/azure-pipelines/win32/exec.ps1 + $ErrorActionPreference = "Stop" + exec { node build/azure-pipelines/common/sql-computeNodeModulesCacheKey.js > .build/yarnlockhash } + displayName: Prepare yarn cache key + + - task: Cache@2 inputs: - keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock' - targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules' - vstsFeed: 'BuildCache' + key: 'nodeModules | $(Agent.OS) | .build/yarnlockhash' + path: .build/node_modules_cache + cacheHitVar: NODE_MODULES_RESTORED + displayName: Restore Cache - Node Modules + + - powershell: | + . build/azure-pipelines/win32/exec.ps1 + $ErrorActionPreference = "Stop" + exec { 7z.exe x .build/node_modules_cache/cache.7z -aos } + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Extract node_modules archive - powershell: | . build/azure-pipelines/win32/exec.ps1 $ErrorActionPreference = "Stop" $env:CHILD_CONCURRENCY="1" exec { yarn --frozen-lockfile } - displayName: Install dependencies env: GITHUB_TOKEN: $(github-distro-mixin-password) - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Install dependencies - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - displayName: Save Cache - Node Modules - inputs: - keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock' - targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules' - vstsFeed: 'BuildCache' - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) + - powershell: | + . build/azure-pipelines/win32/exec.ps1 + $ErrorActionPreference = "Stop" + exec { node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt } + exec { mkdir -Force .build/node_modules_cache } + exec { 7z.exe a .build/node_modules_cache/cache.7z -mx3 `@.build/node_modules_list.txt } + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + displayName: Create node_modules archive - powershell: | . build/azure-pipelines/win32/exec.ps1 $ErrorActionPreference = "Stop" exec { yarn postinstall } displayName: Run postinstall scripts - condition: and(succeeded(), eq(variables['CacheRestored'], 'true')) + condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true')) - powershell: | . build/azure-pipelines/win32/exec.ps1