Merge from vscode a348d103d1256a06a2c9b3f9b406298a9fef6898 (#15681)

* Merge from vscode a348d103d1256a06a2c9b3f9b406298a9fef6898

* Fixes and cleanup

* Distro

* Fix hygiene yarn

* delete no yarn lock changes file

* Fix hygiene

* Fix layer check

* Fix CI

* Skip lib checks

* Remove tests deleted in vs code

* Fix tests

* Distro

* Fix tests and add removed extension point

* Skip failing notebook tests for now

* Disable broken tests and cleanup build folder

* Update yarn.lock and fix smoke tests

* Bump sqlite

* fix contributed actions and file spacing

* Fix user data path

* Update yarn.locks

Co-authored-by: ADS Merger <karlb@microsoft.com>

build/azure-pipelines/product-compile.yml

@@ -1,7 +1,7 @@
 steps:
   - task: NodeTool@0
     inputs:
-      versionSpec: "12.18.3"
+      versionSpec: "14.x"
 
   - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@2
     inputs:
@@ -30,39 +30,41 @@ steps:
       git pull --no-rebase https://github.com/$(VSCODE_MIXIN_REPO).git $(node -p "require('./package.json').distro")
     displayName: Merge distro
 
-  - script: |
-      npx https://aka.ms/enablesecurefeed standAlone
-    displayName: Switch to Terrapin packages
-    timeoutInMinutes: 5
-    condition: and(succeeded(), eq(variables['ENABLE_TERRAPIN'], 'true'))
-
   - script: |
       mkdir -p .build
-      echo -n $(VSCODE_ARCH) > .build/arch
-      echo -n $ENABLE_TERRAPIN > .build/terrapin
+      node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash
     displayName: Prepare yarn cache flags
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
+  # using `genericNodeModules` instead of `nodeModules` here to avoid sharing the cache with builds running inside containers
+  - task: Cache@2
     inputs:
-      keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock"
-      targetfolder: "**/node_modules, !**/node_modules/**/node_modules"
-      vstsFeed: "npm-vscode"
+      key: 'genericNodeModules | $(Agent.OS) | .build/yarnlockhash'
+      path: .build/node_modules_cache
+      cacheHitVar: NODE_MODULES_RESTORED
+    displayName: Restore node_modules cache
 
   - script: |
       set -e
-      export npm_config_arch=$(NPM_ARCH)
+      tar -xzf .build/node_modules_cache/cache.tgz
+    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Extract node_modules cache
 
-      if [ -z "$CC" ] || [ -z "$CXX" ]; then
-        export CC=$(which gcc-5)
-        export CXX=$(which g++-5)
-      fi
+  - script: |
+      set -e
+      npx https://aka.ms/enablesecurefeed standAlone
+    timeoutInMinutes: 5
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), eq(variables['ENABLE_TERRAPIN'], 'true'))
+    displayName: Switch to Terrapin packages
 
-      if [ "$VSCODE_ARCH" == "x64" ]; then
-        export VSCODE_REMOTE_CC=$(which gcc-4.8)
-        export VSCODE_REMOTE_CXX=$(which g++-4.8)
-        export VSCODE_REMOTE_NODE_GYP=$(which node-gyp)
-      fi
+  - script: |
+      set -e
+      sudo apt update -y
+      sudo apt install -y build-essential pkg-config libx11-dev libx11-xcb-dev libxkbfile-dev libsecret-1-dev libnotify-bin
+    displayName: Install build tools
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
 
+  - script: |
+      set -e
       for i in {1..3}; do # try 3 times, for Terrapin
         yarn --frozen-lockfile && break
         if [ $i -eq 3 ]; then
@@ -75,14 +77,15 @@ steps:
       ELECTRON_SKIP_BINARY_DOWNLOAD: 1
       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     displayName: Install dependencies
-    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
-    inputs:
-      keyfile: ".build/arch, .build/terrapin, build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock"
-      targetfolder: "**/node_modules, !**/node_modules/**/node_modules"
-      vstsFeed: "npm-vscode"
-    condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
+  - script: |
+      set -e
+      node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt
+      mkdir -p .build/node_modules_cache
+      tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt
+    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
+    displayName: Create node_modules archive
 
   # Mixin must run before optimize, because the CSS loader will inline small SVGs
   - script: |
@@ -92,12 +95,8 @@ steps:
 
   - script: |
       set -e
-      yarn gulp compile-build
-      yarn gulp compile-extensions-build
-      yarn gulp minify-vscode
-      yarn gulp vscode-reh-linux-x64-min
-      yarn gulp vscode-reh-web-linux-x64-min
-    displayName: Compile
+      yarn npm-run-all -lp core-ci extensions-ci hygiene eslint valid-layers-check
+    displayName: Compile & Hygiene
 
   - script: |
       set -e
@@ -106,6 +105,19 @@ steps:
     displayName: Upload sourcemaps
     condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
 
+  - script: |
+      set -
+      ./build/azure-pipelines/common/extract-telemetry.sh
+    displayName: Extract Telemetry
+    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
+
+  - script: |
+      set -e
+      AZURE_WEBVIEW_STORAGE_ACCESS_KEY="$(vscode-webview-storage-key)" \
+      ./build/azure-pipelines/common/publish-webview.sh
+    displayName: Publish Webview
+    condition: and(succeeded(), ne(variables['VSCODE_PUBLISH'], 'false'))
+
   - script: |
       set -e
       VERSION=`node -p "require(\"./package.json\").version"`
@@ -125,3 +137,15 @@ steps:
       targetPath: $(Build.ArtifactStagingDirectory)/compilation.tar.gz
       artifactName: Compilation
     displayName: Publish compilation artifact
+
+  - script: |
+      set -e
+      VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
+        yarn download-builtin-extensions-cg
+    displayName: Built-in extensions component details
+
+  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+    displayName: "Component Detection"
+    inputs:
+      sourceScanPath: $(Build.SourcesDirectory)
+    continueOnError: true