Re-enable extension media building (#21041)

* Re-enable extension media building * Fix build
2026-01-14 01:25:37 -05:00 · 2022-10-31 10:25:56 -07:00
parent 275a772496
commit 4ade808695
6 changed files with 259 additions and 186 deletions
--- a/build/gulpfile.extensions.js
+++ b/build/gulpfile.extensions.js
@@ -20,6 +20,7 @@ const root = path.dirname(__dirname);
 const commit = util.getVersion(root);
 const plumber = require('gulp-plumber');
 const ext = require('./lib/extensions');
+const product = require('../product.json');

 const extensionsPath = path.join(path.dirname(__dirname), 'extensions');

@@ -34,48 +35,51 @@ const compilations = glob.sync('**/tsconfig.json', {
 	cwd: extensionsPath,
 	ignore: ['**/out/**', '**/node_modules/**']
 });
-// const compilations = [
-// 'authentication-proxy/tsconfig.json',
-// 'configuration-editing/build/tsconfig.json',
-// 'configuration-editing/tsconfig.json',
-// 'css-language-features/client/tsconfig.json',
-// 'css-language-features/server/tsconfig.json',
-// 'debug-auto-launch/tsconfig.json',
-// 'debug-server-ready/tsconfig.json',
-// 'emmet/tsconfig.json',
-// 'extension-editing/tsconfig.json',
-// 'git/tsconfig.json',
-// 'git-base/tsconfig.json',
-// 'github-authentication/tsconfig.json',
-// 'github/tsconfig.json',
-// 'grunt/tsconfig.json',
-// 'gulp/tsconfig.json',
-// 'html-language-features/client/tsconfig.json',
-// 'html-language-features/server/tsconfig.json',
-// 'image-preview/tsconfig.json',
-// 'ipynb/tsconfig.json',
-// 'jake/tsconfig.json',
-// 'json-language-features/client/tsconfig.json',
-// 'json-language-features/server/tsconfig.json',
-// 'markdown-language-features/preview-src/tsconfig.json',
-// 'markdown-language-features/tsconfig.json',
-// 'markdown-math/tsconfig.json',
-// 'merge-conflict/tsconfig.json',
-// 'microsoft-authentication/tsconfig.json',
-// 'npm/tsconfig.json',
-// 'php-language-features/tsconfig.json',
-// 'search-result/tsconfig.json',
-// 'simple-browser/tsconfig.json',
-// 'typescript-language-features/test-workspace/tsconfig.json',
-// 'typescript-language-features/tsconfig.json',
-// 'vscode-api-tests/tsconfig.json',
-// 'vscode-colorize-tests/tsconfig.json',
-// 'vscode-custom-editor-tests/tsconfig.json',
-// 'vscode-notebook-tests/tsconfig.json',
-// 'vscode-test-resolver/tsconfig.json'
-// ];

-const getBaseUrl = out => `https://sqlopsbuilds.blob.core.windows.net/sourcemaps/${commit}/${out}`;
+/**
+const compilations = [
+	'authentication-proxy/tsconfig.json',
+	'configuration-editing/build/tsconfig.json',
+	'configuration-editing/tsconfig.json',
+	'css-language-features/client/tsconfig.json',
+	'css-language-features/server/tsconfig.json',
+	'debug-auto-launch/tsconfig.json',
+	'debug-server-ready/tsconfig.json',
+	'emmet/tsconfig.json',
+	'extension-editing/tsconfig.json',
+	'git/tsconfig.json',
+	'git-base/tsconfig.json',
+	'github-authentication/tsconfig.json',
+	'github/tsconfig.json',
+	'grunt/tsconfig.json',
+	'gulp/tsconfig.json',
+	'html-language-features/client/tsconfig.json',
+	'html-language-features/server/tsconfig.json',
+	'image-preview/tsconfig.json',
+	'ipynb/tsconfig.json',
+	'jake/tsconfig.json',
+	'json-language-features/client/tsconfig.json',
+	'json-language-features/server/tsconfig.json',
+	'markdown-language-features/preview-src/tsconfig.json',
+	'markdown-language-features/tsconfig.json',
+	'markdown-math/tsconfig.json',
+	'merge-conflict/tsconfig.json',
+	'microsoft-authentication/tsconfig.json',
+	'npm/tsconfig.json',
+	'php-language-features/tsconfig.json',
+	'search-result/tsconfig.json',
+	'simple-browser/tsconfig.json',
+	'typescript-language-features/test-workspace/tsconfig.json',
+	'typescript-language-features/tsconfig.json',
+	'vscode-api-tests/tsconfig.json',
+	'vscode-colorize-tests/tsconfig.json',
+	'vscode-custom-editor-tests/tsconfig.json',
+	'vscode-notebook-tests/tsconfig.json',
+	'vscode-test-resolver/tsconfig.json'
+];
+*/
+
+const getBaseUrl = out => `https://sqlopsbuilds.blob.core.windows.net/sourcemaps/${commit}/${out}`; // {{SQL CARBON EDIT}} Use our own sourcemaps URL

 const tasks = compilations.map(function (tsconfigFile) {
 	const absolutePath = path.join(extensionsPath, tsconfigFile);
@@ -217,6 +221,7 @@ exports.watchExtensionMedia = watchExtensionMedia;

 const compileExtensionMediaBuildTask = task.define('compile-extension-media-build', () => ext.buildExtensionMedia(false, '.build/extensions'));
 gulp.task(compileExtensionMediaBuildTask);
+exports.compileExtensionMediaBuildTask = compileExtensionMediaBuildTask;

 //#endregion

@@ -226,7 +231,7 @@ const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimr
 const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(
 	cleanExtensionsBuildTask,
 	task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false).pipe(gulp.dest('.build'))),
-	task.define('bundle-marketplace-extensions-build', () => { ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build')) }), // {{SQL CARBON EDIT}}
+	task.define('bundle-marketplace-extensions-build', () => { ext.packageMarketplaceExtensionsStream(false, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build')) }), // {{SQL CARBON EDIT}}
 ));

 gulp.task(compileExtensionsBuildTask);
--- a/build/lib/builtInExtensions.js
+++ b/build/lib/builtInExtensions.js
@@ -45,15 +45,18 @@ function isUpToDate(extension) {
    }
 }
 function syncMarketplaceExtension(extension) {
+    var _a;
+    const galleryServiceUrl = (_a = productjson.extensionsGallery) === null || _a === void 0 ? void 0 : _a.serviceUrl;
+    const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
    if (isUpToDate(extension)) {
-        log(ansiColors.blue('[marketplace]'), `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
+        log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
        return es.readArray([]);
    }
    rimraf.sync(getExtensionPath(extension));
-    return ext.fromMarketplace(extension.name, extension.version, extension.metadata)
+    return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
        .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
        .pipe(vfs.dest('.build/builtInExtensions'))
-        .on('end', () => log(ansiColors.blue('[marketplace]'), extension.name, ansiColors.green('✔︎')));
+        .on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
 }
 function syncExtension(extension, controlState) {
    if (extension.platforms) {
--- a/build/lib/builtInExtensions.ts
+++ b/build/lib/builtInExtensions.ts
@@ -31,7 +31,7 @@ export interface IExtensionDefinition {
 			flags: string;
 		};
 		publisherDisplayName: string;
-	}
+	};
 }

 const root = path.dirname(path.dirname(__dirname));
@@ -69,17 +69,20 @@ function isUpToDate(extension: IExtensionDefinition): boolean {
 }

 function syncMarketplaceExtension(extension: IExtensionDefinition): Stream {
+	const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
+	const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
+
 	if (isUpToDate(extension)) {
-		log(ansiColors.blue('[marketplace]'), `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
+		log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
 		return es.readArray([]);
 	}

 	rimraf.sync(getExtensionPath(extension));

-	return ext.fromMarketplace(extension.name, extension.version, extension.metadata)
+	return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
 		.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
 		.pipe(vfs.dest('.build/builtInExtensions'))
-		.on('end', () => log(ansiColors.blue('[marketplace]'), extension.name, ansiColors.green('✔︎')));
+		.on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
 }

 function syncExtension(extension: IExtensionDefinition, controlState: 'disabled' | 'marketplace'): Stream {
--- a/build/lib/extensions.js
+++ b/build/lib/extensions.js
@@ -4,13 +4,15 @@
 *  Licensed under the Source EULA. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildExtensionMedia = exports.webpackExtensions = exports.translatePackageJSON = exports.packageRebuildExtensionsStream = exports.cleanRebuildExtensions = exports.packageExternalExtensionsStream = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.vscodeExternalExtensions = exports.fromMarketplace = exports.fromLocalNormal = exports.fromLocal = void 0;
+exports.buildExtensionMedia = exports.webpackExtensions = exports.translatePackageJSON = exports.packageRebuildExtensionsStream = exports.cleanRebuildExtensions = exports.packageExternalExtensionsStream = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.vscodeExternalExtensions = exports.fromGithub = exports.fromMarketplace = exports.fromLocalNormal = exports.fromLocal = void 0;
 const es = require("event-stream");
 const fs = require("fs");
-// import * as cp from 'child_process';  // {{SQL CARBON EDIT}} -- remove unused
+const cp = require("child_process");
 const glob = require("glob");
 const gulp = require("gulp");
 const path = require("path");
+const through2 = require("through2");
+const got_1 = require("got");
 const File = require("vinyl");
 const stats_1 = require("./stats");
 const util2 = require("./util");
@@ -168,16 +170,17 @@ function fromLocalNormal(extensionPath) {
    return result.pipe((0, stats_1.createStatsStream)(path.basename(extensionPath)));
 }
 exports.fromLocalNormal = fromLocalNormal;
+const userAgent = 'VSCode Build';
 const baseHeaders = {
    'X-Market-Client-Id': 'VSCode Build',
-    'User-Agent': 'VSCode Build',
+    'User-Agent': userAgent,
    'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2',
 };
-function fromMarketplace(extensionName, version, metadata) {
+function fromMarketplace(_serviceUrl, { name: extensionName, version, metadata }) {
    const remote = require('gulp-remote-retry-src');
    const json = require('gulp-json-editor');
-    const [, name] = extensionName.split('.');
-    const url = `https://sqlopsextensions.blob.core.windows.net/extensions/${name}/${name}-${version}.vsix`; // {{SQL CARBON EDIT}}
+    const [_publisher, name] = extensionName.split('.'); // {{SQL CARBON EDIT}} We don't have the publisher in our path
+    const url = `https://sqlopsextensions.blob.core.windows.net/extensions/${name}/${name}-${version}.vsix`; // {{SQL CARBON EDIT}} Use our own download URL
    fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...');
    const options = {
        base: url,
@@ -197,6 +200,41 @@ function fromMarketplace(extensionName, version, metadata) {
        .pipe(packageJsonFilter.restore);
 }
 exports.fromMarketplace = fromMarketplace;
+const ghApiHeaders = {
+    Accept: 'application/vnd.github.v3+json',
+    'User-Agent': userAgent,
+};
+if (process.env.GITHUB_TOKEN) {
+    ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
+}
+const ghDownloadHeaders = Object.assign(Object.assign({}, ghApiHeaders), { Accept: 'application/octet-stream' });
+function fromGithub({ name, version, repo, metadata }) {
+    const remote = require('gulp-remote-retry-src');
+    const json = require('gulp-json-editor');
+    fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...');
+    const packageJsonFilter = filter('package.json', { restore: true });
+    return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], {
+        base: 'https://api.github.com',
+        requestOptions: { headers: ghApiHeaders }
+    }).pipe(through2.obj(function (file, _enc, callback) {
+        const asset = JSON.parse(file.contents.toString()).assets.find((a) => a.name.endsWith('.vsix'));
+        if (!asset) {
+            return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`));
+        }
+        const res = got_1.default.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
+        file.contents = res.pipe(through2());
+        callback(null, file);
+    }))
+        .pipe(buffer())
+        .pipe(vzip.src())
+        .pipe(filter('extension/**'))
+        .pipe(rename(p => p.dirname = p.dirname.replace(/^extension\/?/, '')))
+        .pipe(packageJsonFilter)
+        .pipe(buffer())
+        .pipe(json({ __metadata: metadata }))
+        .pipe(packageJsonFilter.restore);
+}
+exports.fromGithub = fromGithub;
 const excludedExtensions = [
    'vscode-api-tests',
    'vscode-colorize-tests',
@@ -204,6 +242,7 @@ const excludedExtensions = [
    'ms-vscode.node-debug',
    'ms-vscode.node-debug2',
    'vscode-notebook-tests',
+    'vscode-custom-editor-tests',
    'integration-tests', // {{SQL CARBON EDIT}}
 ];
 // {{SQL CARBON EDIT}}
@@ -293,7 +332,7 @@ function packageLocalExtensionsStream(forWeb) {
        .filter(({ name }) => excludedExtensions.indexOf(name) === -1)
        .filter(({ name }) => builtInExtensions.every(b => b.name !== name))
        .filter(({ name }) => externalExtensions.indexOf(name) === -1) // {{SQL CARBON EDIT}} Remove external Extensions with separate package
-    );
+        .filter(({ manifestPath }) => (forWeb ? isWebExtension(require(manifestPath)) : true)));
    const localExtensionsStream = minifyExtensionResources(es.merge(...localExtensionsDescriptions.map(extension => {
        return fromLocal(extension.path, forWeb)
            .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
@@ -312,14 +351,14 @@ function packageLocalExtensionsStream(forWeb) {
        .pipe(util2.setExecutableBit(['**/*.sh'])));
 }
 exports.packageLocalExtensionsStream = packageLocalExtensionsStream;
-function packageMarketplaceExtensionsStream(forWeb) {
+function packageMarketplaceExtensionsStream(forWeb, galleryServiceUrl) {
    const marketplaceExtensionsDescriptions = [
        ...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
        ...(forWeb ? webBuiltInExtensions : [])
    ];
    const marketplaceExtensionsStream = minifyExtensionResources(es.merge(...marketplaceExtensionsDescriptions
        .map(extension => {
-        const input = fromMarketplace(extension.name, extension.version, extension.metadata)
+        const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
            .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
        return updateExtensionPackageJSON(input, (data) => {
            delete data.scripts;
@@ -431,16 +470,14 @@ function translatePackageJSON(packageJSON, packageNLSPath) {
 }
 exports.translatePackageJSON = translatePackageJSON;
 const extensionsPath = path.join(root, 'extensions');
-// Additional projects to webpack. These typically build code for webviews
-// const webpackMediaConfigFiles = [
-// 	//	'markdown-language-features/webpack.config.js',
-// 	'simple-browser/webpack.config.js',
-// ];
 // Additional projects to run esbuild on. These typically build code for webviews
-// const esbuildMediaScripts = [
-// 	'markdown-language-features/esbuild.js',
-// 	'markdown-math/esbuild.js',
-// ];
+const esbuildMediaScripts = [
+    'markdown-language-features/esbuild-notebook.js',
+    'markdown-language-features/esbuild-preview.js',
+    'markdown-math/esbuild.js',
+    // 'notebook-renderers/esbuild.js', {{SQL CARBON EDIT}} We don't have this extension
+    'simple-browser/esbuild-preview.js',
+];
 async function webpackExtensions(taskName, isWatch, webpackConfigLocations) {
    const webpack = require('webpack');
    const webpackConfigs = [];
@@ -510,54 +547,44 @@ async function webpackExtensions(taskName, isWatch, webpackConfigLocations) {
    });
 }
 exports.webpackExtensions = webpackExtensions;
-// {{SQL CARBON EDIT}} -- remove unused
-// async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: { script: string, outputRoot?: string }[]) {
-// 	function reporter(stdError: string, script: string) {
-// 		const matches = (stdError || '').match(/\> (.+): error: (.+)?/g);
-// 		fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`);
-// 		for (const match of matches || []) {
-// 			fancyLog.error(match);
-// 		}
-// 	}
-// 	const tasks = scripts.map(({ script, outputRoot }) => {
-// 		return new Promise<void>((resolve, reject) => {
-// 			const args = [script];
-// 			if (isWatch) {
-// 				args.push('--watch');
-// 			}
-// 			if (outputRoot) {
-// 				args.push('--outputRoot', outputRoot);
-// 			}
-// 			const proc = cp.execFile(process.argv[0], args, {}, (error, _stdout, stderr) => {
-// 				if (error) {
-// 					return reject(error);
-// 				}
-// 				reporter(stderr, script);
-// 				if (stderr) {
-// 					return reject();
-// 				}
-// 				return resolve();
-// 			});
-// 			proc.stdout!.on('data', (data) => {
-// 				fancyLog(`${ansiColors.green(taskName)}: ${data.toString('utf8')}`);
-// 			});
-// 		});
-// 	});
-// 	return Promise.all(tasks);
-// }
-async function buildExtensionMedia(_isWatch, _outputRoot) {
-    return undefined;
-    // 	return Promise.all([
-    // 		// webpackExtensions('webpacking extension media', isWatch, webpackMediaConfigFiles.map(p => {
-    // 		// 	return {
-    // 		// 		configPath: path.join(extensionsPath, p),
-    // 		// 		outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
-    // 		// 	};
-    // 		// })),
-    // 		esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
-    // 			script: path.join(extensionsPath, p),
-    // 			outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
-    // 		}))),
-    // 	]);
+async function esbuildExtensions(taskName, isWatch, scripts) {
+    function reporter(stdError, script) {
+        const matches = (stdError || '').match(/\> (.+): error: (.+)?/g);
+        fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`);
+        for (const match of matches || []) {
+            fancyLog.error(match);
+        }
+    }
+    const tasks = scripts.map(({ script, outputRoot }) => {
+        return new Promise((resolve, reject) => {
+            const args = [script];
+            if (isWatch) {
+                args.push('--watch');
+            }
+            if (outputRoot) {
+                args.push('--outputRoot', outputRoot);
+            }
+            const proc = cp.execFile(process.argv[0], args, {}, (error, _stdout, stderr) => {
+                if (error) {
+                    return reject(error);
+                }
+                reporter(stderr, script);
+                if (stderr) {
+                    return reject();
+                }
+                return resolve();
+            });
+            proc.stdout.on('data', (data) => {
+                fancyLog(`${ansiColors.green(taskName)}: ${data.toString('utf8')}`);
+            });
+        });
+    });
+    return Promise.all(tasks);
+}
+async function buildExtensionMedia(isWatch, outputRoot) {
+    return esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
+        script: path.join(extensionsPath, p),
+        outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
+    })));
 }
 exports.buildExtensionMedia = buildExtensionMedia;
--- a/build/lib/extensions.ts
+++ b/build/lib/extensions.ts
@@ -5,10 +5,12 @@

 import * as es from 'event-stream';
 import * as fs from 'fs';
-// import * as cp from 'child_process';  // {{SQL CARBON EDIT}} -- remove unused
+import * as cp from 'child_process';
 import * as glob from 'glob';
 import * as gulp from 'gulp';
 import * as path from 'path';
+import * as through2 from 'through2';
+import got from 'got';
 import { Stream } from 'stream';
 import * as File from 'vinyl';
 import { createStatsStream } from './stats';
@@ -198,18 +200,19 @@ export function fromLocalNormal(extensionPath: string): Stream { // {{SQL CARBON
 	return result.pipe(createStatsStream(path.basename(extensionPath)));
 }

+const userAgent = 'VSCode Build';
 const baseHeaders = {
 	'X-Market-Client-Id': 'VSCode Build',
-	'User-Agent': 'VSCode Build',
+	'User-Agent': userAgent,
 	'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2',
 };

-export function fromMarketplace(extensionName: string, version: string, metadata: any): Stream {
+export function fromMarketplace(_serviceUrl: string, { name: extensionName, version, metadata }: IBuiltInExtension): Stream { // {{SQL CARBON EDIT}} We don't use the passed in service URL
 	const remote = require('gulp-remote-retry-src');
 	const json = require('gulp-json-editor') as typeof import('gulp-json-editor');

-	const [, name] = extensionName.split('.');
-	const url = `https://sqlopsextensions.blob.core.windows.net/extensions/${name}/${name}-${version}.vsix`; // {{SQL CARBON EDIT}}
+	const [_publisher, name] = extensionName.split('.'); // {{SQL CARBON EDIT}} We don't have the publisher in our path
+	const url = `https://sqlopsextensions.blob.core.windows.net/extensions/${name}/${name}-${version}.vsix`; // {{SQL CARBON EDIT}} Use our own download URL

 	fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...');

@@ -232,6 +235,50 @@ export function fromMarketplace(extensionName: string, version: string, metadata
 		.pipe(json({ __metadata: metadata }))
 		.pipe(packageJsonFilter.restore);
 }
+
+const ghApiHeaders: Record<string, string> = {
+	Accept: 'application/vnd.github.v3+json',
+	'User-Agent': userAgent,
+};
+if (process.env.GITHUB_TOKEN) {
+	ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
+}
+const ghDownloadHeaders = {
+	...ghApiHeaders,
+	Accept: 'application/octet-stream',
+};
+
+export function fromGithub({ name, version, repo, metadata }: IBuiltInExtension): Stream {
+	const remote = require('gulp-remote-retry-src');
+	const json = require('gulp-json-editor') as typeof import('gulp-json-editor');
+
+	fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...');
+
+	const packageJsonFilter = filter('package.json', { restore: true });
+
+	return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], {
+		base: 'https://api.github.com',
+		requestOptions: { headers: ghApiHeaders }
+	}).pipe(through2.obj(function (file, _enc, callback) {
+		const asset = JSON.parse(file.contents.toString()).assets.find((a: any) => a.name.endsWith('.vsix'));
+		if (!asset) {
+			return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`));
+		}
+
+		const res = got.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
+		file.contents = res.pipe(through2());
+		callback(null, file);
+	}))
+		.pipe(buffer())
+		.pipe(vzip.src())
+		.pipe(filter('extension/**'))
+		.pipe(rename(p => p.dirname = p.dirname!.replace(/^extension\/?/, '')))
+		.pipe(packageJsonFilter)
+		.pipe(buffer())
+		.pipe(json({ __metadata: metadata }))
+		.pipe(packageJsonFilter.restore);
+}
+
 const excludedExtensions = [
 	'vscode-api-tests',
 	'vscode-colorize-tests',
@@ -239,6 +286,7 @@ const excludedExtensions = [
 	'ms-vscode.node-debug',
 	'ms-vscode.node-debug2',
 	'vscode-notebook-tests',
+	'vscode-custom-editor-tests',
 	'integration-tests', // {{SQL CARBON EDIT}}
 ];

@@ -352,6 +400,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream {
 			.filter(({ name }) => excludedExtensions.indexOf(name) === -1)
 			.filter(({ name }) => builtInExtensions.every(b => b.name !== name))
 			.filter(({ name }) => externalExtensions.indexOf(name) === -1) // {{SQL CARBON EDIT}} Remove external Extensions with separate package
+			.filter(({ manifestPath }) => (forWeb ? isWebExtension(require(manifestPath)) : true))
 	);
 	const localExtensionsStream = minifyExtensionResources(
 		es.merge(
@@ -378,7 +427,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream {
 	);
 }

-export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream {
+export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServiceUrl?: string): Stream {
 	const marketplaceExtensionsDescriptions = [
 		...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
 		...(forWeb ? webBuiltInExtensions : [])
@@ -387,7 +436,7 @@ export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream {
 		es.merge(
 			...marketplaceExtensionsDescriptions
 				.map(extension => {
-					const input = fromMarketplace(extension.name, extension.version, extension.metadata)
+					const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
 						.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
 					return updateExtensionPackageJSON(input, (data: any) => {
 						delete data.scripts;
@@ -494,7 +543,7 @@ export function packageRebuildExtensionsStream(): NodeJS.ReadWriteStream {

 export function translatePackageJSON(packageJSON: string, packageNLSPath: string) {
 	interface NLSFormat {
-		[key: string]: string | { message: string, comment: string[] };
+		[key: string]: string | { message: string; comment: string[] };
 	}
 	const CharCode_PC = '%'.charCodeAt(0);
 	const packageNls: NLSFormat = JSON.parse(fs.readFileSync(packageNLSPath).toString());
@@ -519,19 +568,16 @@ export function translatePackageJSON(packageJSON: string, packageNLSPath: string

 const extensionsPath = path.join(root, 'extensions');

-// Additional projects to webpack. These typically build code for webviews
-// const webpackMediaConfigFiles = [
-// 	//	'markdown-language-features/webpack.config.js',
-// 	'simple-browser/webpack.config.js',
-// ];
-
 // Additional projects to run esbuild on. These typically build code for webviews
-// const esbuildMediaScripts = [
-// 	'markdown-language-features/esbuild.js',
-// 	'markdown-math/esbuild.js',
-// ];
+const esbuildMediaScripts = [
+	'markdown-language-features/esbuild-notebook.js',
+	'markdown-language-features/esbuild-preview.js',
+	'markdown-math/esbuild.js',
+	// 'notebook-renderers/esbuild.js', {{SQL CARBON EDIT}} We don't have this extension
+	'simple-browser/esbuild-preview.js',
+];

-export async function webpackExtensions(taskName: string, isWatch: boolean, webpackConfigLocations: { configPath: string, outputRoot?: string }[]) {
+export async function webpackExtensions(taskName: string, isWatch: boolean, webpackConfigLocations: { configPath: string; outputRoot?: string }[]) {
 	const webpack = require('webpack') as typeof import('webpack');

 	const webpackConfigs: webpack.Configuration[] = [];
@@ -600,56 +646,46 @@ export async function webpackExtensions(taskName: string, isWatch: boolean, webp
 	});
 }

-// {{SQL CARBON EDIT}} -- remove unused
-// async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: { script: string, outputRoot?: string }[]) {
-// 	function reporter(stdError: string, script: string) {
-// 		const matches = (stdError || '').match(/\> (.+): error: (.+)?/g);
-// 		fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`);
-// 		for (const match of matches || []) {
-// 			fancyLog.error(match);
-// 		}
-// 	}
+async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: { script: string; outputRoot?: string }[]) {
+	function reporter(stdError: string, script: string) {
+		const matches = (stdError || '').match(/\> (.+): error: (.+)?/g);
+		fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`);
+		for (const match of matches || []) {
+			fancyLog.error(match);
+		}
+	}

-// 	const tasks = scripts.map(({ script, outputRoot }) => {
-// 		return new Promise<void>((resolve, reject) => {
-// 			const args = [script];
-// 			if (isWatch) {
-// 				args.push('--watch');
-// 			}
-// 			if (outputRoot) {
-// 				args.push('--outputRoot', outputRoot);
-// 			}
-// 			const proc = cp.execFile(process.argv[0], args, {}, (error, _stdout, stderr) => {
-// 				if (error) {
-// 					return reject(error);
-// 				}
-// 				reporter(stderr, script);
-// 				if (stderr) {
-// 					return reject();
-// 				}
-// 				return resolve();
-// 			});
+	const tasks = scripts.map(({ script, outputRoot }) => {
+		return new Promise<void>((resolve, reject) => {
+			const args = [script];
+			if (isWatch) {
+				args.push('--watch');
+			}
+			if (outputRoot) {
+				args.push('--outputRoot', outputRoot);
+			}
+			const proc = cp.execFile(process.argv[0], args, {}, (error, _stdout, stderr) => {
+				if (error) {
+					return reject(error);
+				}
+				reporter(stderr, script);
+				if (stderr) {
+					return reject();
+				}
+				return resolve();
+			});

-// 			proc.stdout!.on('data', (data) => {
-// 				fancyLog(`${ansiColors.green(taskName)}: ${data.toString('utf8')}`);
-// 			});
-// 		});
-// 	});
-// 	return Promise.all(tasks);
-// }
-
-export async function buildExtensionMedia(_isWatch: boolean, _outputRoot?: string) {
-	return undefined;
-	// 	return Promise.all([
-	// 		// webpackExtensions('webpacking extension media', isWatch, webpackMediaConfigFiles.map(p => {
-	// 		// 	return {
-	// 		// 		configPath: path.join(extensionsPath, p),
-	// 		// 		outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
-	// 		// 	};
-	// 		// })),
-	// 		esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
-	// 			script: path.join(extensionsPath, p),
-	// 			outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
-	// 		}))),
-	// 	]);
+			proc.stdout!.on('data', (data) => {
+				fancyLog(`${ansiColors.green(taskName)}: ${data.toString('utf8')}`);
+			});
+		});
+	});
+	return Promise.all(tasks);
+}
+
+export async function buildExtensionMedia(isWatch: boolean, outputRoot?: string) {
+	return esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
+		script: path.join(extensionsPath, p),
+		outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
+	})));
 }
--- a/product.json
+++ b/product.json
@@ -93,7 +93,6 @@
 	},
 	"builtInExtensions": [
 		{
-			"version": "1.61.0",
 			"name": "Microsoft.sqlservernotebook",
 			"version": "0.5.0",
 			"repo": "https://github.com/microsoft/azuredatastudio"