ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 18:46:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Re-enable extension media building (#21041)

Browse Source 
* Re-enable extension media building

* Fix build
This commit is contained in:
Charles Gagnon
2022-10-31 10:25:56 -07:00
committed by GitHub
parent 275a772496
commit 4ade808695
6 changed files with 259 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
build/gulpfile.extensions.js
View File

@@ -20,6 +20,7 @@ const root = path.dirname(__dirname);
const commit = util.getVersion(root); const commit = util.getVersion(root);
const plumber = require('gulp-plumber'); const plumber = require('gulp-plumber');
const ext = require('./lib/extensions'); const ext = require('./lib/extensions');
const product = require('../product.json');
const extensionsPath = path.join(path.dirname(__dirname), 'extensions'); const extensionsPath = path.join(path.dirname(__dirname), 'extensions');
@@ -34,48 +35,51 @@ const compilations = glob.sync('**/tsconfig.json', {
cwd: extensionsPath, cwd: extensionsPath,
ignore: ['**/out/**', '**/node_modules/**'] ignore: ['**/out/**', '**/node_modules/**']
}); });
// const compilations = [
// 'authentication-proxy/tsconfig.json',
// 'configuration-editing/build/tsconfig.json',
// 'configuration-editing/tsconfig.json',
// 'css-language-features/client/tsconfig.json',
// 'css-language-features/server/tsconfig.json',
// 'debug-auto-launch/tsconfig.json',
// 'debug-server-ready/tsconfig.json',
// 'emmet/tsconfig.json',
// 'extension-editing/tsconfig.json',
// 'git/tsconfig.json',
// 'git-base/tsconfig.json',
// 'github-authentication/tsconfig.json',
// 'github/tsconfig.json',
// 'grunt/tsconfig.json',
// 'gulp/tsconfig.json',
// 'html-language-features/client/tsconfig.json',
// 'html-language-features/server/tsconfig.json',
// 'image-preview/tsconfig.json',
// 'ipynb/tsconfig.json',
// 'jake/tsconfig.json',
// 'json-language-features/client/tsconfig.json',
// 'json-language-features/server/tsconfig.json',
// 'markdown-language-features/preview-src/tsconfig.json',
// 'markdown-language-features/tsconfig.json',
// 'markdown-math/tsconfig.json',
// 'merge-conflict/tsconfig.json',
// 'microsoft-authentication/tsconfig.json',
// 'npm/tsconfig.json',
// 'php-language-features/tsconfig.json',
// 'search-result/tsconfig.json',
// 'simple-browser/tsconfig.json',
// 'typescript-language-features/test-workspace/tsconfig.json',
// 'typescript-language-features/tsconfig.json',
// 'vscode-api-tests/tsconfig.json',
// 'vscode-colorize-tests/tsconfig.json',
// 'vscode-custom-editor-tests/tsconfig.json',
// 'vscode-notebook-tests/tsconfig.json',
// 'vscode-test-resolver/tsconfig.json'
// ];
const getBaseUrl = out => `https://sqlopsbuilds.blob.core.windows.net/sourcemaps/${commit}/${out}`; /**
const compilations = [
'authentication-proxy/tsconfig.json',
'configuration-editing/build/tsconfig.json',
'configuration-editing/tsconfig.json',
'css-language-features/client/tsconfig.json',
'css-language-features/server/tsconfig.json',
'debug-auto-launch/tsconfig.json',
'debug-server-ready/tsconfig.json',
'emmet/tsconfig.json',
'extension-editing/tsconfig.json',
'git/tsconfig.json',
'git-base/tsconfig.json',
'github-authentication/tsconfig.json',
'github/tsconfig.json',
'grunt/tsconfig.json',
'gulp/tsconfig.json',
'html-language-features/client/tsconfig.json',
'html-language-features/server/tsconfig.json',
'image-preview/tsconfig.json',
'ipynb/tsconfig.json',
'jake/tsconfig.json',
'json-language-features/client/tsconfig.json',
'json-language-features/server/tsconfig.json',
'markdown-language-features/preview-src/tsconfig.json',
'markdown-language-features/tsconfig.json',
'markdown-math/tsconfig.json',
'merge-conflict/tsconfig.json',
'microsoft-authentication/tsconfig.json',
'npm/tsconfig.json',
'php-language-features/tsconfig.json',
'search-result/tsconfig.json',
'simple-browser/tsconfig.json',
'typescript-language-features/test-workspace/tsconfig.json',
'typescript-language-features/tsconfig.json',
'vscode-api-tests/tsconfig.json',
'vscode-colorize-tests/tsconfig.json',
'vscode-custom-editor-tests/tsconfig.json',
'vscode-notebook-tests/tsconfig.json',
'vscode-test-resolver/tsconfig.json'
];
*/
const getBaseUrl = out => `https://sqlopsbuilds.blob.core.windows.net/sourcemaps/${commit}/${out}`; // {{SQL CARBON EDIT}} Use our own sourcemaps URL
const tasks = compilations.map(function (tsconfigFile) { const tasks = compilations.map(function (tsconfigFile) {
const absolutePath = path.join(extensionsPath, tsconfigFile); const absolutePath = path.join(extensionsPath, tsconfigFile);
@@ -217,6 +221,7 @@ exports.watchExtensionMedia = watchExtensionMedia;
const compileExtensionMediaBuildTask = task.define('compile-extension-media-build', () => ext.buildExtensionMedia(false, '.build/extensions')); const compileExtensionMediaBuildTask = task.define('compile-extension-media-build', () => ext.buildExtensionMedia(false, '.build/extensions'));
gulp.task(compileExtensionMediaBuildTask); gulp.task(compileExtensionMediaBuildTask);
exports.compileExtensionMediaBuildTask = compileExtensionMediaBuildTask;
//#endregion //#endregion
@@ -226,7 +231,7 @@ const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimr
const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series( const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(
cleanExtensionsBuildTask, cleanExtensionsBuildTask,
task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false).pipe(gulp.dest('.build'))), task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false).pipe(gulp.dest('.build'))),
task.define('bundle-marketplace-extensions-build', () => { ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build')) }), // {{SQL CARBON EDIT}} task.define('bundle-marketplace-extensions-build', () => { ext.packageMarketplaceExtensionsStream(false, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build')) }), // {{SQL CARBON EDIT}}
)); ));
gulp.task(compileExtensionsBuildTask); gulp.task(compileExtensionsBuildTask);

9
build/lib/builtInExtensions.js
View File

@@ -45,15 +45,18 @@ function isUpToDate(extension) {
} }
} }
function syncMarketplaceExtension(extension) { function syncMarketplaceExtension(extension) {
var _a;
const galleryServiceUrl = (_a = productjson.extensionsGallery) === null || _a === void 0 ? void 0 : _a.serviceUrl;
const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
if (isUpToDate(extension)) { if (isUpToDate(extension)) {
log(ansiColors.blue('[marketplace]'), `${extension.name}@${extension.version}`, ansiColors.green('✔︎')); log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
return es.readArray([]); return es.readArray([]);
} }
rimraf.sync(getExtensionPath(extension)); rimraf.sync(getExtensionPath(extension));
return ext.fromMarketplace(extension.name, extension.version, extension.metadata) return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)) .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
.pipe(vfs.dest('.build/builtInExtensions')) .pipe(vfs.dest('.build/builtInExtensions'))
.on('end', () => log(ansiColors.blue('[marketplace]'), extension.name, ansiColors.green('✔︎'))); .on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
} }
function syncExtension(extension, controlState) { function syncExtension(extension, controlState) {
if (extension.platforms) { if (extension.platforms) {

11
build/lib/builtInExtensions.ts
View File

@@ -31,7 +31,7 @@ export interface IExtensionDefinition {
flags: string; flags: string;
}; };
publisherDisplayName: string; publisherDisplayName: string;
} };
} }
const root = path.dirname(path.dirname(__dirname)); const root = path.dirname(path.dirname(__dirname));
@@ -69,17 +69,20 @@ function isUpToDate(extension: IExtensionDefinition): boolean {
} }
function syncMarketplaceExtension(extension: IExtensionDefinition): Stream { function syncMarketplaceExtension(extension: IExtensionDefinition): Stream {
const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
if (isUpToDate(extension)) { if (isUpToDate(extension)) {
log(ansiColors.blue('[marketplace]'), `${extension.name}@${extension.version}`, ansiColors.green('✔︎')); log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
return es.readArray([]); return es.readArray([]);
} }
rimraf.sync(getExtensionPath(extension)); rimraf.sync(getExtensionPath(extension));
return ext.fromMarketplace(extension.name, extension.version, extension.metadata) return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)) .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
.pipe(vfs.dest('.build/builtInExtensions')) .pipe(vfs.dest('.build/builtInExtensions'))
.on('end', () => log(ansiColors.blue('[marketplace]'), extension.name, ansiColors.green('✔︎'))); .on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
} }
function syncExtension(extension: IExtensionDefinition, controlState: 'disabled' | 'marketplace'): Stream { function syncExtension(extension: IExtensionDefinition, controlState: 'disabled' | 'marketplace'): Stream {

161
build/lib/extensions.js
View File

@@ -4,13 +4,15 @@
* Licensed under the Source EULA. See License.txt in the project root for license information. * Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/ *--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
exports.buildExtensionMedia = exports.webpackExtensions = exports.translatePackageJSON = exports.packageRebuildExtensionsStream = exports.cleanRebuildExtensions = exports.packageExternalExtensionsStream = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.vscodeExternalExtensions = exports.fromMarketplace = exports.fromLocalNormal = exports.fromLocal = void 0; exports.buildExtensionMedia = exports.webpackExtensions = exports.translatePackageJSON = exports.packageRebuildExtensionsStream = exports.cleanRebuildExtensions = exports.packageExternalExtensionsStream = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.vscodeExternalExtensions = exports.fromGithub = exports.fromMarketplace = exports.fromLocalNormal = exports.fromLocal = void 0;
const es = require("event-stream"); const es = require("event-stream");
const fs = require("fs"); const fs = require("fs");
// import * as cp from 'child_process'; // {{SQL CARBON EDIT}} -- remove unused const cp = require("child_process");
const glob = require("glob"); const glob = require("glob");
const gulp = require("gulp"); const gulp = require("gulp");
const path = require("path"); const path = require("path");
const through2 = require("through2");
const got_1 = require("got");
const File = require("vinyl"); const File = require("vinyl");
const stats_1 = require("./stats"); const stats_1 = require("./stats");
const util2 = require("./util"); const util2 = require("./util");
@@ -168,16 +170,17 @@ function fromLocalNormal(extensionPath) {
return result.pipe((0, stats_1.createStatsStream)(path.basename(extensionPath))); return result.pipe((0, stats_1.createStatsStream)(path.basename(extensionPath)));
} }
exports.fromLocalNormal = fromLocalNormal; exports.fromLocalNormal = fromLocalNormal;
const userAgent = 'VSCode Build';
const baseHeaders = { const baseHeaders = {
'X-Market-Client-Id': 'VSCode Build', 'X-Market-Client-Id': 'VSCode Build',
'User-Agent': 'VSCode Build', 'User-Agent': userAgent,
'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2', 'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2',
}; };
function fromMarketplace(extensionName, version, metadata) { function fromMarketplace(_serviceUrl, { name: extensionName, version, metadata }) {
const remote = require('gulp-remote-retry-src'); const remote = require('gulp-remote-retry-src');
const json = require('gulp-json-editor'); const json = require('gulp-json-editor');
const [, name] = extensionName.split('.'); const [_publisher, name] = extensionName.split('.'); // {{SQL CARBON EDIT}} We don't have the publisher in our path
const url = `https://sqlopsextensions.blob.core.windows.net/extensions/${name}/${name}-${version}.vsix`; // {{SQL CARBON EDIT}} const url = `https://sqlopsextensions.blob.core.windows.net/extensions/${name}/${name}-${version}.vsix`; // {{SQL CARBON EDIT}} Use our own download URL
fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...'); fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...');
const options = { const options = {
base: url, base: url,
@@ -197,6 +200,41 @@ function fromMarketplace(extensionName, version, metadata) {
.pipe(packageJsonFilter.restore); .pipe(packageJsonFilter.restore);
} }
exports.fromMarketplace = fromMarketplace; exports.fromMarketplace = fromMarketplace;
const ghApiHeaders = {
Accept: 'application/vnd.github.v3+json',
'User-Agent': userAgent,
};
if (process.env.GITHUB_TOKEN) {
ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
}
const ghDownloadHeaders = Object.assign(Object.assign({}, ghApiHeaders), { Accept: 'application/octet-stream' });
function fromGithub({ name, version, repo, metadata }) {
const remote = require('gulp-remote-retry-src');
const json = require('gulp-json-editor');
fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...');
const packageJsonFilter = filter('package.json', { restore: true });
return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], {
base: 'https://api.github.com',
requestOptions: { headers: ghApiHeaders }
}).pipe(through2.obj(function (file, _enc, callback) {
const asset = JSON.parse(file.contents.toString()).assets.find((a) => a.name.endsWith('.vsix'));
if (!asset) {
return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`));
}
const res = got_1.default.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
file.contents = res.pipe(through2());
callback(null, file);
}))
.pipe(buffer())
.pipe(vzip.src())
.pipe(filter('extension/**'))
.pipe(rename(p => p.dirname = p.dirname.replace(/^extension\/?/, '')))
.pipe(packageJsonFilter)
.pipe(buffer())
.pipe(json({ __metadata: metadata }))
.pipe(packageJsonFilter.restore);
}
exports.fromGithub = fromGithub;
const excludedExtensions = [ const excludedExtensions = [
'vscode-api-tests', 'vscode-api-tests',
'vscode-colorize-tests', 'vscode-colorize-tests',
@@ -204,6 +242,7 @@ const excludedExtensions = [
'ms-vscode.node-debug', 'ms-vscode.node-debug',
'ms-vscode.node-debug2', 'ms-vscode.node-debug2',
'vscode-notebook-tests', 'vscode-notebook-tests',
'vscode-custom-editor-tests',
'integration-tests', // {{SQL CARBON EDIT}} 'integration-tests', // {{SQL CARBON EDIT}}
]; ];
// {{SQL CARBON EDIT}} // {{SQL CARBON EDIT}}
@@ -293,7 +332,7 @@ function packageLocalExtensionsStream(forWeb) {
.filter(({ name }) => excludedExtensions.indexOf(name) === -1) .filter(({ name }) => excludedExtensions.indexOf(name) === -1)
.filter(({ name }) => builtInExtensions.every(b => b.name !== name)) .filter(({ name }) => builtInExtensions.every(b => b.name !== name))
.filter(({ name }) => externalExtensions.indexOf(name) === -1) // {{SQL CARBON EDIT}} Remove external Extensions with separate package .filter(({ name }) => externalExtensions.indexOf(name) === -1) // {{SQL CARBON EDIT}} Remove external Extensions with separate package
); .filter(({ manifestPath }) => (forWeb ? isWebExtension(require(manifestPath)) : true)));
const localExtensionsStream = minifyExtensionResources(es.merge(...localExtensionsDescriptions.map(extension => { const localExtensionsStream = minifyExtensionResources(es.merge(...localExtensionsDescriptions.map(extension => {
return fromLocal(extension.path, forWeb) return fromLocal(extension.path, forWeb)
.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
@@ -312,14 +351,14 @@ function packageLocalExtensionsStream(forWeb) {
.pipe(util2.setExecutableBit(['**/*.sh']))); .pipe(util2.setExecutableBit(['**/*.sh'])));
} }
exports.packageLocalExtensionsStream = packageLocalExtensionsStream; exports.packageLocalExtensionsStream = packageLocalExtensionsStream;
function packageMarketplaceExtensionsStream(forWeb) { function packageMarketplaceExtensionsStream(forWeb, galleryServiceUrl) {
const marketplaceExtensionsDescriptions = [ const marketplaceExtensionsDescriptions = [
...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)), ...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
...(forWeb ? webBuiltInExtensions : []) ...(forWeb ? webBuiltInExtensions : [])
]; ];
const marketplaceExtensionsStream = minifyExtensionResources(es.merge(...marketplaceExtensionsDescriptions const marketplaceExtensionsStream = minifyExtensionResources(es.merge(...marketplaceExtensionsDescriptions
.map(extension => { .map(extension => {
const input = fromMarketplace(extension.name, extension.version, extension.metadata) const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
return updateExtensionPackageJSON(input, (data) => { return updateExtensionPackageJSON(input, (data) => {
delete data.scripts; delete data.scripts;
@@ -431,16 +470,14 @@ function translatePackageJSON(packageJSON, packageNLSPath) {
} }
exports.translatePackageJSON = translatePackageJSON; exports.translatePackageJSON = translatePackageJSON;
const extensionsPath = path.join(root, 'extensions'); const extensionsPath = path.join(root, 'extensions');
// Additional projects to webpack. These typically build code for webviews
// const webpackMediaConfigFiles = [
// // 'markdown-language-features/webpack.config.js',
// 'simple-browser/webpack.config.js',
// ];
// Additional projects to run esbuild on. These typically build code for webviews // Additional projects to run esbuild on. These typically build code for webviews
// const esbuildMediaScripts = [ const esbuildMediaScripts = [
// 'markdown-language-features/esbuild.js', 'markdown-language-features/esbuild-notebook.js',
// 'markdown-math/esbuild.js', 'markdown-language-features/esbuild-preview.js',
// ]; 'markdown-math/esbuild.js',
// 'notebook-renderers/esbuild.js', {{SQL CARBON EDIT}} We don't have this extension
'simple-browser/esbuild-preview.js',
];
async function webpackExtensions(taskName, isWatch, webpackConfigLocations) { async function webpackExtensions(taskName, isWatch, webpackConfigLocations) {
const webpack = require('webpack'); const webpack = require('webpack');
const webpackConfigs = []; const webpackConfigs = [];
@@ -510,54 +547,44 @@ async function webpackExtensions(taskName, isWatch, webpackConfigLocations) {
}); });
} }
exports.webpackExtensions = webpackExtensions; exports.webpackExtensions = webpackExtensions;
// {{SQL CARBON EDIT}} -- remove unused async function esbuildExtensions(taskName, isWatch, scripts) {
// async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: { script: string, outputRoot?: string }[]) { function reporter(stdError, script) {
// function reporter(stdError: string, script: string) { const matches = (stdError || '').match(/\> (.+): error: (.+)?/g);
// const matches = (stdError || '').match(/\> (.+): error: (.+)?/g); fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`);
// fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`); for (const match of matches || []) {
// for (const match of matches || []) { fancyLog.error(match);
// fancyLog.error(match); }
// } }
// } const tasks = scripts.map(({ script, outputRoot }) => {
// const tasks = scripts.map(({ script, outputRoot }) => { return new Promise((resolve, reject) => {
// return new Promise<void>((resolve, reject) => { const args = [script];
// const args = [script]; if (isWatch) {
// if (isWatch) { args.push('--watch');
// args.push('--watch'); }
// } if (outputRoot) {
// if (outputRoot) { args.push('--outputRoot', outputRoot);
// args.push('--outputRoot', outputRoot); }
// } const proc = cp.execFile(process.argv[0], args, {}, (error, _stdout, stderr) => {
// const proc = cp.execFile(process.argv[0], args, {}, (error, _stdout, stderr) => { if (error) {
// if (error) { return reject(error);
// return reject(error); }
// } reporter(stderr, script);
// reporter(stderr, script); if (stderr) {
// if (stderr) { return reject();
// return reject(); }
// } return resolve();
// return resolve(); });
// }); proc.stdout.on('data', (data) => {
// proc.stdout!.on('data', (data) => { fancyLog(`${ansiColors.green(taskName)}: ${data.toString('utf8')}`);
// fancyLog(`${ansiColors.green(taskName)}: ${data.toString('utf8')}`); });
// }); });
// }); });
// }); return Promise.all(tasks);
// return Promise.all(tasks); }
// } async function buildExtensionMedia(isWatch, outputRoot) {
async function buildExtensionMedia(_isWatch, _outputRoot) { return esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
return undefined; script: path.join(extensionsPath, p),
// return Promise.all([ outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
// // webpackExtensions('webpacking extension media', isWatch, webpackMediaConfigFiles.map(p => { })));
// // return {
// // configPath: path.join(extensionsPath, p),
// // outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
// // };
// // })),
// esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
// script: path.join(extensionsPath, p),
// outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
// }))),
// ]);
} }
exports.buildExtensionMedia = buildExtensionMedia; exports.buildExtensionMedia = buildExtensionMedia;

178
build/lib/extensions.ts
View File

@@ -5,10 +5,12 @@
import * as es from 'event-stream'; import * as es from 'event-stream';
import * as fs from 'fs'; import * as fs from 'fs';
// import * as cp from 'child_process'; // {{SQL CARBON EDIT}} -- remove unused import * as cp from 'child_process';
import * as glob from 'glob'; import * as glob from 'glob';
import * as gulp from 'gulp'; import * as gulp from 'gulp';
import * as path from 'path'; import * as path from 'path';
import * as through2 from 'through2';
import got from 'got';
import { Stream } from 'stream'; import { Stream } from 'stream';
import * as File from 'vinyl'; import * as File from 'vinyl';
import { createStatsStream } from './stats'; import { createStatsStream } from './stats';
@@ -198,18 +200,19 @@ export function fromLocalNormal(extensionPath: string): Stream { // {{SQL CARBON
return result.pipe(createStatsStream(path.basename(extensionPath))); return result.pipe(createStatsStream(path.basename(extensionPath)));
} }
const userAgent = 'VSCode Build';
const baseHeaders = { const baseHeaders = {
'X-Market-Client-Id': 'VSCode Build', 'X-Market-Client-Id': 'VSCode Build',
'User-Agent': 'VSCode Build', 'User-Agent': userAgent,
'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2', 'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2',
}; };
export function fromMarketplace(extensionName: string, version: string, metadata: any): Stream { export function fromMarketplace(_serviceUrl: string, { name: extensionName, version, metadata }: IBuiltInExtension): Stream { // {{SQL CARBON EDIT}} We don't use the passed in service URL
const remote = require('gulp-remote-retry-src'); const remote = require('gulp-remote-retry-src');
const json = require('gulp-json-editor') as typeof import('gulp-json-editor'); const json = require('gulp-json-editor') as typeof import('gulp-json-editor');
const [, name] = extensionName.split('.'); const [_publisher, name] = extensionName.split('.'); // {{SQL CARBON EDIT}} We don't have the publisher in our path
const url = `https://sqlopsextensions.blob.core.windows.net/extensions/${name}/${name}-${version}.vsix`; // {{SQL CARBON EDIT}} const url = `https://sqlopsextensions.blob.core.windows.net/extensions/${name}/${name}-${version}.vsix`; // {{SQL CARBON EDIT}} Use our own download URL
fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...'); fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...');
@@ -232,6 +235,50 @@ export function fromMarketplace(extensionName: string, version: string, metadata
.pipe(json({ __metadata: metadata })) .pipe(json({ __metadata: metadata }))
.pipe(packageJsonFilter.restore); .pipe(packageJsonFilter.restore);
} }
const ghApiHeaders: Record<string, string> = {
Accept: 'application/vnd.github.v3+json',
'User-Agent': userAgent,
};
if (process.env.GITHUB_TOKEN) {
ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
}
const ghDownloadHeaders = {
...ghApiHeaders,
Accept: 'application/octet-stream',
};
export function fromGithub({ name, version, repo, metadata }: IBuiltInExtension): Stream {
const remote = require('gulp-remote-retry-src');
const json = require('gulp-json-editor') as typeof import('gulp-json-editor');
fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...');
const packageJsonFilter = filter('package.json', { restore: true });
return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], {
base: 'https://api.github.com',
requestOptions: { headers: ghApiHeaders }
}).pipe(through2.obj(function (file, _enc, callback) {
const asset = JSON.parse(file.contents.toString()).assets.find((a: any) => a.name.endsWith('.vsix'));
if (!asset) {
return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`));
}
const res = got.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
file.contents = res.pipe(through2());
callback(null, file);
}))
.pipe(buffer())
.pipe(vzip.src())
.pipe(filter('extension/**'))
.pipe(rename(p => p.dirname = p.dirname!.replace(/^extension\/?/, '')))
.pipe(packageJsonFilter)
.pipe(buffer())
.pipe(json({ __metadata: metadata }))
.pipe(packageJsonFilter.restore);
}
const excludedExtensions = [ const excludedExtensions = [
'vscode-api-tests', 'vscode-api-tests',
'vscode-colorize-tests', 'vscode-colorize-tests',
@@ -239,6 +286,7 @@ const excludedExtensions = [
'ms-vscode.node-debug', 'ms-vscode.node-debug',
'ms-vscode.node-debug2', 'ms-vscode.node-debug2',
'vscode-notebook-tests', 'vscode-notebook-tests',
'vscode-custom-editor-tests',
'integration-tests', // {{SQL CARBON EDIT}} 'integration-tests', // {{SQL CARBON EDIT}}
]; ];
@@ -352,6 +400,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream {
.filter(({ name }) => excludedExtensions.indexOf(name) === -1) .filter(({ name }) => excludedExtensions.indexOf(name) === -1)
.filter(({ name }) => builtInExtensions.every(b => b.name !== name)) .filter(({ name }) => builtInExtensions.every(b => b.name !== name))
.filter(({ name }) => externalExtensions.indexOf(name) === -1) // {{SQL CARBON EDIT}} Remove external Extensions with separate package .filter(({ name }) => externalExtensions.indexOf(name) === -1) // {{SQL CARBON EDIT}} Remove external Extensions with separate package
.filter(({ manifestPath }) => (forWeb ? isWebExtension(require(manifestPath)) : true))
); );
const localExtensionsStream = minifyExtensionResources( const localExtensionsStream = minifyExtensionResources(
es.merge( es.merge(
@@ -378,7 +427,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream {
); );
} }
export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream { export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServiceUrl?: string): Stream {
const marketplaceExtensionsDescriptions = [ const marketplaceExtensionsDescriptions = [
...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)), ...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
...(forWeb ? webBuiltInExtensions : []) ...(forWeb ? webBuiltInExtensions : [])
@@ -387,7 +436,7 @@ export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream {
es.merge( es.merge(
...marketplaceExtensionsDescriptions ...marketplaceExtensionsDescriptions
.map(extension => { .map(extension => {
const input = fromMarketplace(extension.name, extension.version, extension.metadata) const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
return updateExtensionPackageJSON(input, (data: any) => { return updateExtensionPackageJSON(input, (data: any) => {
delete data.scripts; delete data.scripts;
@@ -494,7 +543,7 @@ export function packageRebuildExtensionsStream(): NodeJS.ReadWriteStream {
export function translatePackageJSON(packageJSON: string, packageNLSPath: string) { export function translatePackageJSON(packageJSON: string, packageNLSPath: string) {
interface NLSFormat { interface NLSFormat {
[key: string]: string | { message: string, comment: string[] }; [key: string]: string | { message: string; comment: string[] };
} }
const CharCode_PC = '%'.charCodeAt(0); const CharCode_PC = '%'.charCodeAt(0);
const packageNls: NLSFormat = JSON.parse(fs.readFileSync(packageNLSPath).toString()); const packageNls: NLSFormat = JSON.parse(fs.readFileSync(packageNLSPath).toString());
@@ -519,19 +568,16 @@ export function translatePackageJSON(packageJSON: string, packageNLSPath: string
const extensionsPath = path.join(root, 'extensions'); const extensionsPath = path.join(root, 'extensions');
// Additional projects to webpack. These typically build code for webviews
// const webpackMediaConfigFiles = [
// // 'markdown-language-features/webpack.config.js',
// 'simple-browser/webpack.config.js',
// ];
// Additional projects to run esbuild on. These typically build code for webviews // Additional projects to run esbuild on. These typically build code for webviews
// const esbuildMediaScripts = [ const esbuildMediaScripts = [
// 'markdown-language-features/esbuild.js', 'markdown-language-features/esbuild-notebook.js',
// 'markdown-math/esbuild.js', 'markdown-language-features/esbuild-preview.js',
// ]; 'markdown-math/esbuild.js',
// 'notebook-renderers/esbuild.js', {{SQL CARBON EDIT}} We don't have this extension
'simple-browser/esbuild-preview.js',
];
export async function webpackExtensions(taskName: string, isWatch: boolean, webpackConfigLocations: { configPath: string, outputRoot?: string }[]) { export async function webpackExtensions(taskName: string, isWatch: boolean, webpackConfigLocations: { configPath: string; outputRoot?: string }[]) {
const webpack = require('webpack') as typeof import('webpack'); const webpack = require('webpack') as typeof import('webpack');
const webpackConfigs: webpack.Configuration[] = []; const webpackConfigs: webpack.Configuration[] = [];
@@ -600,56 +646,46 @@ export async function webpackExtensions(taskName: string, isWatch: boolean, webp
}); });
} }
// {{SQL CARBON EDIT}} -- remove unused async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: { script: string; outputRoot?: string }[]) {
// async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: { script: string, outputRoot?: string }[]) { function reporter(stdError: string, script: string) {
// function reporter(stdError: string, script: string) { const matches = (stdError || '').match(/\> (.+): error: (.+)?/g);
// const matches = (stdError || '').match(/\> (.+): error: (.+)?/g); fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`);
// fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`); for (const match of matches || []) {
// for (const match of matches || []) { fancyLog.error(match);
// fancyLog.error(match); }
// } }
// }
const tasks = scripts.map(({ script, outputRoot }) => {
// const tasks = scripts.map(({ script, outputRoot }) => { return new Promise<void>((resolve, reject) => {
// return new Promise<void>((resolve, reject) => { const args = [script];
// const args = [script]; if (isWatch) {
// if (isWatch) { args.push('--watch');
// args.push('--watch'); }
// } if (outputRoot) {
// if (outputRoot) { args.push('--outputRoot', outputRoot);
// args.push('--outputRoot', outputRoot); }
// } const proc = cp.execFile(process.argv[0], args, {}, (error, _stdout, stderr) => {
// const proc = cp.execFile(process.argv[0], args, {}, (error, _stdout, stderr) => { if (error) {
// if (error) { return reject(error);
// return reject(error); }
// } reporter(stderr, script);
// reporter(stderr, script); if (stderr) {
// if (stderr) { return reject();
// return reject(); }
// } return resolve();
// return resolve(); });
// });
proc.stdout!.on('data', (data) => {
// proc.stdout!.on('data', (data) => { fancyLog(`${ansiColors.green(taskName)}: ${data.toString('utf8')}`);
// fancyLog(`${ansiColors.green(taskName)}: ${data.toString('utf8')}`); });
// }); });
// }); });
// }); return Promise.all(tasks);
// return Promise.all(tasks); }
// }
export async function buildExtensionMedia(isWatch: boolean, outputRoot?: string) {
export async function buildExtensionMedia(_isWatch: boolean, _outputRoot?: string) { return esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
return undefined; script: path.join(extensionsPath, p),
// return Promise.all([ outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
// // webpackExtensions('webpacking extension media', isWatch, webpackMediaConfigFiles.map(p => { })));
// // return {
// // configPath: path.join(extensionsPath, p),
// // outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
// // };
// // })),
// esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
// script: path.join(extensionsPath, p),
// outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined
// }))),
// ]);
} }

1
product.json
View File

@@ -93,7 +93,6 @@
}, },
"builtInExtensions": [ "builtInExtensions": [
{ {
"version": "1.61.0",
"name": "Microsoft.sqlservernotebook", "name": "Microsoft.sqlservernotebook",
"version": "0.5.0", "version": "0.5.0",
"repo": "https://github.com/microsoft/azuredatastudio" "repo": "https://github.com/microsoft/azuredatastudio"