Enable docker build in CI pipeline (#15476)

* Update gulpfile.reh.js

* Update YAML files

* Bump distro

* Fix yaml file

* Another yaml typo

* Disable web publish step

* Update the build scripts

* YAML update

* Update distro

* Update yaml

* Increase step timeout

* Add comment for disabled code block

* Temp disable component detection task

* Renable comp gov teask

* Bump distro

* Update build scripts

* Fix typo

* Fix docker path

* Bump linux build timeout

* Change file name
This commit is contained in:
Karl Burtram
2021-05-14 17:58:50 -07:00
committed by GitHub
parent c54f991e60
commit 2a0d37aee0
10 changed files with 417 additions and 43 deletions

View File

@@ -7,7 +7,7 @@ SERVER_BUILD_NAME="azuredatastudio-server-$PLATFORM_LINUX"
# create docker
mkdir -p $REPO/.build/docker
docker build -t azuredatastudio-server -f $REPO/build/azure-pipelines/docker/Dockerfile $ROOT/$SERVER_BUILD_NAME
docker build -t azuredatastudio-server -f $REPO/build/azure-pipelines/docker/Dockerfile $ROOT/$SERVER_BUILD_NAME-web
docker save azuredatastudio-server | gzip > $REPO/.build/docker/azuredatastudio-server-docker.tar.gz
node build/azure-pipelines/common/copyArtifacts.js

View File

@@ -79,7 +79,7 @@ steps:
set -e
for f in $(Build.SourcesDirectory)/.build/drop/linux/server/*.tar.gz
do
tar -C $(agent.builddirectory) -zxvf $f
tar -C $(Build.SourcesDirectory)/../ -zxvf $f
rm $f
done
displayName: Unzip artifacts

View File

@@ -14,13 +14,22 @@ TARBALL_PATH="$REPO/.build/linux/archive/$TARBALL_FILENAME"
rm -rf $ROOT/code-*.tar.*
(cd $ROOT && tar -czf $TARBALL_PATH $BUILDNAME)
# Publish Remote Extension Host
LEGACY_SERVER_BUILD_NAME="azuredatastudio-reh-$PLATFORM_LINUX"
SERVER_BUILD_NAME="azuredatastudio-server-$PLATFORM_LINUX"
SERVER_TARBALL_FILENAME="azuredatastudio-server-$PLATFORM_LINUX.tar.gz"
SERVER_TARBALL_PATH="$REPO/.build/linux/server/$SERVER_TARBALL_FILENAME"
# # Publish Remote Extension Host
# LEGACY_SERVER_BUILD_NAME="azuredatastudio-reh-$PLATFORM_LINUX"
# SERVER_BUILD_NAME="azuredatastudio-server-$PLATFORM_LINUX"
# SERVER_TARBALL_FILENAME="azuredatastudio-server-$PLATFORM_LINUX.tar.gz"
# SERVER_TARBALL_PATH="$REPO/.build/linux/server/$SERVER_TARBALL_FILENAME"
# rm -rf $ROOT/azuredatastudio-server-*.tar.*
# (cd $ROOT && mv $LEGACY_SERVER_BUILD_NAME $SERVER_BUILD_NAME && tar --owner=0 --group=0 -czf $SERVER_TARBALL_PATH $SERVER_BUILD_NAME)
# Publish Remote Extension Host (Web)
LEGACY_SERVER_BUILD_NAME_WEB="azuredatastudio-reh-web-$PLATFORM_LINUX"
SERVER_BUILD_NAME_WEB="azuredatastudio-server-$PLATFORM_LINUX-web"
SERVER_TARBALL_FILENAME_WEB="azuredatastudio-server-$PLATFORM_LINUX-web.tar.gz"
SERVER_TARBALL_PATH_WEB="$REPO/.build/linux/server/$SERVER_TARBALL_FILENAME_WEB"
rm -rf $ROOT/azuredatastudio-server-*.tar.*
(cd $ROOT && mv $LEGACY_SERVER_BUILD_NAME $SERVER_BUILD_NAME && tar --owner=0 --group=0 -czf $SERVER_TARBALL_PATH $SERVER_BUILD_NAME)
(cd $ROOT && mv vscode-reh-web-linux-x64 $SERVER_BUILD_NAME_WEB && tar --owner=0 --group=0 -czf $SERVER_TARBALL_PATH_WEB $SERVER_BUILD_NAME_WEB)
node build/azure-pipelines/common/copyArtifacts.js

View File

@@ -218,6 +218,19 @@ steps:
displayName: 'Signing Extensions'
condition: and(succeeded(), eq(variables['signed'], true))
- script: |
set -e
cd ./extensions/mssql/node_modules/@microsoft/ads-kerberos
# npx node-gyp rebuild
yarn install
displayName: Recompile native node modules
- script: |
set -e
VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
yarn gulp vscode-reh-web-linux-x64-min
displayName: Build web server
- script: |
set -e
./build/azure-pipelines/linux/createDrop.sh

View File

@@ -47,7 +47,7 @@ jobs:
- template: linux/sql-product-build-linux.yml
parameters:
extensionsToUnitTest: ["admin-tool-ext-win", "agent", "azdata", "azurecore", "cms", "dacpac", "import", "schema-compare", "notebook", "resource-deployment", "machine-learning", "sql-database-projects", "data-workspace"]
timeoutInMinutes: 70
timeoutInMinutes: 300
- job: LinuxWeb
condition: and(succeeded(), eq(variables['VSCODE_BUILD_WEB'], 'true'), ne(variables['VSCODE_QUALITY'], 'saw'))
@@ -61,15 +61,15 @@ jobs:
steps:
- template: web/sql-product-build-web.yml
# - job: Docker
# condition: and(succeeded(), eq(variables['VSCODE_BUILD_DOCKER'], 'true'))
# pool:
# vmImage: 'Ubuntu-16.04'
# container: linux-x64
# dependsOn:
# - Linux
# steps:
# - template: docker/sql-product-build-docker.yml
- job: Docker
condition: and(succeeded(), eq(variables['VSCODE_BUILD_DOCKER'], 'true'))
pool:
vmImage: 'Ubuntu-16.04'
container: linux-x64
dependsOn:
- Linux
steps:
- template: docker/sql-product-build-docker.yml
- job: Windows
condition: and(succeeded(), eq(variables['VSCODE_BUILD_WIN32'], 'true'))
@@ -79,7 +79,7 @@ jobs:
- Compile
steps:
- template: win32/sql-product-build-win32.yml
timeoutInMinutes: 70
timeoutInMinutes: 300
- job: Windows_Test
condition: and(succeeded(), eq(variables['VSCODE_BUILD_WIN32'], 'true'))

View File

@@ -99,3 +99,4 @@ steps:
# VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
# ./build/azure-pipelines/web/publish.sh
# displayName: Publish

View File

@@ -102,8 +102,6 @@ steps:
$ErrorActionPreference = "Stop"
exec { yarn gulp "package-rebuild-extensions" }
exec { yarn gulp "vscode-win32-x64-min-ci" }
exec { yarn gulp "vscode-reh-win32-x64-min" }
exec { yarn gulp "vscode-reh-web-win32-x64-min" }
exec { yarn gulp "vscode-win32-x64-code-helper" }
exec { yarn gulp "vscode-win32-x64-inno-updater" }
displayName: Build

View File

@@ -6,21 +6,35 @@
'use strict';
const gulp = require('gulp');
const path = require('path');
const es = require('event-stream');
const util = require('./lib/util');
const task = require('./lib/task');
const common = require('./lib/optimize');
const product = require('../product.json');
const rename = require('gulp-rename');
const replace = require('gulp-replace');
const filter = require('gulp-filter');
const _ = require('underscore');
const { getProductionDependencies } = require('./lib/dependencies');
const vfs = require('vinyl-fs');
const packageJson = require('../package.json');
const flatmap = require('gulp-flatmap');
const gunzip = require('gulp-gunzip');
const File = require('vinyl');
const fs = require('fs');
const rename = require('gulp-rename');
const filter = require('gulp-filter');
const glob = require('glob');
const { compileBuildTask } = require('./gulpfile.compile');
const { compileExtensionsBuildTask } = require('./gulpfile.extensions');
const { vscodeWebEntryPoints, vscodeWebResourceIncludes, createVSCodeWebFileContentMapper } = require('./gulpfile.vscode.web');
const cp = require('child_process');
const REPO_ROOT = path.dirname(__dirname);
const commit = util.getVersion(REPO_ROOT);
const BUILD_ROOT = path.dirname(REPO_ROOT);
const REMOTE_FOLDER = path.join(REPO_ROOT, 'remote');
// Targets
const BUILD_TARGETS = [
{ platform: 'win32', arch: 'ia32', pkgTarget: 'node8-win-x86' },
@@ -33,21 +47,92 @@ const BUILD_TARGETS = [
{ platform: 'linux', arch: 'alpine', pkgTarget: 'node8-linux-alpine' },
];
const noop = () => { return Promise.resolve(); };
const serverResources = [
gulp.task('vscode-reh-win32-ia32-min', noop);
gulp.task('vscode-reh-win32-x64-min', noop);
gulp.task('vscode-reh-darwin-min', noop);
gulp.task('vscode-reh-linux-x64-min', noop);
gulp.task('vscode-reh-linux-armhf-min', noop);
gulp.task('vscode-reh-linux-arm64-min', noop);
gulp.task('vscode-reh-linux-alpine-min', noop);
// Bootstrap
'out-build/bootstrap.js',
'out-build/bootstrap-fork.js',
'out-build/bootstrap-amd.js',
'out-build/bootstrap-node.js',
'out-build/paths.js',
gulp.task('vscode-reh-web-win32-ia32-min', noop);
gulp.task('vscode-reh-web-win32-x64-min', noop);
gulp.task('vscode-reh-web-darwin-min', noop);
gulp.task('vscode-reh-web-linux-x64-min', noop);
gulp.task('vscode-reh-web-linux-alpine-min', noop);
// Performance
'out-build/vs/base/common/performance.js',
// main entry points
'out-build/vs/server/cli.js',
'out-build/vs/server/main.js',
// Watcher
'out-build/vs/platform/files/**/*.exe',
'out-build/vs/platform/files/**/*.md',
// Uri transformer
'out-build/vs/server/uriTransformer.js',
// Process monitor
'out-build/vs/base/node/cpuUsage.sh',
'out-build/vs/base/node/ps.sh',
'!**/test/**'
];
let serverWithWebResources = undefined;
try {
serverWithWebResources = [
// Include all of server...
...serverResources,
// ...and all of web
...vscodeWebResourceIncludes
];
} catch (err) {
serverWithWebResources = [
// Include all of server...
...serverResources
];
}
const serverEntryPoints = [
{
name: 'vs/server/remoteExtensionHostAgent',
exclude: ['vs/css', 'vs/nls']
},
{
name: 'vs/server/remoteCli',
exclude: ['vs/css', 'vs/nls']
},
{
name: 'vs/server/remoteExtensionHostProcess',
exclude: ['vs/css', 'vs/nls']
},
{
name: 'vs/platform/files/node/watcher/unix/watcherApp',
exclude: ['vs/css', 'vs/nls']
},
{
name: 'vs/platform/files/node/watcher/nsfw/watcherApp',
exclude: ['vs/css', 'vs/nls']
}
];
let serverWithWebEntryPoints = undefined;
try {
serverWithWebEntryPoints = [
// Include all of server
...serverEntryPoints,
// Include workbench web
...vscodeWebEntryPoints
];
} catch (err) {
serverWithWebEntryPoints = [
// Include all of server
...serverEntryPoints
];
}
function getNodeVersion() {
const yarnrc = fs.readFileSync(path.join(REPO_ROOT, 'remote', '.yarnrc'), 'utf8');
@@ -116,6 +201,270 @@ function nodejs(platform, arch) {
.pipe(rename('node'));
}
function packageTask(type, platform, arch, sourceFolderName, destinationFolderName) {
const destination = path.join(BUILD_ROOT, destinationFolderName);
return () => {
const json = require('gulp-json-editor');
const src = gulp.src(sourceFolderName + '/**', { base: '.' })
.pipe(rename(function (path) { path.dirname = path.dirname.replace(new RegExp('^' + sourceFolderName), 'out'); }))
.pipe(util.setExecutableBit(['**/*.sh']))
.pipe(filter(['**', '!**/*.js.map']));
const workspaceExtensionPoints = ['debuggers', 'jsonValidation'];
const isUIExtension = (manifest) => {
switch (manifest.extensionKind) {
case 'ui': return true;
case 'workspace': return false;
default: {
if (manifest.main) {
return false;
}
if (manifest.contributes && Object.keys(manifest.contributes).some(key => workspaceExtensionPoints.indexOf(key) !== -1)) {
return false;
}
// Default is UI Extension
return true;
}
}
};
const localWorkspaceExtensions = glob.sync('extensions/*/package.json')
.filter((extensionPath) => {
if (type === 'reh-web') {
return true; // web: ship all extensions for now
}
const manifest = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, extensionPath)).toString());
return !isUIExtension(manifest);
}).map((extensionPath) => path.basename(path.dirname(extensionPath)))
.filter(name => name !== 'vscode-api-tests' && name !== 'vscode-test-resolver'); // Do not ship the test extensions
const marketplaceExtensions = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'product.json'), 'utf8')).builtInExtensions
.filter(entry => !entry.platforms || new Set(entry.platforms).has(platform))
.filter(entry => !entry.clientOnly)
.map(entry => entry.name);
const extensionPaths = [...localWorkspaceExtensions, ...marketplaceExtensions]
.map(name => `.build/extensions/${name}/**`);
const extensions = gulp.src(extensionPaths, { base: '.build', dot: true });
const extensionsCommonDependencies = gulp.src('.build/extensions/node_modules/**', { base: '.build', dot: true });
const sources = es.merge(src, extensions, extensionsCommonDependencies)
.pipe(filter(['**', '!**/*.js.map'], { dot: true }));
let version = packageJson.version;
const quality = product.quality;
if (quality && quality !== 'stable') {
version += '-' + quality;
}
const name = product.nameShort;
const packageJsonStream = gulp.src(['remote/package.json'], { base: 'remote' })
.pipe(json({ name, version }));
const date = new Date().toISOString();
const productJsonStream = gulp.src(['product.json'], { base: '.' })
.pipe(json({ commit, date }));
const license = gulp.src(['remote/LICENSE'], { base: 'remote' });
const jsFilter = util.filter(data => !data.isDirectory() && /\.js$/.test(data.path));
const productionDependencies = getProductionDependencies(REMOTE_FOLDER);
const dependenciesSrc = _.flatten(productionDependencies.map(d => path.relative(REPO_ROOT, d.path)).map(d => [`${d}/**`, `!${d}/**/{test,tests}/**`, `!${d}/.bin/**`]));
const deps = gulp.src(dependenciesSrc, { base: 'remote', dot: true })
// filter out unnecessary files, no source maps in server build
.pipe(filter(['**', '!**/package-lock.json', '!**/yarn.lock', '!**/*.js.map']))
.pipe(util.cleanNodeModules(path.join(__dirname, '.moduleignore')))
.pipe(jsFilter)
.pipe(util.stripSourceMappingURL())
.pipe(jsFilter.restore);
const nodePath = `.build/node/v${nodeVersion}/${platform}-${platform === 'darwin' ? 'x64' : arch}`;
const node = gulp.src(`${nodePath}/**`, { base: nodePath, dot: true });
let web = [];
if (type === 'reh-web') {
web = [
'resources/server/favicon.ico',
'resources/server/code-192.png',
'resources/server/code-512.png',
'resources/server/manifest.json'
].map(resource => gulp.src(resource, { base: '.' }).pipe(rename(resource)));
}
let all = es.merge(
packageJsonStream,
productJsonStream,
license,
sources,
deps,
node,
...web
);
let result = all
.pipe(util.skipDirectories())
.pipe(util.fixWin32DirectoryPermissions());
if (platform === 'win32') {
result = es.merge(result,
gulp.src('resources/server/bin/code.cmd', { base: '.' })
.pipe(replace('@@VERSION@@', version))
.pipe(replace('@@COMMIT@@', commit))
.pipe(replace('@@APPNAME@@', product.applicationName))
.pipe(rename(`bin/${product.applicationName}.cmd`)),
// gulp.src('resources/server/bin/helpers/browser.cmd', { base: '.' })
// .pipe(replace('@@VERSION@@', version))
// .pipe(replace('@@COMMIT@@', commit))
// .pipe(replace('@@APPNAME@@', product.applicationName))
// .pipe(rename(`bin/helpers/browser.cmd`)),
gulp.src('resources/server/bin/server.cmd', { base: '.' })
.pipe(rename(`server.cmd`))
);
} else if (platform === 'linux' || platform === 'darwin') {
result = es.merge(result,
gulp.src('resources/server/bin/code.sh', { base: '.' })
.pipe(replace('@@VERSION@@', version))
.pipe(replace('@@COMMIT@@', commit))
.pipe(replace('@@APPNAME@@', product.applicationName))
.pipe(rename(`bin/${product.applicationName}`))
.pipe(util.setExecutableBit()),
// gulp.src('resources/server/bin/helpers/browser.sh', { base: '.' })
// .pipe(replace('@@VERSION@@', version))
// .pipe(replace('@@COMMIT@@', commit))
// .pipe(replace('@@APPNAME@@', product.applicationName))
// .pipe(rename(`bin/helpers/browser.sh`))
// .pipe(util.setExecutableBit()),
gulp.src('resources/server/bin/server.sh', { base: '.' })
.pipe(rename(`server.sh`))
.pipe(util.setExecutableBit())
);
}
return result.pipe(vfs.dest(destination));
};
}
function copyConfigTask(folder) {
const destination = path.join(BUILD_ROOT, folder);
return () => {
const json = require('gulp-json-editor');
return gulp.src(['remote/pkg-package.json'], { base: 'remote' })
.pipe(rename(path => path.basename += '.' + folder))
.pipe(json(obj => {
const pkg = obj.pkg;
pkg.scripts = pkg.scripts && pkg.scripts.map(p => path.join(destination, p));
pkg.assets = pkg.assets && pkg.assets.map(p => path.join(destination, p));
return obj;
}))
.pipe(vfs.dest('out-vscode-reh-pkg'));
};
}
function copyNativeTask(folder) {
const destination = path.join(BUILD_ROOT, folder);
return () => {
const nativeLibraries = gulp.src(['remote/node_modules/**/*.node']);
const license = gulp.src(['remote/LICENSE']);
const result = es.merge(
nativeLibraries,
license
);
return result
.pipe(rename({ dirname: '' }))
.pipe(vfs.dest(destination));
};
}
function packagePkgTask(platform, arch, pkgTarget) {
const folder = path.join(BUILD_ROOT, 'vscode-reh') + (platform ? '-' + platform : '') + (arch ? '-' + arch : '');
return () => {
const cwd = process.cwd();
const config = path.join(cwd, 'out-vscode-reh-pkg', 'pkg-package.vscode-reh-' + platform + '-' + arch + '.json');
process.chdir(folder);
console.log(`TODO`, pkgTarget, config);
return null;
// return pkg.exec(['-t', pkgTarget, '-d', '-c', config, '-o', path.join(folder + '-pkg', platform === 'win32' ? 'vscode-reh.exe' : 'vscode-reh'), './out/remoteExtensionHostAgent.js'])
// .then(() => process.chdir(cwd));
};
}
['reh', 'reh-web'].forEach(type => {
const optimizeTask = task.define(`optimize-vscode-${type}`, task.series(
util.rimraf(`out-vscode-${type}`),
common.optimizeTask({
src: 'out-build',
entryPoints: _.flatten(type === 'reh' ? serverEntryPoints : serverWithWebEntryPoints),
otherSources: [],
resources: type === 'reh' ? serverResources : serverWithWebResources,
loaderConfig: common.loaderConfig(),
out: `out-vscode-${type}`,
inlineAmdImages: true,
bundleInfo: undefined,
fileContentMapper: createVSCodeWebFileContentMapper ? createVSCodeWebFileContentMapper('.build/extensions') : undefined
})
));
const minifyTask = task.define(`minify-vscode-${type}`, task.series(
optimizeTask,
util.rimraf(`out-vscode-${type}-min`),
common.minifyTask(`out-vscode-${type}`, `https://ticino.blob.core.windows.net/sourcemaps/${commit}/core`)
));
gulp.task(minifyTask);
BUILD_TARGETS.forEach(buildTarget => {
const dashed = (str) => (str ? `-${str}` : ``);
const platform = buildTarget.platform;
const arch = buildTarget.arch;
const pkgTarget = buildTarget.pkgTarget;
const copyPkgConfigTask = task.define(`copy-pkg-config${dashed(platform)}${dashed(arch)}`, task.series(
util.rimraf(`out-vscode-${type}-pkg`),
copyConfigTask(`vscode-${type}${dashed(platform)}${dashed(arch)}`)
));
const copyPkgNativeTask = task.define(`copy-pkg-native${dashed(platform)}${dashed(arch)}`, task.series(
util.rimraf(path.join(BUILD_ROOT, `vscode-${type}${dashed(platform)}${dashed(arch)}-pkg`)),
copyNativeTask(`vscode-${type}${dashed(platform)}${dashed(arch)}-pkg`)
));
['', 'min'].forEach(minified => {
const sourceFolderName = `out-vscode-${type}${dashed(minified)}`;
const destinationFolderName = `vscode-${type}${dashed(platform)}${dashed(arch)}`;
const serverTaskCI = task.define(`vscode-${type}${dashed(platform)}${dashed(arch)}${dashed(minified)}-ci`, task.series(
gulp.task(`node-${platform}-${platform === 'darwin' ? 'x64' : arch}`),
util.rimraf(path.join(BUILD_ROOT, destinationFolderName)),
packageTask(type, platform, arch, sourceFolderName, destinationFolderName)
));
gulp.task(serverTaskCI);
const serverTask = task.define(`vscode-${type}${dashed(platform)}${dashed(arch)}${dashed(minified)}`, task.series(
compileBuildTask,
compileExtensionsBuildTask,
minified ? minifyTask : optimizeTask,
serverTaskCI
));
gulp.task(serverTask);
const serverPkgTask = task.define(`vscode-${type}${dashed(platform)}${dashed(arch)}${dashed(minified)}-pkg`, task.series(
task.parallel(
serverTask,
copyPkgConfigTask,
copyPkgNativeTask
),
packagePkgTask(platform, arch, pkgTarget)
));
gulp.task(serverPkgTask);
});
});
});
function mixinServer(watch) {
const packageJSONPath = path.join(path.dirname(__dirname), 'package.json');
function exec(cmdLine) {

View File

@@ -1,7 +1,7 @@
{
"name": "azuredatastudio",
"version": "1.29.0",
"distro": "60affbaa87aad493d53be9930fcff46d377320ae",
"distro": "99c64148d3393e4f67e531240b870f4cee111e06",
"author": {
"name": "Microsoft Corporation"
},

View File

@@ -399,10 +399,14 @@ export class RemoteTerminalChannelClient {
}
public getTerminalLayoutInfo(): Promise<ITerminalsLayoutInfo | undefined> {
const workspace = this._workspaceContextService.getWorkspace();
const args: IGetTerminalLayoutInfoArgs = {
workspaceId: workspace.id,
};
return this._channel.call<ITerminalsLayoutInfo>('$getTerminalLayoutInfo', args);
// {{SQL CARBON EDIT}} - temp disable this code since it refers to non-implemented method
// - currently remote code is ahead of OSS code and they need to catch-up (karlb 5/13/2021)
// const workspace = this._workspaceContextService.getWorkspace();
// const args: IGetTerminalLayoutInfoArgs = {
// workspaceId: workspace.id,
// };
// return this._channel.call<ITerminalsLayoutInfo>('$getTerminalLayoutInfo', args);
return Promise.resolve(undefined);
}
}