Move SQL 2019 extension's notebook code into Azure Data Studio (#4090)

extensions/notebook/src/common/apiWrapper.ts

@@ -0,0 +1,82 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as vscode from 'vscode';
+import * as sqlops from 'sqlops';
+
+/**
+ * Wrapper class to act as a facade over VSCode and Data APIs and allow us to test / mock callbacks into
+ * this API from our code
+ *
+ * @export
+ * @class ApiWrapper
+ */
+export class ApiWrapper {
+	public createOutputChannel(name: string): vscode.OutputChannel {
+		return vscode.window.createOutputChannel(name);
+	}
+
+	public createTerminalWithOptions(options: vscode.TerminalOptions): vscode.Terminal {
+		return vscode.window.createTerminal(options);
+	}
+
+	public getCurrentConnection(): Thenable<sqlops.connection.Connection> {
+		return sqlops.connection.getCurrentConnection();
+	}
+
+	public getWorkspacePathFromUri(uri: vscode.Uri): string | undefined {
+		let workspaceFolder = vscode.workspace.getWorkspaceFolder(uri);
+		return workspaceFolder ? workspaceFolder.uri.fsPath : undefined;
+	}
+
+	public registerCommand(command: string, callback: (...args: any[]) => any, thisArg?: any): vscode.Disposable {
+		return vscode.commands.registerCommand(command, callback, thisArg);
+	}
+
+	public registerCompletionItemProvider(selector: vscode.DocumentSelector, provider: vscode.CompletionItemProvider, ...triggerCharacters: string[]): vscode.Disposable {
+		return vscode.languages.registerCompletionItemProvider(selector, provider, ...triggerCharacters);
+	}
+
+	public registerTaskHandler(taskId: string, handler: (profile: sqlops.IConnectionProfile) => void): void {
+		sqlops.tasks.registerTask(taskId, handler);
+	}
+
+	public showErrorMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+		return vscode.window.showErrorMessage(message, ...items);
+	}
+
+	public showOpenDialog(options: vscode.OpenDialogOptions): Thenable<vscode.Uri[] | undefined> {
+		return vscode.window.showOpenDialog(options);
+	}
+
+	public startBackgroundOperation(operationInfo: sqlops.BackgroundOperationInfo): void {
+		sqlops.tasks.startBackgroundOperation(operationInfo);
+	}
+
+	/**
+	 * Get the configuration for a extensionName
+	 * @param extensionName The string name of the extension to get the configuration for
+	 * @param resource The optional URI, as a URI object or a string, to use to get resource-scoped configurations
+	 */
+	public getConfiguration(extensionName?: string, resource?: vscode.Uri | string): vscode.WorkspaceConfiguration {
+		if (typeof resource === 'string') {
+			try {
+				resource = this.parseUri(resource);
+			} catch (e) {
+				resource = undefined;
+			}
+		} else if (!resource) {
+			// Fix to avoid adding lots of errors to debug console. Expects a valid resource or null, not undefined
+			resource = null;
+		}
+		return vscode.workspace.getConfiguration(extensionName, resource as vscode.Uri);
+	}
+
+	public parseUri(uri: string): vscode.Uri {
+		return vscode.Uri.parse(uri);
+	}
+}

extensions/notebook/src/common/appContext.ts

@@ -0,0 +1,28 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as vscode from 'vscode';
+import { ApiWrapper } from './apiWrapper';
+
+/**
+ * Global context for the application
+ */
+export class AppContext {
+
+	private serviceMap: Map<string, any> = new Map();
+	constructor(public readonly extensionContext: vscode.ExtensionContext, public readonly apiWrapper: ApiWrapper) {
+		this.apiWrapper = apiWrapper || new ApiWrapper();
+	}
+
+	public getService<T>(serviceName: string): T {
+		return this.serviceMap.get(serviceName) as T;
+	}
+
+	public registerService<T>(serviceName: string, service: T): void {
+		this.serviceMap.set(serviceName, service);
+	}
+}

extensions/notebook/src/common/constants.ts

@@ -0,0 +1,48 @@
+'use strict';
+
+// CONFIG VALUES ///////////////////////////////////////////////////////////
+export const extensionConfigSectionName = 'dataManagement';
+export const extensionOutputChannel = 'SQL Server 2019 Preview';
+export const configLogDebugInfo = 'logDebugInfo';
+
+// JUPYTER CONFIG //////////////////////////////////////////////////////////
+export const pythonBundleVersion = '0.0.1';
+export const pythonVersion = '3.6.6';
+export const sparkMagicVersion = '0.12.6.1';
+export const python3 = 'python3';
+export const pysparkkernel = 'pysparkkernel';
+export const sparkkernel = 'sparkkernel';
+export const pyspark3kernel = 'pyspark3kernel';
+export const python3DisplayName = 'Python 3';
+export const defaultSparkKernel = 'pyspark3kernel';
+export const pythonPathConfigKey = 'pythonPath';
+export const notebookConfigKey = 'notebook';
+
+export const outputChannelName = 'dataManagement';
+export const hdfsHost = 'host';
+export const hdfsUser = 'user';
+
+export const winPlatform = 'win32';
+
+export const jupyterNotebookProviderId = 'jupyter';
+export const jupyterConfigRootFolder = 'jupyter_config';
+export const jupyterKernelsMasterFolder = 'kernels_master';
+export const jupyterNotebookLanguageId = 'jupyter-notebook';
+export const jupyterNotebookViewType = 'jupyter-notebook';
+export const jupyterNewNotebookTask = 'jupyter.task.newNotebook';
+export const jupyterOpenNotebookTask = 'jupyter.task.openNotebook';
+export const jupyterNewNotebookCommand = 'jupyter.cmd.newNotebook';
+export const jupyterCommandSetContext = 'jupyter.setContext';
+export const jupyterCommandSetKernel = 'jupyter.setKernel';
+export const jupyterReinstallDependenciesCommand = 'jupyter.reinstallDependencies';
+export const jupyterAnalyzeCommand = 'jupyter.cmd.analyzeNotebook';
+export const jupyterInstallPackages = 'jupyter.cmd.installPackages';
+export const jupyterConfigurePython = 'jupyter.cmd.configurePython';
+
+export enum BuiltInCommands {
+	SetContext = 'setContext'
+}
+
+export enum CommandContext {
+	WizardServiceEnabled = 'wizardservice:enabled'
+}

extensions/notebook/src/common/localizedConstants.ts

@@ -0,0 +1,19 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+// General Constants ///////////////////////////////////////////////////////
+export const msgYes = localize('msgYes', 'Yes');
+export const msgNo = localize('msgNo', 'No');
+
+// Jupyter Constants ///////////////////////////////////////////////////////
+export const msgManagePackagesPowershell = localize('msgManagePackagesPowershell', '<#\n--------------------------------------------------------------------------------\n\tThis is the sandboxed instance of python used by Jupyter server.\n\tTo install packages used by the python kernel use \'.\\python.exe -m pip install\'\n--------------------------------------------------------------------------------\n#>');
+export const msgManagePackagesBash = localize('msgJupyterManagePackagesBash', ': \'\n--------------------------------------------------------------------------------\n\tThis is the sandboxed instance of python used by Jupyter server.\n\tTo install packages used by the python kernel use \'./python3.6 -m pip install\'\n--------------------------------------------------------------------------------\n\'');
+export const msgManagePackagesCmd = localize('msgJupyterManagePackagesCmd', 'REM  This is the sandboxed instance of python used by Jupyter server. To install packages used by the python kernel use \'.\\python.exe -m pip install\'');
+export const msgSampleCodeDataFrame = localize('msgSampleCodeDataFrame', 'This sample code loads the file into a data frame and shows the first 10 results.');

extensions/notebook/src/common/notebookUtils.ts

@@ -0,0 +1,24 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as crypto from 'crypto';
+
+/**
+ * Creates a random token per https://nodejs.org/api/crypto.html#crypto_crypto_randombytes_size_callback.
+ * Defaults to 24 bytes, which creates a 48-char hex string
+ */
+export function getRandomToken(size: number = 24): Promise<string> {
+	return new Promise((resolve, reject) => {
+		crypto.randomBytes(size, (err, buffer) => {
+			if (err) {
+				reject(err);
+			}
+			let token = buffer.toString('hex');
+			resolve(token);
+		});
+	});
+}

extensions/notebook/src/common/ports.ts

@@ -0,0 +1,115 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+// This code is originally from https://github.com/Microsoft/vscode/blob/master/src/vs/base/node/ports.ts
+
+'use strict';
+
+import * as net from 'net';
+
+export class StrictPortFindOptions {
+	constructor(public startPort: number, public minPort: number, public maxport: number) {
+	}
+	public maxRetriesPerStartPort: number = 5;
+	public totalRetryLoops: number = 10;
+	public timeout: number = 5000;
+}
+
+/**
+ * Searches for a free port with additional retries and a function to search in a much larger range if initial
+ * attempt to find a port fails. By skipping to a random port after the first time failing, this should help
+ * reduce the likelihood that no free port can be found.
+ */
+export async function strictFindFreePort(options: StrictPortFindOptions): Promise<number> {
+	let totalRetries = options.totalRetryLoops;
+	let startPort = options.startPort;
+	let port = await findFreePort(startPort, options.maxRetriesPerStartPort, options.timeout);
+	while (port === 0 && totalRetries > 0) {
+		startPort = getRandomInt(options.minPort, options.maxport);
+		port = await findFreePort(startPort, options.maxRetriesPerStartPort, options.timeout);
+		totalRetries--;
+	}
+	return port;
+}
+
+/**
+ * Get a random integer between `min` and `max`.
+ *
+ * @param {number} min - min number
+ * @param {number} max - max number
+ * @return {number} a random integer
+ */
+function getRandomInt(min, max): number {
+	return Math.floor(Math.random() * (max - min + 1) + min);
+}
+
+/**
+ * Given a start point and a max number of retries, will find a port that
+ * is openable. Will return 0 in case no free port can be found.
+ */
+export function findFreePort(startPort: number, giveUpAfter: number, timeout: number): Thenable<number> {
+	let done = false;
+
+	return new Promise(resolve => {
+		const timeoutHandle = setTimeout(() => {
+			if (!done) {
+				done = true;
+				return resolve(0);
+			}
+		}, timeout);
+
+		doFindFreePort(startPort, giveUpAfter, (port) => {
+			if (!done) {
+				done = true;
+				clearTimeout(timeoutHandle);
+				return resolve(port);
+			}
+		});
+	});
+}
+
+function doFindFreePort(startPort: number, giveUpAfter: number, clb: (port: number) => void): void {
+	if (giveUpAfter === 0) {
+		return clb(0);
+	}
+
+	const client = new net.Socket();
+
+	// If we can connect to the port it means the port is already taken so we continue searching
+	client.once('connect', () => {
+		dispose(client);
+
+		return doFindFreePort(startPort + 1, giveUpAfter - 1, clb);
+	});
+
+	client.once('data', () => {
+		// this listener is required since node.js 8.x
+	});
+
+	client.once('error', (err: Error & { code?: string }) => {
+		dispose(client);
+
+		// If we receive any non ECONNREFUSED error, it means the port is used but we cannot connect
+		if (err.code !== 'ECONNREFUSED') {
+			return doFindFreePort(startPort + 1, giveUpAfter - 1, clb);
+		}
+
+		// Otherwise it means the port is free to use!
+		return clb(startPort);
+	});
+
+	client.connect(startPort, '127.0.0.1');
+}
+
+function dispose(socket: net.Socket): void {
+	try {
+		socket.removeAllListeners('connect');
+		socket.removeAllListeners('error');
+		socket.end();
+		socket.destroy();
+		socket.unref();
+	} catch (error) {
+		console.error(error); // otherwise this error would get lost in the callback chain
+	}
+}

extensions/notebook/src/common/promise.ts

@@ -0,0 +1,26 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+/**
+ * Deferred promise
+ */
+export class Deferred<T> {
+	promise: Promise<T>;
+	resolve: (value?: T | PromiseLike<T>) => void;
+	reject: (reason?: any) => void;
+	constructor() {
+		this.promise = new Promise<T>((resolve, reject) => {
+			this.resolve = resolve;
+			this.reject = reject;
+		});
+	}
+
+	then<TResult>(onfulfilled?: (value: T) => TResult | Thenable<TResult>, onrejected?: (reason: any) => TResult | Thenable<TResult>): Thenable<TResult>;
+	then<TResult>(onfulfilled?: (value: T) => TResult | Thenable<TResult>, onrejected?: (reason: any) => void): Thenable<TResult>;
+	then<TResult>(onfulfilled?: (value: T) => TResult | Thenable<TResult>, onrejected?: (reason: any) => TResult | Thenable<TResult>): Thenable<TResult> {
+		return this.promise.then(onfulfilled, onrejected);
+	}
+}

extensions/notebook/src/common/utils.ts

@@ -0,0 +1,126 @@
+'use strict';
+
+import * as childProcess from 'child_process';
+import * as fs from 'fs-extra';
+import * as nls from 'vscode-nls';
+import * as vscode from 'vscode';
+
+const localize = nls.loadMessageBundle();
+
+export function getKnoxUrl(host: string, port: string): string {
+	return `https://${host}:${port}/gateway`;
+}
+
+export function getLivyUrl(serverName: string, port: string): string {
+	return this.getKnoxUrl(serverName, port) + '/default/livy/v1/';
+}
+
+export async function mkDir(dirPath: string, outputChannel?: vscode.OutputChannel): Promise<void> {
+	if (!await fs.exists(dirPath)) {
+		if (outputChannel) {
+			outputChannel.appendLine(localize('mkdirOutputMsg', '... Creating {0}', dirPath));
+		}
+		await fs.ensureDir(dirPath);
+	}
+}
+
+export function getErrorMessage(error: Error | string): string {
+	return (error instanceof Error) ? error.message : error;
+}
+
+// COMMAND EXECUTION HELPERS ///////////////////////////////////////////////
+export function executeBufferedCommand(cmd: string, options: childProcess.ExecOptions, outputChannel?: vscode.OutputChannel): Thenable<string> {
+	return new Promise<string>((resolve, reject) => {
+		if (outputChannel) {
+			outputChannel.appendLine(`    > ${cmd}`);
+		}
+
+		let child = childProcess.exec(cmd, options, (err, stdout) => {
+			if (err) {
+				reject(err);
+			} else {
+				resolve(stdout);
+			}
+		});
+
+		// Add listeners to print stdout and stderr if an output channel was provided
+		if (outputChannel) {
+			child.stdout.on('data', data => { outputDataChunk(data, outputChannel, '    stdout: '); });
+			child.stderr.on('data', data => { outputDataChunk(data, outputChannel, '    stderr: '); });
+		}
+	});
+}
+
+export function executeStreamedCommand(cmd: string, outputChannel?: vscode.OutputChannel): Thenable<void> {
+	return new Promise<void>((resolve, reject) => {
+		// Start the command
+		if (outputChannel) {
+			outputChannel.appendLine(`    > ${cmd}`);
+		}
+		let child = childProcess.spawn(cmd, [], { shell: true, detached: false });
+
+		// Add listeners to resolve/reject the promise on exit
+		child.on('error', reject);
+		child.on('exit', (code: number) => {
+			if (code === 0) {
+				resolve();
+			} else {
+				reject(localize('executeCommandProcessExited', 'Process exited with code {0}', code));
+			}
+		});
+
+		// Add listeners to print stdout and stderr if an output channel was provided
+		if (outputChannel) {
+			child.stdout.on('data', data => { outputDataChunk(data, outputChannel, '    stdout: '); });
+			child.stderr.on('data', data => { outputDataChunk(data, outputChannel, '    stderr: '); });
+		}
+	});
+}
+
+export function getUserHome(): string {
+	return process.env.HOME || process.env.USERPROFILE;
+}
+
+export enum Platform {
+	Mac,
+	Linux,
+	Windows,
+	Others
+}
+
+export function getOSPlatform(): Platform {
+	switch (process.platform) {
+		case 'win32':
+			return Platform.Windows;
+		case 'darwin':
+			return Platform.Mac;
+		case 'linux':
+			return Platform.Linux;
+		default:
+			return Platform.Others;
+	}
+}
+
+export function getOSPlatformId(): string {
+	var platformId = undefined;
+	switch (process.platform) {
+		case 'win32':
+			platformId = 'win-x64';
+			break;
+		case 'darwin':
+			platformId = 'osx';
+			break;
+		default:
+			platformId = 'linux-x64';
+			break;
+	}
+	return platformId;
+}
+
+// PRIVATE HELPERS /////////////////////////////////////////////////////////
+function outputDataChunk(data: string | Buffer, outputChannel: vscode.OutputChannel, header: string): void {
+	data.toString().split(/\r?\n/)
+		.forEach(line => {
+			outputChannel.appendLine(header + line);
+		});
+}