Machine Learning Services extension with package management feature (#8622)

* Machine Learning Services extension with package management feature

extensions/machine-learning-services/src/common/apiWrapper.ts

@@ -0,0 +1,65 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as azdata from 'azdata';
+
+/**
+ * Wrapper class to act as a facade over VSCode and Data APIs and allow us to test / mock callbacks into
+ * this API from our code
+ */
+export class ApiWrapper {
+	public createOutputChannel(name: string): vscode.OutputChannel {
+		return vscode.window.createOutputChannel(name);
+	}
+
+	public createTerminalWithOptions(options: vscode.TerminalOptions): vscode.Terminal {
+		return vscode.window.createTerminal(options);
+	}
+
+	public getCurrentConnection(): Thenable<azdata.connection.ConnectionProfile> {
+		return azdata.connection.getCurrentConnection();
+	}
+
+	public getCredentials(connectionId: string): Thenable<{ [name: string]: string }> {
+		return azdata.connection.getCredentials(connectionId);
+	}
+
+	public registerCommand(command: string, callback: (...args: any[]) => any, thisArg?: any): vscode.Disposable {
+		return vscode.commands.registerCommand(command, callback, thisArg);
+	}
+
+	public executeCommand<T>(command: string, ...rest: any[]): Thenable<T | undefined> {
+		return vscode.commands.executeCommand(command, ...rest);
+	}
+
+	public getUriForConnection(connectionId: string): Thenable<string> {
+		return azdata.connection.getUriForConnection(connectionId);
+	}
+
+	public getProvider<T extends azdata.DataProvider>(providerId: string, providerType: azdata.DataProviderType): T {
+		return azdata.dataprotocol.getProvider<T>(providerId, providerType);
+	}
+
+	public showErrorMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+		return vscode.window.showErrorMessage(message, ...items);
+	}
+
+	public showInfoMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+		return vscode.window.showInformationMessage(message, ...items);
+	}
+
+	public showOpenDialog(options: vscode.OpenDialogOptions): Thenable<vscode.Uri[] | undefined> {
+		return vscode.window.showOpenDialog(options);
+	}
+
+	public startBackgroundOperation(operationInfo: azdata.BackgroundOperationInfo): void {
+		azdata.tasks.startBackgroundOperation(operationInfo);
+	}
+
+	public getExtension(extensionId: string): vscode.Extension<any> | undefined {
+		return vscode.extensions.getExtension(extensionId);
+	}
+}

extensions/machine-learning-services/src/common/config.ts

@@ -0,0 +1,36 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import * as nbExtensionApis from '../typings/notebookServices';
+
+const configFileName = 'config.json';
+
+/**
+ * Extension Configuration
+ */
+export class Config {
+
+	private _configValues: any;
+
+	constructor(private _root: string) {
+	}
+
+	/**
+	 * Loads the config values
+	 */
+	public async load(): Promise<void> {
+		const rawConfig = await fs.readFile(path.join(this._root, configFileName));
+		this._configValues = JSON.parse(rawConfig.toString());
+	}
+
+	/**
+	 * Returns the config value of required packages
+	 */
+	public get requiredPythonPackages(): nbExtensionApis.IPackageDetails[] {
+		return this._configValues.requiredPythonPackages;
+	}
+}

extensions/machine-learning-services/src/common/constants.ts

@@ -0,0 +1,28 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as nls from 'vscode-nls';
+
+const localize = nls.loadMessageBundle();
+
+export const winPlatform = 'win32';
+export const pythonBundleVersion = '0.0.1';
+export const managePackagesCommand = 'jupyter.cmd.managePackages';
+export const mlManagePackagesCommand = 'ml.command.managePackages';
+export const extensionOutputChannel = 'Machine Learning Services';
+export const notebookExtensionName = 'Microsoft.notebook';
+
+// Localized texts
+//
+export const managePackageCommandError = localize('ml.managePackages.error', "Either no connection is available or the server does not have external script enabled.");
+export function installDependenciesError(err: string): string { return localize('ml.installDependencies.error', "Failed to install dependencies. Error: {0}", err); }
+export const installDependenciesMsgTaskName = localize('ml.installDependencies.msgTaskName', "Installing Machine Learning extension dependencies");
+export const installDependenciesPackages = localize('ml.installDependencies.packages', "Installing required packages ...");
+export const installDependenciesPackagesAlreadyInstalled = localize('ml.installDependencies.packagesAlreadyInstalled', "Required packages are already installed.");
+export function installDependenciesGetPackagesError(err: string): string { return localize('ml.installDependencies.getPackagesError', "Failed to get installed python packages. Error: {0}", err); }
+export const packageManagerNoConnection = localize('ml.packageManager.NoConnection', "No connection selected");
+export const notebookExtensionNotLoaded = localize('ml.notebookExtensionNotLoaded', "Notebook extension is not loaded");

extensions/machine-learning-services/src/common/processService.ts

@@ -0,0 +1,81 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as vscode from 'vscode';
+import * as childProcess from 'child_process';
+
+const ExecScriptsTimeoutInSeconds = 600000;
+export class ProcessService {
+
+	public async execScripts(exeFilePath: string, scripts: string[], outputChannel?: vscode.OutputChannel): Promise<void> {
+		return new Promise<void>((resolve, reject) => {
+
+			const scriptExecution = childProcess.spawn(exeFilePath);
+			let output: string;
+			scripts.forEach(script => {
+				scriptExecution.stdin.write(`${script}\n`);
+			});
+			scriptExecution.stdin.end();
+
+			// Add listeners to print stdout and stderr if an output channel was provided
+			if (outputChannel) {
+				scriptExecution.stdout.on('data', data => {
+					this.outputDataChunk(data, outputChannel, '    stdout: ');
+					output = output + data.toString();
+				});
+				scriptExecution.stderr.on('data', data => {
+					this.outputDataChunk(data, outputChannel, '    stderr: ');
+					output = output + data.toString();
+				});
+			}
+
+			scriptExecution.on('exit', (code) => {
+				if (code === 0) {
+					resolve();
+				} else {
+					reject(`Process exited with code: ${code}. output: ${output}`);
+				}
+			});
+			setTimeout(() => {
+				try {
+					scriptExecution.kill();
+				} catch (error) {
+					console.log(error);
+				}
+			}, ExecScriptsTimeoutInSeconds);
+		});
+	}
+
+	public async executeBufferedCommand(cmd: string, outputChannel?: vscode.OutputChannel): Promise<string> {
+		return new Promise<string>((resolve, reject) => {
+			if (outputChannel) {
+				outputChannel.appendLine(`    > ${cmd}`);
+			}
+
+			let child = childProcess.exec(cmd, (err, stdout) => {
+				if (err) {
+					reject(err);
+				} else {
+					resolve(stdout);
+				}
+			});
+
+			// Add listeners to print stdout and stderr if an output channel was provided
+			if (outputChannel) {
+				child.stdout.on('data', data => { this.outputDataChunk(data, outputChannel, '    stdout: '); });
+				child.stderr.on('data', data => { this.outputDataChunk(data, outputChannel, '    stderr: '); });
+			}
+		});
+	}
+
+	private outputDataChunk(data: string | Buffer, outputChannel: vscode.OutputChannel, header: string): void {
+		data.toString().split(/\r?\n/)
+			.forEach(line => {
+				outputChannel.appendLine(header + line);
+			});
+	}
+}

extensions/machine-learning-services/src/common/queryRunner.ts

@@ -0,0 +1,124 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as azdata from 'azdata';
+import * as nbExtensionApis from '../typings/notebookServices';
+import { ApiWrapper } from './apiWrapper';
+
+const listPythonPackagesQuery = `
+EXEC sp_execute_external_script
+@language=N'Python',
+@script=N'import pkg_resources
+import pandas
+OutputDataSet = pandas.DataFrame([(d.project_name, d.version) for d in pkg_resources.working_set])'
+`;
+
+const checkMlInstalledQuery = `
+Declare @tablevar table(name NVARCHAR(MAX), min INT, max INT, config_value bit, run_value bit)
+insert into @tablevar(name, min, max, config_value, run_value) exec sp_configure
+
+Declare @external_script_enabled bit
+SELECT @external_script_enabled=config_value FROM @tablevar WHERE name = 'external scripts enabled'
+SELECT @external_script_enabled`;
+
+const checkPythonInstalledQuery = `
+
+SELECT is_installed
+FROM sys.dm_db_external_language_stats s, sys.external_languages l
+WHERE s.external_language_id = l.external_language_id AND language = 'Python'`;
+
+const modifyExternalScriptConfigQuery = `
+
+EXEC sp_configure 'external scripts enabled', #CONFIG_VALUE#;
+RECONFIGURE WITH OVERRIDE;
+
+Declare @tablevar table(name NVARCHAR(MAX), min INT, max INT, config_value bit, run_value bit)
+insert into @tablevar(name, min, max, config_value, run_value) exec sp_configure
+
+Declare @external_script_enabled bit
+SELECT @external_script_enabled=config_value FROM @tablevar WHERE name = 'external scripts enabled'
+SELECT @external_script_enabled`;
+
+/**
+ * SQL Query runner
+ */
+export class QueryRunner {
+
+	constructor(private _apiWrapper: ApiWrapper) {
+	}
+
+	/**
+	 * Returns python packages installed in SQL server instance
+	 * @param connection SQL Connection
+	 */
+	public async getPythonPackages(connection: azdata.connection.ConnectionProfile): Promise<nbExtensionApis.IPackageDetails[]> {
+		let packages: nbExtensionApis.IPackageDetails[] = [];
+		let result = await this.runQuery(connection, listPythonPackagesQuery);
+		if (result && result.rows.length > 0) {
+			packages = result.rows.map(row => {
+				return {
+					name: row[0].displayValue,
+					version: row[1].displayValue
+				};
+			});
+		}
+		return packages;
+	}
+
+	/**
+	 * Updates External Script Config in a SQL server instance
+	 * @param connection SQL Connection
+	 * @param enable if true the config will be enabled otherwise it will be disabled
+	 */
+	public async updateExternalScriptConfig(connection: azdata.connection.ConnectionProfile, enable: boolean): Promise<void> {
+		let query = modifyExternalScriptConfigQuery;
+		let configValue = enable ? '1' : '0';
+		query = query.replace('#CONFIG_VALUE#', configValue);
+
+		await this.runQuery(connection, query);
+	}
+
+	/**
+	 * Returns true if python installed in the give SQL server instance
+	 */
+	public async isPythonInstalled(connection: azdata.connection.ConnectionProfile): Promise<boolean> {
+		let result = await this.runQuery(connection, checkPythonInstalledQuery);
+		let isInstalled = false;
+		if (result && result.rows && result.rows.length > 0) {
+			isInstalled = result.rows[0][0].displayValue === '1';
+		}
+		return isInstalled;
+	}
+
+	/**
+	 * Returns true if mls is installed in the give SQL server instance
+	 */
+	public async isMachineLearningServiceEnabled(connection: azdata.connection.ConnectionProfile): Promise<boolean> {
+		let result = await this.runQuery(connection, checkMlInstalledQuery);
+		let isEnabled = false;
+		if (result && result.rows && result.rows.length > 0) {
+			isEnabled = result.rows[0][0].displayValue === '1';
+		}
+		return isEnabled;
+	}
+
+	private async runQuery(connection: azdata.connection.ConnectionProfile, query: string): Promise<azdata.SimpleExecuteResult | undefined> {
+		let result: azdata.SimpleExecuteResult | undefined = undefined;
+		try {
+			if (connection) {
+				let connectionUri = await this._apiWrapper.getUriForConnection(connection.connectionId);
+				let queryProvider = this._apiWrapper.getProvider<azdata.QueryProvider>(connection.providerId, azdata.DataProviderType.QueryProvider);
+				if (queryProvider) {
+					result = await queryProvider.runQueryAndReturn(connectionUri, query);
+				}
+			}
+		} catch (error) {
+			console.log(error);
+		}
+		return result;
+	}
+}

extensions/machine-learning-services/src/common/utils.ts

@@ -0,0 +1,48 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as uuid from 'uuid';
+import * as path from 'path';
+import * as os from 'os';
+import * as fs from 'fs';
+import * as constants from '../common/constants';
+import { promisify } from 'util';
+
+export async function execCommandOnTempFile<T>(content: string, command: (filePath: string) => Promise<T>): Promise<T> {
+	let tempFilePath: string = '';
+	try {
+		tempFilePath = path.join(os.tmpdir(), `ads_ml_temp_${uuid.v4()}`);
+		await fs.promises.writeFile(tempFilePath, content);
+		let result = await command(tempFilePath);
+		return result;
+	}
+	finally {
+		await fs.promises.unlink(tempFilePath);
+	}
+}
+
+export async function exists(path: string): Promise<boolean> {
+	return promisify(fs.exists)(path);
+}
+
+export async function createFolder(dirPath: string): Promise<void> {
+	let folderExists = await exists(dirPath);
+	if (!folderExists) {
+		await fs.promises.mkdir(dirPath);
+	}
+}
+
+export function getPythonInstallationLocation(rootFolder: string) {
+	return path.join(rootFolder, 'python');
+}
+
+export function getPythonExePath(rootFolder: string): string {
+	return path.join(
+		getPythonInstallationLocation(rootFolder),
+		constants.pythonBundleVersion,
+		process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
+}