Machine Learning Services R Packages (#8870)

* R Package management in Machine learning services extension

extensions/machine-learning-services/src/common/apiWrapper.ts

@@ -69,4 +69,8 @@ export class ApiWrapper {
 	public getExtension(extensionId: string): vscode.Extension<any> | undefined {
 		return vscode.extensions.getExtension(extensionId);
 	}
+
+	public getConfiguration(section?: string, resource?: vscode.Uri | null): vscode.WorkspaceConfiguration {
+		return vscode.workspace.getConfiguration(section, resource);
+	}
 }

extensions/machine-learning-services/src/common/config.ts

@@ -1,36 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { promises as fs } from 'fs';
-import * as path from 'path';
-import * as nbExtensionApis from '../typings/notebookServices';
-
-const configFileName = 'config.json';
-
-/**
- * Extension Configuration
- */
-export class Config {
-
-	private _configValues: any;
-
-	constructor(private _root: string) {
-	}
-
-	/**
-	 * Loads the config values
-	 */
-	public async load(): Promise<void> {
-		const rawConfig = await fs.readFile(path.join(this._root, configFileName));
-		this._configValues = JSON.parse(rawConfig.toString());
-	}
-
-	/**
-	 * Returns the config value of required packages
-	 */
-	public get requiredPythonPackages(): nbExtensionApis.IPackageDetails[] {
-		return this._configValues.requiredPythonPackages;
-	}
-}

extensions/machine-learning-services/src/common/constants.ts

@@ -14,6 +14,7 @@ export const pythonBundleVersion = '0.0.1';
 export const managePackagesCommand = 'jupyter.cmd.managePackages';
 export const pythonLanguageName = 'Python';
 export const rLanguageName = 'R';
+export const rLPackagedFolderName = 'r_packages';
 
 export const mlEnableMlsCommand = 'mls.command.enableMls';
 export const mlDisableMlsCommand = 'mls.command.disableMls';
@@ -25,6 +26,13 @@ export const notebookExtensionName = 'Microsoft.notebook';
 export const mlManagePackagesCommand = 'mls.command.managePackages';
 export const mlOdbcDriverCommand = 'mls.command.odbcdriver';
 export const mlsDocumentsCommand = 'mls.command.mlsdocs';
+export const mlsDependenciesCommand = 'mls.command.dependencies';
+
+// Configurations
+//
+export const mlsConfigKey = 'machineLearningServices';
+export const pythonPathConfigKey = 'pythonPath';
+export const rPathConfigKey = 'rPath';
 
 // Localized texts
 //
@@ -47,6 +55,18 @@ export const mlsConfigAction = localize('mls.configAction', "Action");
 export const mlsExternalExecuteScriptTitle = localize('mls.externalExecuteScriptTitle', "External Execute Script");
 export const mlsPythonLanguageTitle = localize('mls.pythonLanguageTitle', "Python");
 export const mlsRLanguageTitle = localize('mls.rLanguageTitle', "R");
+export const downloadError = localize('mls.downloadError', "Error while downloading");
+export const downloadingProgress = localize('mls.downloadingProgress', "Downloading");
+export const pythonConfigError = localize('mls.pythonConfigError', "Python executable is not configured");
+export const rConfigError = localize('mls.rConfigError', "R executable is not configured");
+export const installingDependencies = localize('mls.installingDependencies', "Installing dependencies ...");
+export const resourceNotFoundError = localize('mls.resourceNotFound', "Could not find the specified resource");
+export function httpGetRequestError(code: number, message: string): string {
+	return localize('mls.httpGetRequestError', "Package info request failed with error: {0} {1}",
+		code,
+		message);
+}
+
 
 // Links
 //

extensions/machine-learning-services/src/common/httpClient.ts

@@ -0,0 +1,82 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as azdata from 'azdata';
+import * as vscode from 'vscode';
+import * as fs from 'fs';
+import * as request from 'request';
+import * as constants from './constants';
+
+const DownloadTimeout = 20000;
+const GetTimeout = 10000;
+export class HttpClient {
+
+	public async fetch(url: string): Promise<any> {
+		return new Promise<any>((resolve, reject) => {
+			request.get(url, { timeout: GetTimeout }, (error, response, body) => {
+				if (error) {
+					return reject(error);
+				}
+
+				if (response.statusCode === 404) {
+					return reject(constants.resourceNotFoundError);
+				}
+
+				if (response.statusCode !== 200) {
+					return reject(
+						constants.httpGetRequestError(
+							response.statusCode,
+							response.statusMessage));
+				}
+
+				resolve(body);
+			});
+		});
+	}
+
+	public download(downloadUrl: string, targetPath: string, backgroundOperation: azdata.BackgroundOperation, outputChannel: vscode.OutputChannel): Promise<void> {
+		return new Promise((resolve, reject) => {
+
+			let totalMegaBytes: number | undefined = undefined;
+			let receivedBytes = 0;
+			let printThreshold = 0.1;
+			let downloadRequest = request.get(downloadUrl, { timeout: DownloadTimeout })
+				.on('error', downloadError => {
+					backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, constants.downloadError);
+					reject(downloadError);
+				})
+				.on('response', (response) => {
+					if (response.statusCode !== 200) {
+						backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, constants.downloadError);
+						return reject(response.statusMessage);
+					}
+					let contentLength = response.headers['content-length'];
+					let totalBytes = parseInt(contentLength || '0');
+					totalMegaBytes = totalBytes / (1024 * 1024);
+					outputChannel.appendLine(`'Downloading' (0 / ${totalMegaBytes.toFixed(2)} MB)`);
+				})
+				.on('data', (data) => {
+					receivedBytes += data.length;
+					if (totalMegaBytes) {
+						let receivedMegaBytes = receivedBytes / (1024 * 1024);
+						let percentage = receivedMegaBytes / totalMegaBytes;
+						if (percentage >= printThreshold) {
+							outputChannel.appendLine(`${constants.downloadingProgress} (${receivedMegaBytes.toFixed(2)} / ${totalMegaBytes.toFixed(2)} MB)`);
+							printThreshold += 0.1;
+						}
+					}
+				});
+			downloadRequest.pipe(fs.createWriteStream(targetPath))
+				.on('close', async () => {
+					resolve();
+				})
+				.on('error', (downloadError) => {
+					backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, 'Error');
+					reject(downloadError);
+					downloadRequest.abort();
+				});
+		});
+	}
+}

extensions/machine-learning-services/src/common/processService.ts

@@ -13,12 +13,12 @@ export class ProcessService {
 
 	public timeout = ExecScriptsTimeoutInSeconds;
 
-	public async execScripts(exeFilePath: string, scripts: string[], outputChannel?: vscode.OutputChannel): Promise<void> {
-		return new Promise<void>((resolve, reject) => {
+	public async execScripts(exeFilePath: string, scripts: string[], args?: string[], outputChannel?: vscode.OutputChannel): Promise<string> {
+		return new Promise<string>((resolve, reject) => {
 
-			const scriptExecution = childProcess.spawn(exeFilePath);
+			const scriptExecution = childProcess.spawn(exeFilePath, args);
 			let timer: NodeJS.Timeout;
-			let output: string;
+			let output: string = '';
 			scripts.forEach(script => {
 				scriptExecution.stdin.write(`${script}\n`);
 			});
@@ -41,7 +41,7 @@ export class ProcessService {
 					clearTimeout(timer);
 				}
 				if (code === 0) {
-					resolve();
+					resolve(output);
 				} else {
 					reject(`Process exited with code: ${code}. output: ${output}`);
 				}

extensions/machine-learning-services/src/common/queryRunner.ts

@@ -10,6 +10,8 @@ import * as nbExtensionApis from '../typings/notebookServices';
 import { ApiWrapper } from './apiWrapper';
 import * as constants from '../common/constants';
 
+const maxNumberOfRetries = 3;
+
 const listPythonPackagesQuery = `
 EXEC sp_execute_external_script
 @language=N'Python',
@@ -18,6 +20,20 @@ import pandas
 OutputDataSet = pandas.DataFrame([(d.project_name, d.version) for d in pkg_resources.working_set])'
 `;
 
+const listRPackagesQuery = `
+EXEC sp_execute_external_script
+@language=N'R',
+@script=N'
+OutputDataSet <- as.data.frame(installed.packages()[,c(1,3)])'
+`;
+
+const listRAvailablePackagesQuery = `
+EXEC sp_execute_external_script
+@language=N'R',
+@script=N'
+OutputDataSet <- as.data.frame(installed.packages()[,c(1,3)])'
+`;
+
 const checkMlInstalledQuery = `
 Declare @tablevar table(name NVARCHAR(MAX), min INT, max INT, config_value bit, run_value bit)
 insert into @tablevar(name, min, max, config_value, run_value) exec sp_configure
@@ -57,8 +73,36 @@ export class QueryRunner {
 	 * @param connection SQL Connection
 	 */
 	public async getPythonPackages(connection: azdata.connection.ConnectionProfile): Promise<nbExtensionApis.IPackageDetails[]> {
+		return this.getPackages(connection, listPythonPackagesQuery);
+	}
+
+	/**
+	 * Returns python packages installed in SQL server instance
+	 * @param connection SQL Connection
+	 */
+	public async getRPackages(connection: azdata.connection.ConnectionProfile): Promise<nbExtensionApis.IPackageDetails[]> {
+		return this.getPackages(connection, listRPackagesQuery);
+	}
+
+	/**
+ * Returns python packages installed in SQL server instance
+ * @param connection SQL Connection
+ */
+	public async getRAvailablePackages(connection: azdata.connection.ConnectionProfile): Promise<nbExtensionApis.IPackageDetails[]> {
+		return this.getPackages(connection, listRAvailablePackagesQuery);
+	}
+
+	private async getPackages(connection: azdata.connection.ConnectionProfile, script: string): Promise<nbExtensionApis.IPackageDetails[]> {
 		let packages: nbExtensionApis.IPackageDetails[] = [];
-		let result = await this.runQuery(connection, listPythonPackagesQuery);
+		let result: azdata.SimpleExecuteResult | undefined = undefined;
+
+		for (let index = 0; index < maxNumberOfRetries; index++) {
+			result = await this.runQuery(connection, script);
+			if (result && result.rowCount > 0) {
+				break;
+			}
+		}
+
 		if (result && result.rows.length > 0) {
 			packages = result.rows.map(row => {
 				return {

extensions/machine-learning-services/src/common/utils.ts

@@ -47,6 +47,59 @@ export function getPythonExePath(rootFolder: string): string {
 		process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
 }
 
+export function getPackageFilePath(rootFolder: string, packageName: string): string {
+	return path.join(
+		rootFolder,
+		constants.rLPackagedFolderName,
+		packageName);
+}
+
+export function getRPackagesFolderPath(rootFolder: string): string {
+	return path.join(
+		rootFolder,
+		constants.rLPackagedFolderName);
+}
+
+/**
+ * Compares two version strings to see which is greater.
+ * @param first First version string to compare.
+ * @param second Second version string to compare.
+ * @returns 1 if the first version is greater, -1 if it's less, and 0 otherwise.
+ */
+export function comparePackageVersions(first: string, second: string): number {
+	let firstVersion = first.split('.').map(numStr => Number.parseInt(numStr));
+	let secondVersion = second.split('.').map(numStr => Number.parseInt(numStr));
+
+	// If versions have different lengths, then append zeroes to the shorter one
+	if (firstVersion.length > secondVersion.length) {
+		let diff = firstVersion.length - secondVersion.length;
+		secondVersion = secondVersion.concat(new Array(diff).fill(0));
+	} else if (secondVersion.length > firstVersion.length) {
+		let diff = secondVersion.length - firstVersion.length;
+		firstVersion = firstVersion.concat(new Array(diff).fill(0));
+	}
+
+	for (let i = 0; i < firstVersion.length; ++i) {
+		if (firstVersion[i] > secondVersion[i]) {
+			return 1;
+		} else if (firstVersion[i] < secondVersion[i]) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
+export function sortPackageVersions(versions: string[], ascending: boolean = true) {
+	return versions.sort((first, second) => {
+		let compareResult = comparePackageVersions(first, second);
+		if (ascending) {
+			return compareResult;
+		} else {
+			return compareResult * -1;
+		}
+	});
+}
+
 export function isWindows(): boolean {
 	return process.platform === 'win32';
 }