ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 10:58:30 -05:00
Code Issues Packages Projects Releases Wiki Activity

Machine Learning Services R Packages (#8870)

Browse Source 
* R Package management in Machine learning services extension
This commit is contained in:
Leila Lali
2020-01-15 12:19:22 -08:00
committed by GitHub
parent d3105beb43
commit 09b578a169
29 changed files with 1330 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/machine-learning-services/.gitignore vendored
View File

@@ -1,2 +1,4 @@
*.vsix
python/**
r_packages/**
*.zip

1
extensions/machine-learning-services/.vscodeignore
View File

@@ -1,4 +1,5 @@
src/**
tsconfig.json
python/**
r_packages/**
out/test/**

6
extensions/machine-learning-services/config.json
View File

@@ -1,6 +0,0 @@
{
"requiredPythonPackages": [
{ "name": "pymssql", "version": "2.1.4" },
{ "name": "sqlmlutils", "version": ""}
]
}

22
extensions/machine-learning-services/package.json
View File

@@ -26,6 +26,22 @@
"Microsoft.notebook"
],
"contributes": {
"configuration": {
"type": "object",
"title": "%mls.configuration.title%",
"properties": {
"machineLearningServices.pythonPath": {
"type": "string",
"default": "python",
"description": "%mls.pythonPath.description%"
},
"machineLearningServices.rPath": {
"type": "string",
"default": "r",
"description": "%mls.rPath.description%"
}
}
},
"commands": [
{
"command": "mls.command.managePackages",
@@ -38,6 +54,10 @@
{
"command": "mls.command.mlsdocs",
"title": "%mls.command.mlsdocs%"
},
{
"command": "mls.command.dependencies",
"title": "%mls.command.dependencies%"
}
],
"dashboard.tabs": [
@@ -77,12 +97,14 @@
]
},
"dependencies": {
"request": "^2.88.0",
"vscode-nls": "^4.0.0",
"vscode-languageclient": "^5.3.0-next.1"
},
"devDependencies": {
"@types/mocha": "^5.2.5",
"@types/node": "^10.14.8",
"@types/request": "^2.48.1",
"mocha": "^5.2.0",
"mocha-junit-reporter": "^1.17.0",
"mocha-multi-reporters": "^1.1.7",

7
extensions/machine-learning-services/package.nls.json
View File

@@ -4,8 +4,11 @@
"title.tasks": "Tasks",
"title.configurations": "Configurations",
"title.endpoints": "Endpoints",
"title.books": "Machine Learning Services Books",
"mls.command.managePackages": "Manage Packages in SQL Server",
"mls.command.odbcdriver": "Install ODBC Driver for SQL Server",
"mls.command.mlsdocs": "Machine Learning Services Documentation"
"mls.command.mlsdocs": "Machine Learning Services Documentation",
"mls.configuration.title": "Machine Learning Services configurations",
"mls.pythonPath.description": "Local path to a preexisting python installation used by Machine Learning Services.",
"mls.rPath.description": "Local path to a preexisting python installation used by Machine Learning Services.",
"mls.command.dependencies": "Install Machine Learning Services Dependencies"
}

4
extensions/machine-learning-services/src/common/apiWrapper.ts
View File

@@ -69,4 +69,8 @@ export class ApiWrapper {
public getExtension(extensionId: string): vscode.Extension<any> | undefined {
return vscode.extensions.getExtension(extensionId);
}
public getConfiguration(section?: string, resource?: vscode.Uri | null): vscode.WorkspaceConfiguration {
return vscode.workspace.getConfiguration(section, resource);
}
}

36
extensions/machine-learning-services/src/common/config.ts
View File

@@ -1,36 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { promises as fs } from 'fs';
import * as path from 'path';
import * as nbExtensionApis from '../typings/notebookServices';
const configFileName = 'config.json';
/**
* Extension Configuration
*/
export class Config {
private _configValues: any;
constructor(private _root: string) {
}
/**
* Loads the config values
*/
public async load(): Promise<void> {
const rawConfig = await fs.readFile(path.join(this._root, configFileName));
this._configValues = JSON.parse(rawConfig.toString());
}
/**
* Returns the config value of required packages
*/
public get requiredPythonPackages(): nbExtensionApis.IPackageDetails[] {
return this._configValues.requiredPythonPackages;
}
}

20
extensions/machine-learning-services/src/common/constants.ts
View File

@@ -14,6 +14,7 @@ export const pythonBundleVersion = '0.0.1';
export const managePackagesCommand = 'jupyter.cmd.managePackages';
export const pythonLanguageName = 'Python';
export const rLanguageName = 'R';
export const rLPackagedFolderName = 'r_packages';
export const mlEnableMlsCommand = 'mls.command.enableMls';
export const mlDisableMlsCommand = 'mls.command.disableMls';
@@ -25,6 +26,13 @@ export const notebookExtensionName = 'Microsoft.notebook';
export const mlManagePackagesCommand = 'mls.command.managePackages';
export const mlOdbcDriverCommand = 'mls.command.odbcdriver';
export const mlsDocumentsCommand = 'mls.command.mlsdocs';
export const mlsDependenciesCommand = 'mls.command.dependencies';
// Configurations
//
export const mlsConfigKey = 'machineLearningServices';
export const pythonPathConfigKey = 'pythonPath';
export const rPathConfigKey = 'rPath';
// Localized texts
//
@@ -47,6 +55,18 @@ export const mlsConfigAction = localize('mls.configAction', "Action");
export const mlsExternalExecuteScriptTitle = localize('mls.externalExecuteScriptTitle', "External Execute Script");
export const mlsPythonLanguageTitle = localize('mls.pythonLanguageTitle', "Python");
export const mlsRLanguageTitle = localize('mls.rLanguageTitle', "R");
export const downloadError = localize('mls.downloadError', "Error while downloading");
export const downloadingProgress = localize('mls.downloadingProgress', "Downloading");
export const pythonConfigError = localize('mls.pythonConfigError', "Python executable is not configured");
export const rConfigError = localize('mls.rConfigError', "R executable is not configured");
export const installingDependencies = localize('mls.installingDependencies', "Installing dependencies ...");
export const resourceNotFoundError = localize('mls.resourceNotFound', "Could not find the specified resource");
export function httpGetRequestError(code: number, message: string): string {
return localize('mls.httpGetRequestError', "Package info request failed with error: {0} {1}",
code,
message);
}
// Links
//

82
extensions/machine-learning-services/src/common/httpClient.ts Normal file
View File

@@ -0,0 +1,82 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as azdata from 'azdata';
import * as vscode from 'vscode';
import * as fs from 'fs';
import * as request from 'request';
import * as constants from './constants';
const DownloadTimeout = 20000;
const GetTimeout = 10000;
export class HttpClient {
public async fetch(url: string): Promise<any> {
return new Promise<any>((resolve, reject) => {
request.get(url, { timeout: GetTimeout }, (error, response, body) => {
if (error) {
return reject(error);
}
if (response.statusCode === 404) {
return reject(constants.resourceNotFoundError);
}
if (response.statusCode !== 200) {
return reject(
constants.httpGetRequestError(
response.statusCode,
response.statusMessage));
}
resolve(body);
});
});
}
public download(downloadUrl: string, targetPath: string, backgroundOperation: azdata.BackgroundOperation, outputChannel: vscode.OutputChannel): Promise<void> {
return new Promise((resolve, reject) => {
let totalMegaBytes: number | undefined = undefined;
let receivedBytes = 0;
let printThreshold = 0.1;
let downloadRequest = request.get(downloadUrl, { timeout: DownloadTimeout })
.on('error', downloadError => {
backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, constants.downloadError);
reject(downloadError);
})
.on('response', (response) => {
if (response.statusCode !== 200) {
backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, constants.downloadError);
return reject(response.statusMessage);
}
let contentLength = response.headers['content-length'];
let totalBytes = parseInt(contentLength || '0');
totalMegaBytes = totalBytes / (1024 * 1024);
outputChannel.appendLine(`'Downloading' (0 / ${totalMegaBytes.toFixed(2)} MB)`);
})
.on('data', (data) => {
receivedBytes += data.length;
if (totalMegaBytes) {
let receivedMegaBytes = receivedBytes / (1024 * 1024);
let percentage = receivedMegaBytes / totalMegaBytes;
if (percentage >= printThreshold) {
outputChannel.appendLine(`${constants.downloadingProgress} (${receivedMegaBytes.toFixed(2)} / ${totalMegaBytes.toFixed(2)} MB)`);
printThreshold += 0.1;
}
}
});
downloadRequest.pipe(fs.createWriteStream(targetPath))
.on('close', async () => {
resolve();
})
.on('error', (downloadError) => {
backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, 'Error');
reject(downloadError);
downloadRequest.abort();
});
});
}
}

10
extensions/machine-learning-services/src/common/processService.ts
View File

@@ -13,12 +13,12 @@ export class ProcessService {
public timeout = ExecScriptsTimeoutInSeconds;
public async execScripts(exeFilePath: string, scripts: string[], outputChannel?: vscode.OutputChannel): Promise<void> {
return new Promise<void>((resolve, reject) => {
public async execScripts(exeFilePath: string, scripts: string[], args?: string[], outputChannel?: vscode.OutputChannel): Promise<string> {
return new Promise<string>((resolve, reject) => {
const scriptExecution = childProcess.spawn(exeFilePath);
const scriptExecution = childProcess.spawn(exeFilePath, args);
let timer: NodeJS.Timeout;
let output: string;
let output: string = '';
scripts.forEach(script => {
scriptExecution.stdin.write(`${script}\n`);
});
@@ -41,7 +41,7 @@ export class ProcessService {
clearTimeout(timer);
}
if (code === 0) {
resolve();
resolve(output);
} else {
reject(`Process exited with code: ${code}. output: ${output}`);
}

46
extensions/machine-learning-services/src/common/queryRunner.ts
View File

@@ -10,6 +10,8 @@ import * as nbExtensionApis from '../typings/notebookServices';
import { ApiWrapper } from './apiWrapper';
import * as constants from '../common/constants';
const maxNumberOfRetries = 3;
const listPythonPackagesQuery = `
EXEC sp_execute_external_script
@language=N'Python',
@@ -18,6 +20,20 @@ import pandas
OutputDataSet = pandas.DataFrame([(d.project_name, d.version) for d in pkg_resources.working_set])'
`;
const listRPackagesQuery = `
EXEC sp_execute_external_script
@language=N'R',
@script=N'
OutputDataSet <- as.data.frame(installed.packages()[,c(1,3)])'
`;
const listRAvailablePackagesQuery = `
EXEC sp_execute_external_script
@language=N'R',
@script=N'
OutputDataSet <- as.data.frame(installed.packages()[,c(1,3)])'
`;
const checkMlInstalledQuery = `
Declare @tablevar table(name NVARCHAR(MAX), min INT, max INT, config_value bit, run_value bit)
insert into @tablevar(name, min, max, config_value, run_value) exec sp_configure
@@ -57,8 +73,36 @@ export class QueryRunner {
* @param connection SQL Connection
*/
public async getPythonPackages(connection: azdata.connection.ConnectionProfile): Promise<nbExtensionApis.IPackageDetails[]> {
return this.getPackages(connection, listPythonPackagesQuery);
}
/**
* Returns python packages installed in SQL server instance
* @param connection SQL Connection
*/
public async getRPackages(connection: azdata.connection.ConnectionProfile): Promise<nbExtensionApis.IPackageDetails[]> {
return this.getPackages(connection, listRPackagesQuery);
}
/**
* Returns python packages installed in SQL server instance
* @param connection SQL Connection
*/
public async getRAvailablePackages(connection: azdata.connection.ConnectionProfile): Promise<nbExtensionApis.IPackageDetails[]> {
return this.getPackages(connection, listRAvailablePackagesQuery);
}
private async getPackages(connection: azdata.connection.ConnectionProfile, script: string): Promise<nbExtensionApis.IPackageDetails[]> {
let packages: nbExtensionApis.IPackageDetails[] = [];
let result = await this.runQuery(connection, listPythonPackagesQuery);
let result: azdata.SimpleExecuteResult | undefined = undefined;
for (let index = 0; index < maxNumberOfRetries; index++) {
result = await this.runQuery(connection, script);
if (result && result.rowCount > 0) {
break;
}
}
if (result && result.rows.length > 0) {
packages = result.rows.map(row => {
return {

53
extensions/machine-learning-services/src/common/utils.ts
View File

@@ -47,6 +47,59 @@ export function getPythonExePath(rootFolder: string): string {
process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
}
export function getPackageFilePath(rootFolder: string, packageName: string): string {
return path.join(
rootFolder,
constants.rLPackagedFolderName,
packageName);
}
export function getRPackagesFolderPath(rootFolder: string): string {
return path.join(
rootFolder,
constants.rLPackagedFolderName);
}
/**
* Compares two version strings to see which is greater.
* @param first First version string to compare.
* @param second Second version string to compare.
* @returns 1 if the first version is greater, -1 if it's less, and 0 otherwise.
*/
export function comparePackageVersions(first: string, second: string): number {
let firstVersion = first.split('.').map(numStr => Number.parseInt(numStr));
let secondVersion = second.split('.').map(numStr => Number.parseInt(numStr));
// If versions have different lengths, then append zeroes to the shorter one
if (firstVersion.length > secondVersion.length) {
let diff = firstVersion.length - secondVersion.length;
secondVersion = secondVersion.concat(new Array(diff).fill(0));
} else if (secondVersion.length > firstVersion.length) {
let diff = secondVersion.length - firstVersion.length;
firstVersion = firstVersion.concat(new Array(diff).fill(0));
}
for (let i = 0; i < firstVersion.length; ++i) {
if (firstVersion[i] > secondVersion[i]) {
return 1;
} else if (firstVersion[i] < secondVersion[i]) {
return -1;
}
}
return 0;
}
export function sortPackageVersions(versions: string[], ascending: boolean = true) {
return versions.sort((first, second) => {
let compareResult = comparePackageVersions(first, second);
if (ascending) {
return compareResult;
} else {
return compareResult * -1;
}
});
}
export function isWindows(): boolean {
return process.platform === 'win32';
}

11
extensions/machine-learning-services/src/configurations/config.json Normal file
View File

@@ -0,0 +1,11 @@
{
"requiredPythonPackages": [
{ "name": "pymssql", "version": "2.1.4" },
{ "name": "sqlmlutils", "version": ""}
],
"requiredRPackages": [
{ "name": "RODBCext", "repository": "https://cran.microsoft.com" },
{ "name": "sqlmlutils", "fileName": "sqlmlutils_0.7.1.zip", "downloadUrl": "https://github.com/microsoft/sqlmlutils/blob/master/R/dist/sqlmlutils_0.7.1.zip?raw=true"}
]
}

71
extensions/machine-learning-services/src/configurations/config.ts Normal file
View File

@@ -0,0 +1,71 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import { ApiWrapper } from '../common/apiWrapper';
import * as constants from '../common/constants';
import { promises as fs } from 'fs';
import * as path from 'path';
import { PackageConfigModel } from './packageConfigModel';
const configFileName = 'config.json';
const defaultPythonExecutable = 'python';
const defaultRExecutable = 'r';
/**
* Extension Configuration from app settings
*/
export class Config {
private _configValues: any;
private _mlsConfig: vscode.WorkspaceConfiguration | undefined;
constructor(private _root: string, private _apiWrapper: ApiWrapper) {
}
/**
* Loads the config values
*/
public async load(): Promise<void> {
const rawConfig = await fs.readFile(path.join(this._root, 'src', 'configurations', configFileName));
this._configValues = JSON.parse(rawConfig.toString());
}
/**
* Returns the config value of required python packages
*/
public get requiredPythonPackages(): PackageConfigModel[] {
return this._configValues.requiredPythonPackages;
}
/**
* Returns the config value of required r packages
*/
public get requiredRPackages(): PackageConfigModel[] {
return this._configValues.requiredRPackages;
}
/**
* Returns python path from user settings
*/
public get pythonExecutable(): string {
return this.config.get(constants.pythonPathConfigKey) || defaultPythonExecutable;
}
/**
* Returns r path from user settings
*/
public get rExecutable(): string {
return this.config.get(constants.rPathConfigKey) || defaultRExecutable;
}
private get config(): vscode.WorkspaceConfiguration {
if (!this._mlsConfig) {
this._mlsConfig = this._apiWrapper.getConfiguration(constants.mlsConfigKey);
}
return this._mlsConfig;
}
}

35
extensions/machine-learning-services/src/configurations/packageConfigModel.ts Normal file
View File

@@ -0,0 +1,35 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
/**
* The model for package config value
*/
export interface PackageConfigModel {
/**
* Package name
*/
name: string;
/**
* Package version
*/
version?: string;
/**
* Package repository
*/
repository?: string;
/**
* Package download url
*/
downloadUrl?: string;
/**
* Package file name if package has download url
*/
fileName?: string;
}

37
extensions/machine-learning-services/src/controllers/mainController.ts
View File

@@ -13,9 +13,10 @@ import * as constants from '../common/constants';
import { ApiWrapper } from '../common/apiWrapper';
import { QueryRunner } from '../common/queryRunner';
import { ProcessService } from '../common/processService';
import { Config } from '../common/config';
import { Config } from '../configurations/config';
import { ServerConfigWidget } from '../widgets/serverConfigWidgets';
import { ServerConfigManager } from '../serverConfig/serverConfigManager';
import { HttpClient } from '../common/httpClient';
/**
* The main controller class that initializes the extension
@@ -31,11 +32,12 @@ export default class MainController implements vscode.Disposable {
private _queryRunner: QueryRunner,
private _processService: ProcessService,
private _packageManager?: PackageManager,
private _serverConfigManager?: ServerConfigManager
private _serverConfigManager?: ServerConfigManager,
private _httpClient?: HttpClient
) {
this._outputChannel = this._apiWrapper.createOutputChannel(constants.extensionOutputChannel);
this._rootPath = this._context.extensionPath;
this._config = new Config(this._rootPath);
this._config = new Config(this._rootPath, this._apiWrapper);
}
/**
@@ -78,6 +80,9 @@ export default class MainController implements vscode.Disposable {
this._apiWrapper.registerCommand(constants.mlManagePackagesCommand, (async () => {
await packageManager.managePackages();
}));
this._apiWrapper.registerCommand(constants.mlsDependenciesCommand, (async () => {
await packageManager.installDependencies();
}));
this._apiWrapper.registerTaskHandler(constants.mlManagePackagesCommand, async () => {
await packageManager.managePackages();
});
@@ -87,12 +92,6 @@ export default class MainController implements vscode.Disposable {
this._apiWrapper.registerTaskHandler(constants.mlsDocumentsCommand, async () => {
await this.serverConfigManager.openDocuments();
});
try {
await packageManager.installDependencies();
} catch (err) {
this._outputChannel.appendLine(err);
}
}
/**
@@ -100,15 +99,18 @@ export default class MainController implements vscode.Disposable {
*/
public getPackageManager(nbApis: nbExtensionApis.IExtensionApi): PackageManager {
if (!this._packageManager) {
this._packageManager = new PackageManager(nbApis, this._outputChannel, this._rootPath, this._apiWrapper, this._queryRunner, this._processService, this._config);
this._packageManager = new PackageManager(this._outputChannel, this._rootPath, this._apiWrapper, this._queryRunner, this._processService, this._config, this.httpClient);
this._packageManager.init();
this._packageManager.packageManageProviders.forEach(provider => {
nbApis.registerPackageManager(provider.providerId, provider);
});
}
return this._packageManager;
}
/**
* Returns the server config manager instance
*/
* Returns the server config manager instance
*/
public get serverConfigManager(): ServerConfigManager {
if (!this._serverConfigManager) {
this._serverConfigManager = new ServerConfigManager(this._apiWrapper, this._queryRunner);
@@ -116,6 +118,17 @@ export default class MainController implements vscode.Disposable {
return this._serverConfigManager;
}
/**
* Returns the server config manager instance
*/
public get httpClient(): HttpClient {
if (!this._httpClient) {
this._httpClient = new HttpClient();
}
return this._httpClient;
}
/**
* Config instance
*/

115
extensions/machine-learning-services/src/packageManagement/SqlPackageManageProviderBase.ts Normal file
View File

@@ -0,0 +1,115 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as azdata from 'azdata';
import { ApiWrapper } from '../common/apiWrapper';
import * as constants from '../common/constants';
import * as nbExtensionApis from '../typings/notebookServices';
import * as utils from '../common/utils';
export enum ScriptMode {
Install = 'install',
Uninstall = 'uninstall'
}
export abstract class SqlPackageManageProviderBase {
/**
* Base class for all SQL package managers
*/
constructor(protected _apiWrapper: ApiWrapper) {
}
/**
* Returns location title
*/
public async getLocationTitle(): Promise<string> {
let connection = await this.getCurrentConnection();
if (connection) {
return `${connection.serverName} ${connection.databaseName ? connection.databaseName : ''}`;
}
return constants.packageManagerNoConnection;
}
protected async getCurrentConnection(): Promise<azdata.connection.ConnectionProfile> {
return await this._apiWrapper.getCurrentConnection();
}
/**
* Installs given packages
* @param packages Packages to install
* @param useMinVersion minimum version
*/
public async installPackages(packages: nbExtensionApis.IPackageDetails[], useMinVersion: boolean): Promise<void> {
if (packages) {
await Promise.all(packages.map(x => this.installPackage(x, useMinVersion)));
}
//TODO: use useMinVersion
console.log(useMinVersion);
}
private async installPackage(packageDetail: nbExtensionApis.IPackageDetails, useMinVersion: boolean): Promise<void> {
if (useMinVersion) {
let packageOverview = await this.getPackageOverview(packageDetail.name);
if (packageOverview && packageOverview.versions) {
let minVersion = packageOverview.versions[packageOverview.versions.length - 1];
packageDetail.version = minVersion;
}
}
await this.executeScripts(ScriptMode.Install, packageDetail);
}
/**
* Uninstalls given packages
* @param packages Packages to uninstall
*/
public async uninstallPackages(packages: nbExtensionApis.IPackageDetails[]): Promise<void> {
if (packages) {
await Promise.all(packages.map(x => this.executeScripts(ScriptMode.Uninstall, x)));
}
}
/**
* Returns package overview for given name
* @param packageName Package Name
*/
public async getPackageOverview(packageName: string): Promise<nbExtensionApis.IPackageOverview> {
let packageOverview = await this.fetchPackage(packageName);
if (packageOverview && packageOverview.versions) {
packageOverview.versions = utils.sortPackageVersions(packageOverview.versions, false);
}
return packageOverview;
}
/**
* Returns list of packages
*/
public async listPackages(): Promise<nbExtensionApis.IPackageDetails[]> {
let packages = await this.fetchPackages();
if (packages) {
packages = packages.sort((a, b) => this.comparePackages(a, b));
} else {
packages = [];
}
return packages;
}
private comparePackages(p1: nbExtensionApis.IPackageDetails, p2: nbExtensionApis.IPackageDetails): number {
if (p1 && p2) {
let compare = p1.name.localeCompare(p2.name);
if (compare === 0) {
compare = utils.comparePackageVersions(p1.version, p2.version);
}
return compare;
}
return p1 ? 1 : -1;
}
protected abstract fetchPackage(packageName: string): Promise<nbExtensionApis.IPackageOverview>;
protected abstract fetchPackages(): Promise<nbExtensionApis.IPackageDetails[]>;
protected abstract executeScripts(scriptMode: ScriptMode, packageDetails: nbExtensionApis.IPackageDetails): Promise<void>;
}

128
extensions/machine-learning-services/src/packageManagement/packageManager.ts
View File

@@ -8,61 +8,97 @@
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as nbExtensionApis from '../typings/notebookServices';
import { SqlPythonPackageManageProvider } from './sqlPackageManageProvider';
import { SqlPythonPackageManageProvider } from './sqlPythonPackageManageProvider';
import { QueryRunner } from '../common/queryRunner';
import * as utils from '../common/utils';
import * as constants from '../common/constants';
import { ApiWrapper } from '../common/apiWrapper';
import { ProcessService } from '../common/processService';
import { Config } from '../common/config';
import { Config } from '../configurations/config';
import { isNullOrUndefined } from 'util';
import { SqlRPackageManageProvider } from './sqlRPackageManageProvider';
import { HttpClient } from '../common/httpClient';
import { PackageConfigModel } from '../configurations/packageConfigModel';
export class PackageManager {
private _pythonExecutable: string = '';
private _pythonInstallationLocation: string = '';
private _sqlPackageManager: SqlPythonPackageManageProvider | undefined = undefined;
private _rExecutable: string = '';
private _sqlPythonPackagePackageManager: SqlPythonPackageManageProvider;
private _sqlRPackageManager: SqlRPackageManageProvider;
public dependenciesInstalled: boolean = false;
/**
* Creates a new instance of PackageManager
*/
constructor(
private _nbExtensionApis: nbExtensionApis.IExtensionApi,
private _outputChannel: vscode.OutputChannel,
private _rootFolder: string,
private _apiWrapper: ApiWrapper,
private _queryRunner: QueryRunner,
private _processService: ProcessService,
private _config: Config) {
private _config: Config,
private _httpClient: HttpClient) {
this._sqlPythonPackagePackageManager = new SqlPythonPackageManageProvider(this._outputChannel, this._apiWrapper, this._queryRunner, this._processService, this._config, this._httpClient);
this._sqlRPackageManager = new SqlRPackageManageProvider(this._outputChannel, this._apiWrapper, this._queryRunner, this._processService, this._config);
}
/**
* Initializes the instance and resister SQL package manager with manage package dialog
*/
public init(): void {
this._pythonInstallationLocation = utils.getPythonInstallationLocation(this._rootFolder);
this._pythonExecutable = utils.getPythonExePath(this._rootFolder);
this._sqlPackageManager = new SqlPythonPackageManageProvider(this._nbExtensionApis, this._outputChannel, this._rootFolder, this._apiWrapper, this._queryRunner, this._processService);
this._nbExtensionApis.registerPackageManager(SqlPythonPackageManageProvider.ProviderId, this._sqlPackageManager);
this._pythonExecutable = this._config.pythonExecutable;
this._rExecutable = this._config.rExecutable;
}
/**
* Returns packageManageProviders
*/
public get packageManageProviders(): nbExtensionApis.IPackageManageProvider[] {
return [
this._sqlPythonPackagePackageManager,
this._sqlRPackageManager
];
}
/**
* Executes manage package command for SQL server packages.
*/
public async managePackages(): Promise<void> {
try {
// Only execute the command if there's a valid connection with ml configuration enabled
//
let connection = await this.getCurrentConnection();
let isPythonInstalled = await this._queryRunner.isPythonInstalled(connection);
let isRInstalled = await this._queryRunner.isRInstalled(connection);
let defaultProvider: SqlRPackageManageProvider | SqlPythonPackageManageProvider | undefined;
if (connection && isPythonInstalled) {
defaultProvider = this._sqlPythonPackagePackageManager;
} else if (connection && isRInstalled) {
defaultProvider = this._sqlRPackageManager;
}
if (connection && defaultProvider) {
// Only execute the command if there's a valid connection with ml configuration enabled
//
let connection = await this.getCurrentConnection();
let isPythonInstalled = await this._queryRunner.isPythonInstalled(connection);
if (connection && isPythonInstalled && this._sqlPackageManager) {
this._apiWrapper.executeCommand(constants.managePackagesCommand, {
multiLocations: false,
defaultLocation: this._sqlPackageManager.packageTarget.location,
defaultProviderId: SqlPythonPackageManageProvider.ProviderId
});
} else {
this._apiWrapper.showInfoMessage(constants.managePackageCommandError);
// Install dependencies
//
if (!this.dependenciesInstalled) {
this._apiWrapper.showInfoMessage(constants.installingDependencies);
await this.installDependencies();
this.dependenciesInstalled = true;
}
// Execute the command
//
this._apiWrapper.executeCommand(constants.managePackagesCommand, {
multiLocations: false,
defaultLocation: defaultProvider.packageTarget.location,
defaultProviderId: defaultProvider.providerId
});
} else {
this._apiWrapper.showInfoMessage(constants.managePackageCommandError);
}
} catch (err) {
this._outputChannel.appendLine(err);
}
}
@@ -78,16 +114,13 @@ export class PackageManager {
isCancelable: false,
operation: async op => {
try {
if (!(await utils.exists(this._pythonExecutable))) {
// Install python
//
await utils.createFolder(this._pythonInstallationLocation);
await this.jupyterInstallation.installPythonPackage(op, false, this._pythonInstallationLocation, this._outputChannel);
}
await utils.createFolder(utils.getRPackagesFolderPath(this._rootFolder));
// Install required packages
//
await this.installRequiredPythonPackages();
await Promise.all([
this.installRequiredPythonPackages(),
this.installRequiredRPackages(op)]);
op.updateStatus(azdata.TaskStatus.Succeeded);
resolve();
} catch (error) {
@@ -100,10 +133,21 @@ export class PackageManager {
});
}
private async installRequiredRPackages(startBackgroundOperation: azdata.BackgroundOperation): Promise<void> {
if (!this._rExecutable) {
throw new Error(constants.rConfigError);
}
await Promise.all(this._config.requiredRPackages.map(x => this.installRPackage(x, startBackgroundOperation)));
}
/**
* Installs required python packages
*/
private async installRequiredPythonPackages(): Promise<void> {
if (!this._pythonExecutable) {
throw new Error(constants.pythonConfigError);
}
let installedPackages = await this.getInstalledPipPackages();
let fileContent = '';
this._config.requiredPythonPackages.forEach(packageDetails => {
@@ -117,7 +161,7 @@ export class PackageManager {
if (fileContent) {
this._outputChannel.appendLine(constants.installDependenciesPackages);
let result = await utils.execCommandOnTempFile<string>(fileContent, async (tempFilePath) => {
return await this.installPackages(tempFilePath);
return await this.installPipPackage(tempFilePath);
});
this._outputChannel.appendLine(result);
} else {
@@ -145,12 +189,26 @@ export class PackageManager {
return await this._apiWrapper.getCurrentConnection();
}
private get jupyterInstallation(): nbExtensionApis.IJupyterServerInstallation {
return this._nbExtensionApis.getJupyterController().jupyterInstallation;
}
private async installPackages(requirementFilePath: string): Promise<string> {
private async installPipPackage(requirementFilePath: string): Promise<string> {
let cmd = `"${this._pythonExecutable}" -m pip install -r "${requirementFilePath}"`;
return await this._processService.executeBufferedCommand(cmd, this._outputChannel);
}
private async installRPackage(model: PackageConfigModel, startBackgroundOperation: azdata.BackgroundOperation): Promise<string> {
let output = '';
let cmd = '';
if (model.downloadUrl) {
const packageFile = utils.getPackageFilePath(this._rootFolder, model.fileName || model.name);
const packageExist = await utils.exists(packageFile);
if (!packageExist) {
await this._httpClient.download(model.downloadUrl, packageFile, startBackgroundOperation, this._outputChannel);
}
cmd = `"${this._rExecutable}" CMD INSTALL ${packageFile}`;
output = await this._processService.executeBufferedCommand(cmd, this._outputChannel);
} else if (model.repository) {
cmd = `"${this._rExecutable}" -e "install.packages('${model.name}', repos='${model.repository}')"`;
output = output + await this._processService.executeBufferedCommand(cmd, this._outputChannel);
}
return output;
}
}

183
extensions/machine-learning-services/src/packageManagement/sqlPackageManageProvider.ts
View File

@@ -1,183 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as nbExtensionApis from '../typings/notebookServices';
import * as utils from '../common/utils';
import * as constants from '../common/constants';
import { QueryRunner } from '../common/queryRunner';
import { ApiWrapper } from '../common/apiWrapper';
import { ProcessService } from '../common/processService';
const installMode = 'install';
const uninstallMode = 'uninstall';
const localPythonProviderId = 'localhost_Pip';
/**
* Manage Package Provider for python packages inside SQL server databases
*/
export class SqlPythonPackageManageProvider implements nbExtensionApis.IPackageManageProvider {
private _pythonExecutable: string;
public static ProviderId = 'sql_Python';
/**
* Creates new a instance
*/
constructor(
private _nbExtensionApis: nbExtensionApis.IExtensionApi,
private _outputChannel: vscode.OutputChannel,
private _rootFolder: string,
private _apiWrapper: ApiWrapper,
private _queryRunner: QueryRunner,
private _processService: ProcessService) {
this._pythonExecutable = utils.getPythonExePath(this._rootFolder);
}
/**
* Returns provider Id
*/
public get providerId(): string {
return SqlPythonPackageManageProvider.ProviderId;
}
/**
* Returns package target
*/
public get packageTarget(): nbExtensionApis.IPackageTarget {
return { location: 'SQL', packageType: 'Python' };
}
/**
* Returns list of packages
*/
public async listPackages(): Promise<nbExtensionApis.IPackageDetails[]> {
let packages = await this._queryRunner.getPythonPackages(await this.getCurrentConnection());
if (packages) {
packages = packages.sort((a, b) => a.name.localeCompare(b.name));
} else {
packages = [];
}
return packages;
}
/**
* Installs given packages
* @param packages Packages to install
* @param useMinVersion minimum version
*/
async installPackages(packages: nbExtensionApis.IPackageDetails[], useMinVersion: boolean): Promise<void> {
if (packages) {
// TODO: install package as parallel
for (let index = 0; index < packages.length; index++) {
const element = packages[index];
await this.updatePackage(element, installMode);
}
}
//TODO: use useMinVersion
console.log(useMinVersion);
}
/**
* Execute a script to install or uninstall a python package inside current SQL Server connection
* @param packageDetails Packages to install or uninstall
* @param scriptMode can be 'install' or 'uninstall'
*/
private async updatePackage(packageDetails: nbExtensionApis.IPackageDetails, scriptMode: string): Promise<void> {
let connection = await this.getCurrentConnection();
let credentials = await this._apiWrapper.getCredentials(connection.connectionId);
if (connection) {
let port = '1433';
let server = connection.serverName;
let database = connection.databaseName ? `, database="${connection.databaseName}"` : '';
let index = connection.serverName.indexOf(',');
if (index > 0) {
port = connection.serverName.substring(index + 1);
server = connection.serverName.substring(0, index);
}
let pythonConnectionParts = `server="${server}", port=${port}, uid="${connection.userName}", pwd="${credentials[azdata.ConnectionOptionSpecialType.password]}"${database})`;
let pythonCommandScript = scriptMode === installMode ?
`pkgmanager.install(package="${packageDetails.name}", version="${packageDetails.version}")` :
`pkgmanager.uninstall(package_name="${packageDetails.name}")`;
let scripts: string[] = [
'import sqlmlutils',
`connection = sqlmlutils.ConnectionInfo(driver="ODBC Driver 17 for SQL Server", ${pythonConnectionParts}`,
'pkgmanager = sqlmlutils.SQLPackageManager(connection)',
pythonCommandScript
];
await this._processService.execScripts(this._pythonExecutable, scripts, this._outputChannel);
}
}
/**
* Uninstalls given packages
* @param packages Packages to uninstall
*/
async uninstallPackages(packages: nbExtensionApis.IPackageDetails[]): Promise<void> {
for (let index = 0; index < packages.length; index++) {
const element = packages[index];
await this.updatePackage(element, uninstallMode);
}
}
/**
* Returns true if the provider can be used
*/
async canUseProvider(): Promise<boolean> {
let connection = await this.getCurrentConnection();
if (connection && await this._queryRunner.isPythonInstalled(connection)) {
return true;
}
return false;
}
/**
* Returns package overview for given name
* @param packageName Package Name
*/
async getPackageOverview(packageName: string): Promise<nbExtensionApis.IPackageOverview> {
let packagePreview: nbExtensionApis.IPackageOverview = {
name: packageName,
versions: [],
summary: ''
};
let pythonPackageProvider = this.pythonPackageProvider;
if (pythonPackageProvider) {
packagePreview = await pythonPackageProvider.getPackageOverview(packageName);
}
return packagePreview;
}
/**
* Returns location title
*/
async getLocationTitle(): Promise<string> {
let connection = await this.getCurrentConnection();
if (connection) {
return `${connection.serverName} ${connection.databaseName ? connection.databaseName : ''}`;
}
return constants.packageManagerNoConnection;
}
private get pythonPackageProvider(): nbExtensionApis.IPackageManageProvider | undefined {
let providers = this._nbExtensionApis.getPackageManagers();
if (providers && providers.has(localPythonProviderId)) {
return providers.get(localPythonProviderId);
}
return undefined;
}
private async getCurrentConnection(): Promise<azdata.connection.ConnectionProfile> {
return await this._apiWrapper.getCurrentConnection();
}
}

135
extensions/machine-learning-services/src/packageManagement/sqlPythonPackageManageProvider.ts Normal file
View File

@@ -0,0 +1,135 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as nbExtensionApis from '../typings/notebookServices';
import { QueryRunner } from '../common/queryRunner';
import { ApiWrapper } from '../common/apiWrapper';
import { ProcessService } from '../common/processService';
import { Config } from '../configurations/config';
import { SqlPackageManageProviderBase, ScriptMode } from './SqlPackageManageProviderBase';
import { HttpClient } from '../common/httpClient';
import * as utils from '../common/utils';
/**
* Manage Package Provider for python packages inside SQL server databases
*/
export class SqlPythonPackageManageProvider extends SqlPackageManageProviderBase implements nbExtensionApis.IPackageManageProvider {
public static ProviderId = 'sql_Python';
/**
* Creates new a instance
*/
constructor(
private _outputChannel: vscode.OutputChannel,
apiWrapper: ApiWrapper,
private _queryRunner: QueryRunner,
private _processService: ProcessService,
private _config: Config,
private _httpClient: HttpClient) {
super(apiWrapper);
}
/**
* Returns provider Id
*/
public get providerId(): string {
return SqlPythonPackageManageProvider.ProviderId;
}
/**
* Returns package target
*/
public get packageTarget(): nbExtensionApis.IPackageTarget {
return { location: 'SQL', packageType: 'Python' };
}
/**
* Returns list of packages
*/
protected async fetchPackages(): Promise<nbExtensionApis.IPackageDetails[]> {
return await this._queryRunner.getPythonPackages(await this.getCurrentConnection());
}
/**
* Execute a script to install or uninstall a python package inside current SQL Server connection
* @param packageDetails Packages to install or uninstall
* @param scriptMode can be 'install' or 'uninstall'
*/
protected async executeScripts(scriptMode: ScriptMode, packageDetails: nbExtensionApis.IPackageDetails): Promise<void> {
let connection = await this.getCurrentConnection();
let credentials = await this._apiWrapper.getCredentials(connection.connectionId);
if (connection) {
let port = '1433';
let server = connection.serverName;
let database = connection.databaseName ? `, database="${connection.databaseName}"` : '';
let index = connection.serverName.indexOf(',');
if (index > 0) {
port = connection.serverName.substring(index + 1);
server = connection.serverName.substring(0, index);
}
let pythonConnectionParts = `server="${server}", port=${port}, uid="${connection.userName}", pwd="${credentials[azdata.ConnectionOptionSpecialType.password]}"${database})`;
let pythonCommandScript = scriptMode === ScriptMode.Install ?
`pkgmanager.install(package="${packageDetails.name}", version="${packageDetails.version}")` :
`pkgmanager.uninstall(package_name="${packageDetails.name}")`;
let scripts: string[] = [
'import sqlmlutils',
`connection = sqlmlutils.ConnectionInfo(driver="ODBC Driver 17 for SQL Server", ${pythonConnectionParts}`,
'pkgmanager = sqlmlutils.SQLPackageManager(connection)',
pythonCommandScript
];
let pythonExecutable = this._config.pythonExecutable;
await this._processService.execScripts(pythonExecutable, scripts, [], this._outputChannel);
}
}
/**
* Returns true if the provider can be used
*/
async canUseProvider(): Promise<boolean> {
let connection = await this.getCurrentConnection();
if (connection && await this._queryRunner.isPythonInstalled(connection)) {
return true;
}
return false;
}
private getPackageLink(packageName: string): string {
return `https://pypi.org/pypi/${packageName}/json`;
}
protected async fetchPackage(packageName: string): Promise<nbExtensionApis.IPackageOverview> {
let body = await this._httpClient.fetch(this.getPackageLink(packageName));
let packagesJson = JSON.parse(body);
let versionNums: string[] = [];
let packageSummary = '';
if (packagesJson) {
if (packagesJson.releases) {
let versionKeys = Object.keys(packagesJson.releases);
versionKeys = versionKeys.filter(versionKey => {
let releaseInfo = packagesJson.releases[versionKey];
return Array.isArray(releaseInfo) && releaseInfo.length > 0;
});
versionNums = utils.sortPackageVersions(versionKeys, false);
}
if (packagesJson.info && packagesJson.info.summary) {
packageSummary = packagesJson.info.summary;
}
}
return {
name: packageName,
versions: versionNums,
summary: packageSummary
};
}
}

112
extensions/machine-learning-services/src/packageManagement/sqlRPackageManageProvider.ts Normal file
View File

@@ -0,0 +1,112 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as nbExtensionApis from '../typings/notebookServices';
import { QueryRunner } from '../common/queryRunner';
import { ApiWrapper } from '../common/apiWrapper';
import { ProcessService } from '../common/processService';
import { Config } from '../configurations/config';
import { SqlPackageManageProviderBase, ScriptMode } from './SqlPackageManageProviderBase';
/**
* Manage Package Provider for r packages inside SQL server databases
*/
export class SqlRPackageManageProvider extends SqlPackageManageProviderBase implements nbExtensionApis.IPackageManageProvider {
public static ProviderId = 'sql_R';
/**
* Creates new a instance
*/
constructor(
private _outputChannel: vscode.OutputChannel,
apiWrapper: ApiWrapper,
private _queryRunner: QueryRunner,
private _processService: ProcessService,
private _config: Config) {
super(apiWrapper);
}
/**
* Returns provider Id
*/
public get providerId(): string {
return SqlRPackageManageProvider.ProviderId;
}
/**
* Returns package target
*/
public get packageTarget(): nbExtensionApis.IPackageTarget {
return { location: 'SQL', packageType: 'R' };
}
/**
* Returns list of packages
*/
protected async fetchPackages(): Promise<nbExtensionApis.IPackageDetails[]> {
return await this._queryRunner.getRPackages(await this.getCurrentConnection());
}
/**
* Execute a script to install or uninstall a r package inside current SQL Server connection
* @param packageDetails Packages to install or uninstall
* @param scriptMode can be 'install' or 'uninstall'
*/
protected async executeScripts(scriptMode: ScriptMode, packageDetails: nbExtensionApis.IPackageDetails): Promise<void> {
let connection = await this.getCurrentConnection();
let credentials = await this._apiWrapper.getCredentials(connection.connectionId);
if (connection) {
let database = connection.databaseName ? `, database="${connection.databaseName}"` : '';
let connectionParts = `server="${connection.serverName}", uid="${connection.userName}", pwd="${credentials[azdata.ConnectionOptionSpecialType.password]}"${database}`;
let rCommandScript = scriptMode === ScriptMode.Install ? 'sql_install.packages' : 'sql_remove.packages';
let scripts: string[] = [
'formals(quit)$save <- formals(q)$save <- "no"',
'library(sqlmlutils)',
`connection <- connectionInfo(${connectionParts})`,
`pkgs <- c("${packageDetails.name}")`,
`${rCommandScript}(connectionString = connection, pkgs, scope = "PUBLIC")`,
'q()'
];
let rExecutable = this._config.rExecutable;
await this._processService.execScripts(`${rExecutable}`, scripts, ['--vanilla'], this._outputChannel);
}
}
/**
* Returns true if the provider can be used
*/
async canUseProvider(): Promise<boolean> {
let connection = await this.getCurrentConnection();
if (connection && await this._queryRunner.isRInstalled(connection)) {
return true;
}
return false;
}
/**
* Returns package overview for given name
* @param packageName Package Name
*/
protected async fetchPackage(packageName: string): Promise<nbExtensionApis.IPackageOverview> {
let packagePreview: nbExtensionApis.IPackageOverview = {
name: packageName,
versions: [],
summary: ''
};
let connection = await this.getCurrentConnection();
let availablePackages = await this._queryRunner.getRAvailablePackages(connection);
let versions = availablePackages.filter(x => x.name === packageName).map(x => x.version);
packagePreview.versions = versions;
return packagePreview;
}
}

6
extensions/machine-learning-services/src/test/common/processService.test.ts
View File

@@ -29,11 +29,11 @@ function createContext(): TestContext {
};
}
function execFolderListCommand(context: TestContext, service : ProcessService): Promise<void> {
function execFolderListCommand(context: TestContext, service : ProcessService): Promise<string> {
if (utils.isWindows()) {
return service.execScripts('cmd', ['dir', '.'], context.outputChannel);
return service.execScripts('cmd', ['dir', '.'], [], context.outputChannel);
} else {
return service.execScripts('/bin/sh', ['-c', 'ls'], context.outputChannel);
return service.execScripts('/bin/sh', ['-c', 'ls'], [], context.outputChannel);
}
}

70
extensions/machine-learning-services/src/test/mainController.test.ts
View File

@@ -15,9 +15,13 @@ import { QueryRunner } from '../common/queryRunner';
import { ProcessService } from '../common/processService';
import MainController from '../controllers/mainController';
import { PackageManager } from '../packageManagement/packageManager';
import * as nbExtensionApis from '../typings/notebookServices';
interface TestContext {
notebookExtension: vscode.Extension<any>;
jupyterInstallation: nbExtensionApis.IJupyterServerInstallation;
jupyterController: nbExtensionApis.IJupyterController;
nbExtensionApis: nbExtensionApis.IExtensionApi;
apiWrapper: TypeMoq.IMock<ApiWrapper>;
queryRunner: TypeMoq.IMock<QueryRunner>;
processService: TypeMoq.IMock<ProcessService>;
@@ -25,11 +29,49 @@ interface TestContext {
outputChannel: vscode.OutputChannel;
extension: vscode.Extension<any>;
packageManager: TypeMoq.IMock<PackageManager>;
workspaceConfig: vscode.WorkspaceConfiguration;
}
function createContext(): TestContext {
let packages = new Map<string, nbExtensionApis.IPackageManageProvider>();
let jupyterInstallation: nbExtensionApis.IJupyterServerInstallation = {
installCondaPackages: () => { return Promise.resolve(); },
getInstalledPipPackages: () => { return Promise.resolve([]); },
installPipPackages: () => { return Promise.resolve(); },
uninstallPipPackages: () => { return Promise.resolve(); },
uninstallCondaPackages: () => { return Promise.resolve(); },
executeBufferedCommand: () => { return Promise.resolve(''); },
executeStreamedCommand: () => { return Promise.resolve(); },
pythonExecutable: '',
pythonInstallationPath: '',
installPythonPackage: () => { return Promise.resolve(); }
};
let jupyterController = {
jupyterInstallation: jupyterInstallation
};
let extensionPath = path.join(__dirname, '..', '..');
let extensionApi: nbExtensionApis.IExtensionApi = {
getJupyterController: () => { return jupyterController; },
registerPackageManager: (providerId: string, packageManagerProvider: nbExtensionApis.IPackageManageProvider) => {
packages.set(providerId, packageManagerProvider);
},
getPackageManagers: () => { return packages; },
};
return {
jupyterInstallation: jupyterInstallation,
jupyterController: jupyterController,
nbExtensionApis: extensionApi,
notebookExtension: {
id: '',
extensionPath: '',
isActive: true,
packageJSON: '',
extensionKind: vscode.ExtensionKind.UI,
exports: extensionApi,
activate: () => {return Promise.resolve();}
},
apiWrapper: TypeMoq.Mock.ofType(ApiWrapper),
queryRunner: TypeMoq.Mock.ofType(QueryRunner),
processService: TypeMoq.Mock.ofType(ProcessService),
@@ -67,6 +109,12 @@ function createContext(): TestContext {
extensionKind: vscode.ExtensionKind.UI,
exports: {},
activate: () => { return Promise.resolve(); }
},
workspaceConfig: {
get: () => {return 'value';},
has: () => {return true;},
inspect: () => {return undefined;},
update: () => {return Promise.reject();},
}
};
}
@@ -85,6 +133,9 @@ describe('Main Controller', () => {
it('initialize Should install dependencies successfully', async function (): Promise<void> {
let testContext = createContext();
testContext.apiWrapper.setup(x => x.getExtension(TypeMoq.It.isAny())).returns(() => testContext.notebookExtension);
testContext.apiWrapper.setup(x => x.getConfiguration(TypeMoq.It.isAny())).returns(() => testContext.workspaceConfig);
testContext.apiWrapper.setup(x => x.createOutputChannel(TypeMoq.It.isAny())).returns(() => testContext.outputChannel);
testContext.apiWrapper.setup(x => x.getExtension(TypeMoq.It.isAny())).returns(() => testContext.extension);
testContext.packageManager.setup(x => x.managePackages()).returns(() => Promise.resolve());
@@ -92,25 +143,10 @@ describe('Main Controller', () => {
testContext.apiWrapper.setup(x => x.registerCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny()));
let controller = createController(testContext);
await controller.activate();
should.deepEqual(controller.config.requiredPythonPackages, [
{ name: 'pymssql', version: '2.1.4' },
{ name: 'sqlmlutils', version: '' }
]);
});
it('initialize Should show and error in output channel if installing dependencies fails', async function (): Promise<void> {
let errorReported = false;
let testContext = createContext();
testContext.apiWrapper.setup(x => x.createOutputChannel(TypeMoq.It.isAny())).returns(() => testContext.outputChannel);
testContext.apiWrapper.setup(x => x.getExtension(TypeMoq.It.isAny())).returns(() => testContext.extension);
testContext.packageManager.setup(x => x.managePackages()).returns(() => Promise.resolve());
testContext.packageManager.setup(x => x.installDependencies()).returns(() => Promise.reject());
testContext.apiWrapper.setup(x => x.registerCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny()));
testContext.outputChannel.appendLine = () => {
errorReported = true;
};
let controller = createController(testContext);
await controller.activate();
should.equal(errorReported, true);
});
});

72
extensions/machine-learning-services/src/test/packageManagement/packageManager.test.ts
View File

@@ -11,14 +11,12 @@ import * as should from 'should';
import 'mocha';
import * as TypeMoq from 'typemoq';
import { PackageManager } from '../../packageManagement/packageManager';
import { SqlPythonPackageManageProvider } from '../../packageManagement/sqlPackageManageProvider';
import { createContext, TestContext } from './utils';
describe('Package Manager', () => {
it('Should initialize SQL package manager successfully', async function (): Promise<void> {
let testContext = createContext();
should.doesNotThrow(() => createPackageManager(testContext));
should.equal(testContext.nbExtensionApis.getPackageManagers().has(SqlPythonPackageManageProvider.ProviderId), true);
});
it('Manage Package command Should execute the command for valid connection', async function (): Promise<void> {
@@ -32,6 +30,18 @@ describe('Package Manager', () => {
testContext.apiWrapper.verify(x => x.executeCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()), TypeMoq.Times.once());
});
it('Manage Package command Should execute the command if r installed', async function (): Promise<void> {
let testContext = createContext();
let connection = new azdata.connection.ConnectionProfile();
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => {return Promise.resolve(connection);});
testContext.apiWrapper.setup(x => x.executeCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {return Promise.resolve();});
testContext.queryRunner.setup(x => x.isPythonInstalled(connection)).returns(() => {return Promise.resolve(false);});
testContext.queryRunner.setup(x => x.isRInstalled(connection)).returns(() => {return Promise.resolve(true);});
let packageManager = createPackageManager(testContext);
await packageManager.managePackages();
testContext.apiWrapper.verify(x => x.executeCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()), TypeMoq.Times.once());
});
it('Manage Package command Should show an error for connection without python installed', async function (): Promise<void> {
let testContext = createContext();
let connection = new azdata.connection.ConnectionProfile();
@@ -39,6 +49,7 @@ describe('Package Manager', () => {
testContext.apiWrapper.setup(x => x.executeCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {return Promise.resolve();});
testContext.apiWrapper.setup(x => x.showInfoMessage(TypeMoq.It.isAny()));
testContext.queryRunner.setup(x => x.isPythonInstalled(connection)).returns(() => {return Promise.resolve(false);});
testContext.queryRunner.setup(x => x.isRInstalled(connection)).returns(() => {return Promise.resolve(false);});
let packageManager = createPackageManager(testContext);
await packageManager.managePackages();
testContext.apiWrapper.verify(x => x.showInfoMessage(TypeMoq.It.isAny()), TypeMoq.Times.once());
@@ -56,9 +67,9 @@ describe('Package Manager', () => {
testContext.apiWrapper.verify(x => x.showInfoMessage(TypeMoq.It.isAny()), TypeMoq.Times.once());
});
it('installDependencies Should install python if does not exist', async function (): Promise<void> {
it('installDependencies Should download sqlmlutils if does not exist', async function (): Promise<void> {
let testContext = createContext();
let pythonInstalled = false;
let installedPackages = `[
{"name":"pymssql","version":"2.1.4"},
{"name":"sqlmlutils","version":"1.1.1"}
@@ -66,35 +77,14 @@ describe('Package Manager', () => {
testContext.apiWrapper.setup(x => x.startBackgroundOperation(TypeMoq.It.isAny())).returns((operationInfo: azdata.BackgroundOperationInfo) => {
operationInfo.operation(testContext.op);
});
testContext.jupyterInstallation.installPythonPackage = () => {
pythonInstalled = true;
return Promise.resolve();
};
testContext.processService.setup(x => x.executeBufferedCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {return Promise.resolve(installedPackages);});
let packageManager = createPackageManager(testContext);
await packageManager.installDependencies();
should.equal(testContext.getOpStatus(), azdata.TaskStatus.Succeeded);
should.equal(pythonInstalled, true);
});
testContext.httpClient.verify(x => x.download(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()), TypeMoq.Times.once());
it('installDependencies Should fail the task if installing python fails', async function (): Promise<void> {
let testContext = createContext();
let installedPackages = `[
{"name":"pymssql","version":"2.1.4"},
{"name":"sqlmlutils","version":"1.1.1"}
]`;
testContext.apiWrapper.setup(x => x.startBackgroundOperation(TypeMoq.It.isAny())).returns((operationInfo: azdata.BackgroundOperationInfo) => {
operationInfo.operation(testContext.op);
});
testContext.jupyterInstallation.installPythonPackage = () => {
return Promise.reject();
};
testContext.processService.setup(x => x.executeBufferedCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {return Promise.resolve(installedPackages);});
let packageManager = createPackageManager(testContext);
await should(packageManager.installDependencies()).rejected();
should.equal(testContext.getOpStatus(), azdata.TaskStatus.Failed);
});
it('installDependencies Should not install packages if already installed', async function (): Promise<void> {
@@ -107,9 +97,6 @@ describe('Package Manager', () => {
testContext.apiWrapper.setup(x => x.startBackgroundOperation(TypeMoq.It.isAny())).returns((operationInfo: azdata.BackgroundOperationInfo) => {
operationInfo.operation(testContext.op);
});
testContext.jupyterInstallation.installPythonPackage = () => {
return Promise.resolve();
};
testContext.processService.setup(x => x.executeBufferedCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((command) => {
if (command.indexOf('pip install') > 0) {
packagesInstalled = true;
@@ -132,9 +119,6 @@ describe('Package Manager', () => {
testContext.apiWrapper.setup(x => x.startBackgroundOperation(TypeMoq.It.isAny())).returns((operationInfo: azdata.BackgroundOperationInfo) => {
operationInfo.operation(testContext.op);
});
testContext.jupyterInstallation.installPythonPackage = () => {
return Promise.resolve();
};
testContext.processService.setup(x => x.executeBufferedCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((command) => {
if (command.indexOf('pip install') > 0) {
packagesInstalled = true;
@@ -154,9 +138,7 @@ describe('Package Manager', () => {
testContext.apiWrapper.setup(x => x.startBackgroundOperation(TypeMoq.It.isAny())).returns((operationInfo: azdata.BackgroundOperationInfo) => {
operationInfo.operation(testContext.op);
});
testContext.jupyterInstallation.installPythonPackage = () => {
return Promise.resolve();
};
testContext.processService.setup(x => x.executeBufferedCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((command,) => {
if (command.indexOf('pip list') > 0) {
return Promise.reject();
@@ -174,7 +156,7 @@ describe('Package Manager', () => {
should.equal(packagesInstalled, true);
});
it('installDependencies Should fail if install packages fails', async function (): Promise<void> {
it('installDependencies Should fail if download packages fails', async function (): Promise<void> {
let testContext = createContext();
let packagesInstalled = false;
let installedPackages = `[
@@ -183,9 +165,7 @@ describe('Package Manager', () => {
testContext.apiWrapper.setup(x => x.startBackgroundOperation(TypeMoq.It.isAny())).returns((operationInfo: azdata.BackgroundOperationInfo) => {
operationInfo.operation(testContext.op);
});
testContext.jupyterInstallation.installPythonPackage = () => {
return Promise.resolve();
};
testContext.httpClient.setup(x => x.download(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => Promise.reject());
testContext.processService.setup(x => x.executeBufferedCommand(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((command) => {
if (command.indexOf('pip list') > 0) {
return Promise.resolve(installedPackages);
@@ -207,15 +187,23 @@ describe('Package Manager', () => {
{ name: 'pymssql', version: '2.1.4' },
{ name: 'sqlmlutils', version: '' }
]);
testContext.config.setup(x => x.requiredRPackages).returns( () => [
{ name: 'RODBCext', repository: 'https://cran.microsoft.com' },
{ name: 'sqlmlutils', fileName: 'sqlmlutils_0.7.1.zip', downloadUrl: 'https://github.com/microsoft/sqlmlutils/blob/master/R/dist/sqlmlutils_0.7.1.zip?raw=true'}
]);
testContext.httpClient.setup(x => x.download(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => Promise.resolve());
testContext.config.setup(x => x.pythonExecutable).returns(() => 'python');
testContext.config.setup(x => x.rExecutable).returns(() => 'r');
let packageManager = new PackageManager(
testContext.nbExtensionApis,
testContext.outputChannel,
'',
testContext.apiWrapper.object,
testContext.queryRunner.object,
testContext.processService.object,
testContext.config.object);
testContext.config.object,
testContext.httpClient.object);
packageManager.init();
packageManager.dependenciesInstalled = true;
return packageManager;
}
});

92
extensions/machine-learning-services/src/test/packageManagement/sqlPackageManageProvider.test.ts → extensions/machine-learning-services/src/test/packageManagement/sqlPythonPackageManageProvider.test.ts
View File

@@ -10,11 +10,11 @@ import * as should from 'should';
import 'mocha';
import * as TypeMoq from 'typemoq';
import * as constants from '../../common/constants';
import { SqlPythonPackageManageProvider } from '../../packageManagement/sqlPackageManageProvider';
import { SqlPythonPackageManageProvider } from '../../packageManagement/sqlPythonPackageManageProvider';
import { createContext, TestContext } from './utils';
import * as nbExtensionApis from '../../typings/notebookServices';
describe('SQL Package Manager', () => {
describe('SQL Python Package Manager', () => {
it('Should create SQL package manager successfully', async function (): Promise<void> {
let testContext = createContext();
should.doesNotThrow(() => createProvider(testContext));
@@ -59,6 +59,38 @@ describe('SQL Package Manager', () => {
should.deepEqual(actual, expected);
});
it('listPackages Should return packages sorted by name and version', async function (): Promise<void> {
let testContext = createContext();
let packages: nbExtensionApis.IPackageDetails[] = [
{
'name': 'b-name',
'version': '1.1.1'
},
{
'name': 'b-name',
'version': '1.1.2'
}
];
let connection = new azdata.connection.ConnectionProfile();
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.queryRunner.setup(x => x.getPythonPackages(TypeMoq.It.isAny())).returns(() => Promise.resolve(packages));
let provider = createProvider(testContext);
let actual = await provider.listPackages();
let expected = [
{
'name': 'b-name',
'version': '1.1.1'
},
{
'name': 'b-name',
'version': '1.1.2'
}
];
should.deepEqual(actual, expected);
});
it('listPackages Should return empty packages if undefined packages returned', async function (): Promise<void> {
let testContext = createContext();
@@ -106,7 +138,7 @@ describe('SQL Package Manager', () => {
let credentials = { [azdata.ConnectionOptionSpecialType.password]: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((path, scripts: string[]) => {
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((path, scripts: string[]) => {
if (path && scripts.find(x => x.indexOf('install') > 0) &&
scripts.find(x => x.indexOf('port=1433') > 0) &&
@@ -118,7 +150,7 @@ describe('SQL Package Manager', () => {
packagesUpdated = true;
}
return Promise.resolve();
return Promise.resolve('');
});
let provider = createProvider(testContext);
@@ -147,7 +179,7 @@ describe('SQL Package Manager', () => {
let credentials = { [azdata.ConnectionOptionSpecialType.password]: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((path, scripts: string[]) => {
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((path, scripts: string[]) => {
if (path && scripts.find(x => x.indexOf('uninstall') > 0) &&
scripts.find(x => x.indexOf('port=1433') > 0) &&
@@ -158,7 +190,7 @@ describe('SQL Package Manager', () => {
packagesUpdated = true;
}
return Promise.resolve();
return Promise.resolve('');
});
let provider = createProvider(testContext);
@@ -187,7 +219,7 @@ describe('SQL Package Manager', () => {
let credentials = { [azdata.ConnectionOptionSpecialType.password]: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((path, scripts: string[]) => {
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((path, scripts: string[]) => {
if (path && scripts.find(x => x.indexOf('install') > 0) &&
scripts.find(x => x.indexOf('port=3433') > 0) &&
@@ -199,7 +231,7 @@ describe('SQL Package Manager', () => {
packagesUpdated = true;
}
return Promise.resolve();
return Promise.resolve('');
});
let provider = createProvider(testContext);
@@ -218,9 +250,9 @@ describe('SQL Package Manager', () => {
let credentials = { ['azdata.ConnectionOptionSpecialType.password']: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {
packagesUpdated = true;
return Promise.resolve();
return Promise.resolve('');
});
@@ -240,9 +272,9 @@ describe('SQL Package Manager', () => {
let credentials = { ['azdata.ConnectionOptionSpecialType.password']: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {
packagesUpdated = true;
return Promise.resolve();
return Promise.resolve('');
});
@@ -289,33 +321,16 @@ describe('SQL Package Manager', () => {
should.deepEqual(actual, true);
});
it('getPackageOverview Should not return undefined if python package provider not found', async function (): Promise<void> {
let testContext = createContext();
let provider = createProvider(testContext);
let actual = await provider.getPackageOverview('package name');
should.notEqual(actual, undefined);
});
it('getPackageOverview Should return package info using python packages provider', async function (): Promise<void> {
let testContext = createContext();
let packagePreview = {
'name': 'a-name',
'versions': ['1.1.2'],
'summary': ''
name: 'package name',
versions: ['0.0.2', '0.0.1'],
summary: 'package summary'
};
let pythonPackageManager: nbExtensionApis.IPackageManageProvider = {
providerId: 'localhost_Pip',
packageTarget: { location: '', packageType: '' },
listPackages: () => { return Promise.resolve([]); },
installPackages: () => { return Promise.resolve(); },
uninstallPackages: () => { return Promise.resolve(); },
canUseProvider: () => { return Promise.resolve(true); },
getLocationTitle: () => { return Promise.resolve(''); },
getPackageOverview: () => { return Promise.resolve(packagePreview); }
};
testContext.nbExtensionApis.registerPackageManager(pythonPackageManager.providerId, pythonPackageManager);
testContext.httpClient.setup(x => x.fetch(TypeMoq.It.isAny())).returns(() => {
return Promise.resolve(`{"info":{"summary":"package summary"}, "releases":{"0.0.1":[{"comment_text":""}], "0.0.2":[{"comment_text":""}]}}`);
});
let provider = createProvider(testContext);
let actual = await provider.getPackageOverview('package name');
@@ -362,12 +377,13 @@ describe('SQL Package Manager', () => {
});
function createProvider(testContext: TestContext): SqlPythonPackageManageProvider {
testContext.config.setup(x => x.pythonExecutable).returns(() => 'python');
return new SqlPythonPackageManageProvider(
testContext.nbExtensionApis,
testContext.outputChannel,
'',
testContext.apiWrapper.object,
testContext.queryRunner.object,
testContext.processService.object);
testContext.processService.object,
testContext.config.object,
testContext.httpClient.object);
}
});

313
extensions/machine-learning-services/src/test/packageManagement/sqlRPackageManageProvider.test.ts Normal file
View File

@@ -0,0 +1,313 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as azdata from 'azdata';
import * as should from 'should';
import 'mocha';
import * as TypeMoq from 'typemoq';
import * as constants from '../../common/constants';
import { SqlRPackageManageProvider } from '../../packageManagement/sqlRPackageManageProvider';
import { createContext, TestContext } from './utils';
import * as nbExtensionApis from '../../typings/notebookServices';
describe('SQL R Package Manager', () => {
it('Should create SQL package manager successfully', async function (): Promise<void> {
let testContext = createContext();
should.doesNotThrow(() => createProvider(testContext));
});
it('Should return provider Id and target correctly', async function (): Promise<void> {
let testContext = createContext();
let provider = createProvider(testContext);
should.deepEqual(SqlRPackageManageProvider.ProviderId, provider.providerId);
should.deepEqual({ location: 'SQL', packageType: 'R' }, provider.packageTarget);
});
it('listPackages Should return packages sorted by name', async function (): Promise<void> {
let testContext = createContext();
let packages: nbExtensionApis.IPackageDetails[] = [
{
'name': 'b-name',
'version': '1.1.1'
},
{
'name': 'a-name',
'version': '1.1.2'
}
];
let connection = new azdata.connection.ConnectionProfile();
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.queryRunner.setup(x => x.getRPackages(TypeMoq.It.isAny())).returns(() => Promise.resolve(packages));
let provider = createProvider(testContext);
let actual = await provider.listPackages();
let expected = [
{
'name': 'a-name',
'version': '1.1.2'
},
{
'name': 'b-name',
'version': '1.1.1'
}
];
should.deepEqual(actual, expected);
});
it('listPackages Should return empty packages if undefined packages returned', async function (): Promise<void> {
let testContext = createContext();
let connection = new azdata.connection.ConnectionProfile();
let packages: nbExtensionApis.IPackageDetails[];
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.queryRunner.setup(x => x.getRPackages(TypeMoq.It.isAny())).returns(() => Promise.resolve(packages));
let provider = createProvider(testContext);
let actual = await provider.listPackages();
let expected: nbExtensionApis.IPackageDetails[] = [];
should.deepEqual(actual, expected);
});
it('listPackages Should return empty packages if empty packages returned', async function (): Promise<void> {
let testContext = createContext();
let connection = new azdata.connection.ConnectionProfile();
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.queryRunner.setup(x => x.getRPackages(TypeMoq.It.isAny())).returns(() => Promise.resolve([]));
let provider = createProvider(testContext);
let actual = await provider.listPackages();
let expected: nbExtensionApis.IPackageDetails[] = [];
should.deepEqual(actual, expected);
});
it('installPackages Should install given packages successfully', async function (): Promise<void> {
let testContext = createContext();
let packagesUpdated = false;
let packages: nbExtensionApis.IPackageDetails[] = [
{
'name': 'a-name',
'version': '1.1.2'
},
{
'name': 'b-name',
'version': '1.1.1'
}
];
let connection = new azdata.connection.ConnectionProfile();
connection.serverName = 'serverName';
connection.databaseName = 'databaseName';
let credentials = { [azdata.ConnectionOptionSpecialType.password]: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((path, scripts: string[]) => {
if (path && scripts.find(x => x.indexOf('install') > 0) &&
scripts.find(x => x.indexOf('server="serverName"') > 0) &&
scripts.find(x => x.indexOf('database="databaseName"') > 0) &&
scripts.find(x => x.indexOf('"a-name"') > 0) &&
scripts.find(x => x.indexOf('pwd="password"') > 0)) {
packagesUpdated = true;
}
return Promise.resolve('');
});
let provider = createProvider(testContext);
await provider.installPackages(packages, false);
should.deepEqual(packagesUpdated, true);
});
it('uninstallPackages Should uninstall given packages successfully', async function (): Promise<void> {
let testContext = createContext();
let packagesUpdated = false;
let packages: nbExtensionApis.IPackageDetails[] = [
{
'name': 'a-name',
'version': '1.1.2'
},
{
'name': 'b-name',
'version': '1.1.1'
}
];
let connection = new azdata.connection.ConnectionProfile();
connection.serverName = 'serverName';
connection.databaseName = 'databaseName';
let credentials = { [azdata.ConnectionOptionSpecialType.password]: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns((path, scripts: string[]) => {
if (path && scripts.find(x => x.indexOf('remove') > 0) &&
scripts.find(x => x.indexOf('server="serverName"') > 0) &&
scripts.find(x => x.indexOf('database="databaseName"') > 0) &&
scripts.find(x => x.indexOf('"a-name"') > 0) &&
scripts.find(x => x.indexOf('pwd="password"') > 0)) {
packagesUpdated = true;
}
return Promise.resolve('');
});
let provider = createProvider(testContext);
await provider.uninstallPackages(packages);
should.deepEqual(packagesUpdated, true);
});
it('installPackages Should not install any packages give empty list', async function (): Promise<void> {
let testContext = createContext();
let packagesUpdated = false;
let packages: nbExtensionApis.IPackageDetails[] = [
];
let connection = new azdata.connection.ConnectionProfile();
let credentials = { ['azdata.ConnectionOptionSpecialType.password']: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {
packagesUpdated = true;
return Promise.resolve('');
});
let provider = createProvider(testContext);
await provider.installPackages(packages, false);
should.deepEqual(packagesUpdated, false);
});
it('uninstallPackages Should not uninstall any packages give empty list', async function (): Promise<void> {
let testContext = createContext();
let packagesUpdated = false;
let packages: nbExtensionApis.IPackageDetails[] = [
];
let connection = new azdata.connection.ConnectionProfile();
let credentials = { ['azdata.ConnectionOptionSpecialType.password']: 'password' };
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.apiWrapper.setup(x => x.getCredentials(TypeMoq.It.isAny())).returns(() => { return Promise.resolve(credentials); });
testContext.processService.setup(x => x.execScripts(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => {
packagesUpdated = true;
return Promise.resolve('');
});
let provider = createProvider(testContext);
await provider.uninstallPackages(packages);
should.deepEqual(packagesUpdated, false);
});
it('canUseProvider Should return false for no connection', async function (): Promise<void> {
let testContext = createContext();
let connection: azdata.connection.ConnectionProfile;
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
let provider = createProvider(testContext);
let actual = await provider.canUseProvider();
should.deepEqual(actual, false);
});
it('canUseProvider Should return false if connection does not have r installed', async function (): Promise<void> {
let testContext = createContext();
let connection = new azdata.connection.ConnectionProfile();
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.queryRunner.setup(x => x.isRInstalled(TypeMoq.It.isAny())).returns(() => Promise.resolve(false));
let provider = createProvider(testContext);
let actual = await provider.canUseProvider();
should.deepEqual(actual, false);
});
it('canUseProvider Should return true if connection has r installed', async function (): Promise<void> {
let testContext = createContext();
let connection = new azdata.connection.ConnectionProfile();
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
testContext.queryRunner.setup(x => x.isRInstalled(TypeMoq.It.isAny())).returns(() => Promise.resolve(true));
let provider = createProvider(testContext);
let actual = await provider.canUseProvider();
should.deepEqual(actual, true);
});
it('getPackageOverview Should return package info successfully', async function (): Promise<void> {
let testContext = createContext();
let packagePreview = {
'name': 'a-name',
'versions': ['1.1.2'],
'summary': ''
};
let allPackages = [{
'name': 'a-name',
'version': '1.1.2'
}];
testContext.queryRunner.setup(x => x.getRAvailablePackages(TypeMoq.It.isAny())).returns(() => Promise.resolve(allPackages));
let provider = createProvider(testContext);
let actual = await provider.getPackageOverview('a-name');
should.deepEqual(actual, packagePreview);
});
it('getLocationTitle Should default string for no connection', async function (): Promise<void> {
let testContext = createContext();
let connection: azdata.connection.ConnectionProfile;
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
let provider = createProvider(testContext);
let actual = await provider.getLocationTitle();
should.deepEqual(actual, constants.packageManagerNoConnection);
});
it('getLocationTitle Should return connection title string for valid connection', async function (): Promise<void> {
let testContext = createContext();
let connection = new azdata.connection.ConnectionProfile();
connection.serverName = 'serverName';
connection.databaseName = 'databaseName';
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
let provider = createProvider(testContext);
let actual = await provider.getLocationTitle();
should.deepEqual(actual, `${connection.serverName} ${connection.databaseName}`);
});
it('getLocationTitle Should return server name as connection title if there is not database name', async function (): Promise<void> {
let testContext = createContext();
let connection = new azdata.connection.ConnectionProfile();
connection.serverName = 'serverName';
testContext.apiWrapper.setup(x => x.getCurrentConnection()).returns(() => { return Promise.resolve(connection); });
let provider = createProvider(testContext);
let actual = await provider.getLocationTitle();
should.deepEqual(actual, `${connection.serverName} `);
});
function createProvider(testContext: TestContext): SqlRPackageManageProvider {
testContext.config.setup(x => x.rExecutable).returns(() => 'r');
return new SqlRPackageManageProvider(
testContext.outputChannel,
testContext.apiWrapper.object,
testContext.queryRunner.object,
testContext.processService.object,
testContext.config.object);
}
});

37
extensions/machine-learning-services/src/test/packageManagement/utils.ts
View File

@@ -7,17 +7,15 @@
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as nbExtensionApis from '../../typings/notebookServices';
import * as TypeMoq from 'typemoq';
import { ApiWrapper } from '../../common/apiWrapper';
import { QueryRunner } from '../../common/queryRunner';
import { ProcessService } from '../../common/processService';
import { Config } from '../../common/config';
import { Config } from '../../configurations/config';
import { HttpClient } from '../../common/httpClient';
export interface TestContext {
jupyterInstallation: nbExtensionApis.IJupyterServerInstallation;
jupyterController: nbExtensionApis.IJupyterController;
nbExtensionApis: nbExtensionApis.IExtensionApi;
outputChannel: vscode.OutputChannel;
processService: TypeMoq.IMock<ProcessService>;
apiWrapper: TypeMoq.IMock<ApiWrapper>;
@@ -25,39 +23,13 @@ export interface TestContext {
config: TypeMoq.IMock<Config>;
op: azdata.BackgroundOperation;
getOpStatus: () => azdata.TaskStatus;
httpClient: TypeMoq.IMock<HttpClient>;
}
export function createContext(): TestContext {
let opStatus: azdata.TaskStatus;
let packages = new Map<string, nbExtensionApis.IPackageManageProvider>();
let jupyterInstallation: nbExtensionApis.IJupyterServerInstallation = {
installCondaPackages: () => { return Promise.resolve(); },
getInstalledPipPackages: () => { return Promise.resolve([]); },
installPipPackages: () => { return Promise.resolve(); },
uninstallPipPackages: () => { return Promise.resolve(); },
uninstallCondaPackages: () => { return Promise.resolve(); },
executeBufferedCommand: () => { return Promise.resolve(''); },
executeStreamedCommand: () => { return Promise.resolve(); },
pythonExecutable: '',
pythonInstallationPath: '',
installPythonPackage: () => { return Promise.resolve(); }
};
let jupyterController = {
jupyterInstallation: jupyterInstallation
};
return {
jupyterInstallation: jupyterInstallation,
jupyterController: jupyterController,
nbExtensionApis: {
getJupyterController: () => { return jupyterController; },
registerPackageManager: (providerId: string, packageManagerProvider: nbExtensionApis.IPackageManageProvider) => {
packages.set(providerId, packageManagerProvider);
},
getPackageManagers: () => { return packages; },
},
outputChannel: {
name: '',
append: () => { },
@@ -71,6 +43,7 @@ export function createContext(): TestContext {
apiWrapper: TypeMoq.Mock.ofType(ApiWrapper),
queryRunner: TypeMoq.Mock.ofType(QueryRunner),
config: TypeMoq.Mock.ofType(Config),
httpClient: TypeMoq.Mock.ofType(HttpClient),
op: {
updateStatus: (status: azdata.TaskStatus) => {
opStatus = status;

34
extensions/machine-learning-services/yarn.lock
View File

@@ -2,16 +2,41 @@
# yarn lockfile v1
"@types/caseless@*":
version "0.12.2"
resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8"
integrity sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==
"@types/mocha@^5.2.5":
version "5.2.6"
resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-5.2.6.tgz#b8622d50557dd155e9f2f634b7d68fd38de5e94b"
integrity sha512-1axi39YdtBI7z957vdqXI4Ac25e7YihYQtJa+Clnxg1zTJEaIRbndt71O3sP4GAMgiAm0pY26/b9BrY4MR/PMw==
"@types/node@*":
version "13.1.6"
resolved "https://registry.yarnpkg.com/@types/node/-/node-13.1.6.tgz#076028d0b0400be8105b89a0a55550c86684ffec"
integrity sha512-Jg1F+bmxcpENHP23sVKkNuU3uaxPnsBMW0cLjleiikFKomJQbsn0Cqk2yDvQArqzZN6ABfBkZ0To7pQ8sLdWDg==
"@types/node@^10.14.8":
version "10.14.17"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.17.tgz#b96d4dd3e427382482848948041d3754d40fd5ce"
integrity sha512-p/sGgiPaathCfOtqu2fx5Mu1bcjuP8ALFg4xpGgNkcin7LwRyzUKniEHBKdcE1RPsenq5JVPIpMTJSygLboygQ==
"@types/request@^2.48.1":
version "2.48.4"
resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.4.tgz#df3d43d7b9ed3550feaa1286c6eabf0738e6cf7e"
integrity sha512-W1t1MTKYR8PxICH+A4HgEIPuAC3sbljoEVfyZbeFJJDbr30guDspJri2XOaM2E+Un7ZjrihaDi7cf6fPa2tbgw==
dependencies:
"@types/caseless" "*"
"@types/node" "*"
"@types/tough-cookie" "*"
form-data "^2.5.0"
"@types/tough-cookie@*":
version "2.3.6"
resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-2.3.6.tgz#c880579e087d7a0db13777ff8af689f4ffc7b0d5"
integrity sha512-wHNBMnkoEBiRAd3s8KTKwIuO9biFtTf0LehITzBhSco+HQI0xkXZbLOD55SW3Aqw3oUkHstkm5SPv58yaAdFPQ==
ajv@^6.5.5:
version "6.10.0"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.10.0.tgz#90d0d54439da587cd7e843bfb7045f50bd22bdf1"
@@ -440,6 +465,15 @@ forever-agent@~0.6.1:
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=
form-data@^2.5.0:
version "2.5.1"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4"
integrity sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==
dependencies:
asynckit "^0.4.0"
combined-stream "^1.0.6"
mime-types "^2.1.12"
form-data@~2.3.2:
version "2.3.3"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"

4
extensions/notebook/src/dialog/managePackages/installedPackagesTab.ts
View File

@@ -144,10 +144,10 @@ export class InstalledPackagesTab {
this.dialog.model.currentPackageType)
});
if (packageData && packageData.length > 0) {
if (packageData) {
await this.installedPackagesTable.updateProperties({
data: packageData,
selectedRows: [0]
selectedRows: packageData.length > 0 ? [0] : []
});
await this.uninstallPackageButton.updateProperties({ enabled: true });
}