ML - dashboard icons and links (#10153)

* ML - dashboard icons and links
This commit is contained in:
Leila Lali
2020-04-28 21:21:30 -07:00
committed by GitHub
parent 046995f2a5
commit 04af41c424
145 changed files with 387 additions and 134 deletions

View File

@@ -0,0 +1,117 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as azdata from 'azdata';
import { ApiWrapper } from '../common/apiWrapper';
import * as nbExtensionApis from '../typings/notebookServices';
import * as utils from '../common/utils';
export enum ScriptMode {
Install = 'install',
Uninstall = 'uninstall'
}
export abstract class SqlPackageManageProviderBase {
/**
* Base class for all SQL package managers
*/
constructor(protected _apiWrapper: ApiWrapper) {
}
/**
* Returns database names
*/
public async getLocations(): Promise<nbExtensionApis.IPackageLocation[]> {
let connection = await this.getCurrentConnection();
if (connection) {
let databases = await this._apiWrapper.listDatabases(connection.connectionId);
return databases.map(x => {
return { displayName: x, name: x };
});
}
return [];
}
protected async getCurrentConnection(): Promise<azdata.connection.ConnectionProfile> {
return await this._apiWrapper.getCurrentConnection();
}
/**
* Installs given packages
* @param packages Packages to install
* @param useMinVersion minimum version
*/
public async installPackages(packages: nbExtensionApis.IPackageDetails[], useMinVersion: boolean, databaseName: string): Promise<void> {
if (packages) {
await Promise.all(packages.map(x => this.installPackage(x, useMinVersion, databaseName)));
}
//TODO: use useMinVersion
console.log(useMinVersion);
}
private async installPackage(packageDetail: nbExtensionApis.IPackageDetails, useMinVersion: boolean, databaseName: string): Promise<void> {
if (useMinVersion) {
let packageOverview = await this.getPackageOverview(packageDetail.name);
if (packageOverview && packageOverview.versions) {
let minVersion = packageOverview.versions[packageOverview.versions.length - 1];
packageDetail.version = minVersion;
}
}
await this.executeScripts(ScriptMode.Install, packageDetail, databaseName);
}
/**
* Uninstalls given packages
* @param packages Packages to uninstall
*/
public async uninstallPackages(packages: nbExtensionApis.IPackageDetails[], databaseName: string): Promise<void> {
if (packages) {
await Promise.all(packages.map(x => this.executeScripts(ScriptMode.Uninstall, x, databaseName)));
}
}
/**
* Returns package overview for given name
* @param packageName Package Name
*/
public async getPackageOverview(packageName: string): Promise<nbExtensionApis.IPackageOverview> {
let packageOverview = await this.fetchPackage(packageName);
if (packageOverview && packageOverview.versions) {
packageOverview.versions = utils.sortPackageVersions(packageOverview.versions, false);
}
return packageOverview;
}
/**
* Returns list of packages
*/
public async listPackages(databaseName: string): Promise<nbExtensionApis.IPackageDetails[]> {
let packages = await this.fetchPackages(databaseName);
if (packages) {
packages = packages.sort((a, b) => this.comparePackages(a, b));
} else {
packages = [];
}
return packages;
}
private comparePackages(p1: nbExtensionApis.IPackageDetails, p2: nbExtensionApis.IPackageDetails): number {
if (p1 && p2) {
let compare = p1.name.localeCompare(p2.name);
if (compare === 0) {
compare = utils.comparePackageVersions(p1.version, p2.version);
}
return compare;
}
return p1 ? 1 : -1;
}
protected abstract fetchPackage(packageName: string): Promise<nbExtensionApis.IPackageOverview>;
protected abstract fetchPackages(databaseName: string): Promise<nbExtensionApis.IPackageDetails[]>;
protected abstract executeScripts(scriptMode: ScriptMode, packageDetails: nbExtensionApis.IPackageDetails, databaseName: string): Promise<void>;
}

View File

@@ -0,0 +1,95 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import { QueryRunner } from '../common/queryRunner';
import * as constants from '../common/constants';
import { ApiWrapper } from '../common/apiWrapper';
import * as utils from '../common/utils';
import * as nbExtensionApis from '../typings/notebookServices';
export class PackageManagementService {
/**
* Creates a new instance of ServerConfigManager
*/
constructor(
private _apiWrapper: ApiWrapper,
private _queryRunner: QueryRunner,
) {
}
/**
* Opens server config documents
*/
public async openDocuments(): Promise<boolean> {
return await this._apiWrapper.openExternal(vscode.Uri.parse(constants.mlsDocuments));
}
/**
* Returns true if mls is installed in the give SQL server instance
*/
public async isMachineLearningServiceEnabled(connection: azdata.connection.ConnectionProfile): Promise<boolean> {
return this._queryRunner.isMachineLearningServiceEnabled(connection);
}
/**
* Returns true if R installed in the give SQL server instance
*/
public async isRInstalled(connection: azdata.connection.ConnectionProfile): Promise<boolean> {
return this._queryRunner.isRInstalled(connection);
}
/**
* Returns true if python installed in the give SQL server instance
*/
public async isPythonInstalled(connection: azdata.connection.ConnectionProfile): Promise<boolean> {
return this._queryRunner.isPythonInstalled(connection);
}
/**
* Updates external script config
* @param connection SQL Connection
* @param enable if true external script will be enabled
*/
public async enableExternalScriptConfig(connection: azdata.connection.ConnectionProfile): Promise<boolean> {
let current = await this._queryRunner.isMachineLearningServiceEnabled(connection);
if (current) {
return current;
}
let confirmed = await utils.promptConfirm(constants.confirmEnableExternalScripts, this._apiWrapper);
if (confirmed) {
await this._queryRunner.updateExternalScriptConfig(connection, true);
current = await this._queryRunner.isMachineLearningServiceEnabled(connection);
if (current) {
this._apiWrapper.showInfoMessage(constants.mlsEnabledMessage);
} else {
this._apiWrapper.showErrorMessage(constants.mlsConfigUpdateFailed);
}
} else {
this._apiWrapper.showErrorMessage(constants.externalScriptsIsRequiredError);
}
return current;
}
/**
* Returns python packages installed in SQL server instance
* @param connection SQL Connection
*/
public async getPythonPackages(connection: azdata.connection.ConnectionProfile, databaseName: string): Promise<nbExtensionApis.IPackageDetails[]> {
return this._queryRunner.getPythonPackages(connection, databaseName);
}
/**
* Returns python packages installed in SQL server instance
* @param connection SQL Connection
*/
public async getRPackages(connection: azdata.connection.ConnectionProfile, databaseName: string): Promise<nbExtensionApis.IPackageDetails[]> {
return this._queryRunner.getRPackages(connection, databaseName);
}
}

View File

@@ -0,0 +1,219 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as nbExtensionApis from '../typings/notebookServices';
import { SqlPythonPackageManageProvider } from './sqlPythonPackageManageProvider';
import * as utils from '../common/utils';
import * as constants from '../common/constants';
import { ApiWrapper } from '../common/apiWrapper';
import { ProcessService } from '../common/processService';
import { Config } from '../configurations/config';
import { isNullOrUndefined } from 'util';
import { SqlRPackageManageProvider } from './sqlRPackageManageProvider';
import { HttpClient } from '../common/httpClient';
import { PackageConfigModel } from '../configurations/packageConfigModel';
import { PackageManagementService } from './packageManagementService';
export class PackageManager {
private _sqlPythonPackagePackageManager: SqlPythonPackageManageProvider;
private _sqlRPackageManager: SqlRPackageManageProvider;
public dependenciesInstalled: boolean = false;
/**
* Creates a new instance of PackageManager
*/
constructor(
private _outputChannel: vscode.OutputChannel,
private _rootFolder: string,
private _apiWrapper: ApiWrapper,
private _service: PackageManagementService,
private _processService: ProcessService,
private _config: Config,
private _httpClient: HttpClient) {
this._sqlPythonPackagePackageManager = new SqlPythonPackageManageProvider(this._outputChannel, this._apiWrapper, this._service, this._processService, this._config, this._httpClient);
this._sqlRPackageManager = new SqlRPackageManageProvider(this._outputChannel, this._apiWrapper, this._service, this._processService, this._config, this._httpClient);
}
/**
* Initializes the instance and resister SQL package manager with manage package dialog
*/
public init(): void {
}
private get pythonExecutable(): string {
return this._config.pythonExecutable;
}
private get _rExecutable(): string {
return this._config.rExecutable;
}
/**
* Returns packageManageProviders
*/
public get packageManageProviders(): nbExtensionApis.IPackageManageProvider[] {
return [
this._sqlPythonPackagePackageManager,
this._sqlRPackageManager
];
}
/**
* Executes manage package command for SQL server packages.
*/
public async managePackages(): Promise<void> {
try {
await this.enableExternalScript();
// Only execute the command if there's a valid connection with ml configuration enabled
//
let connection = await this.getCurrentConnection();
let isPythonInstalled = await this._service.isPythonInstalled(connection);
let isRInstalled = await this._service.isRInstalled(connection);
let defaultProvider: SqlRPackageManageProvider | SqlPythonPackageManageProvider | undefined;
if (connection && isPythonInstalled && this._sqlPythonPackagePackageManager.canUseProvider) {
defaultProvider = this._sqlPythonPackagePackageManager;
} else if (connection && isRInstalled && this._sqlRPackageManager.canUseProvider) {
defaultProvider = this._sqlRPackageManager;
}
if (connection && defaultProvider) {
await this.enableExternalScript();
// Install dependencies
//
if (!this.dependenciesInstalled) {
await this.installDependencies();
this.dependenciesInstalled = true;
}
// Execute the command
//
this._apiWrapper.executeCommand(constants.managePackagesCommand, {
defaultLocation: defaultProvider.packageTarget.location,
defaultProviderId: defaultProvider.providerId
});
} else {
this._apiWrapper.showInfoMessage(constants.managePackageCommandError);
}
} catch (err) {
this._apiWrapper.showErrorMessage(err);
}
}
public async enableExternalScript(): Promise<void> {
let connection = await this.getCurrentConnection();
if (!await this._service.enableExternalScriptConfig(connection)) {
throw Error(constants.externalScriptsIsRequiredError);
}
}
/**
* Installs dependencies for the extension
*/
public async installDependencies(): Promise<void> {
await utils.executeTasks(this._apiWrapper, constants.installPackageMngDependenciesMsgTaskName, [
this.installRequiredPythonPackages(this._config.requiredSqlPythonPackages),
this.installRequiredRPackages()], true);
}
private async installRequiredRPackages(): Promise<void> {
if (!this._config.rEnabled) {
return;
}
if (!this._rExecutable) {
throw new Error(constants.rConfigError);
}
await utils.createFolder(utils.getRPackagesFolderPath(this._rootFolder));
await Promise.all(this._config.requiredSqlRPackages.map(x => this.installRPackage(x)));
}
/**
* Installs required python packages
*/
public async installRequiredPythonPackages(requiredPackages: PackageConfigModel[]): Promise<void> {
if (!this._config.pythonEnabled) {
return;
}
if (!this.pythonExecutable) {
throw new Error(constants.pythonConfigError);
}
if (!requiredPackages || requiredPackages.length === 0) {
return;
}
let installedPackages = await this.getInstalledPipPackages();
let fileContent = '';
requiredPackages.forEach(packageDetails => {
let hasVersion = ('version' in packageDetails) && !isNullOrUndefined(packageDetails['version']) && packageDetails['version'].length > 0;
if (!installedPackages.find(x => x.name === packageDetails['name']
&& (!hasVersion || utils.comparePackageVersions(packageDetails['version'] || '', x.version) <= 0))) {
let packageNameDetail = hasVersion ? `${packageDetails.name}==${packageDetails.version}` : `${packageDetails.name}`;
fileContent = `${fileContent}${packageNameDetail}\n`;
}
});
if (fileContent) {
let confirmed = await utils.promptConfirm(constants.confirmInstallPythonPackages(fileContent), this._apiWrapper);
if (confirmed) {
this._outputChannel.appendLine(constants.installDependenciesPackages);
let result = await utils.execCommandOnTempFile<string>(fileContent, async (tempFilePath) => {
return await this.installPipPackage(tempFilePath);
});
this._outputChannel.appendLine(result);
} else {
throw Error(constants.requiredPackagesNotInstalled);
}
} else {
this._outputChannel.appendLine(constants.installDependenciesPackagesAlreadyInstalled);
}
}
private async getInstalledPipPackages(): Promise<nbExtensionApis.IPackageDetails[]> {
try {
let cmd = `"${this.pythonExecutable}" -m pip list --format=json`;
let packagesInfo = await this._processService.executeBufferedCommand(cmd, undefined);
let packagesResult: nbExtensionApis.IPackageDetails[] = [];
if (packagesInfo && packagesInfo.indexOf(']') > 0) {
packagesResult = <nbExtensionApis.IPackageDetails[]>JSON.parse(packagesInfo.substr(0, packagesInfo.indexOf(']') + 1));
}
return packagesResult;
}
catch (err) {
this._outputChannel.appendLine(constants.installDependenciesGetPackagesError(err ? err.message : ''));
return [];
}
}
private async getCurrentConnection(): Promise<azdata.connection.ConnectionProfile> {
return await this._apiWrapper.getCurrentConnection();
}
private async installPipPackage(requirementFilePath: string): Promise<string> {
let cmd = `"${this.pythonExecutable}" -m pip install -r "${requirementFilePath}"`;
return await this._processService.executeBufferedCommand(cmd, this._outputChannel);
}
private async installRPackage(model: PackageConfigModel): Promise<string> {
let output = '';
let cmd = '';
if (model.downloadUrl) {
const packageFile = utils.getPackageFilePath(this._rootFolder, model.fileName || model.name);
const packageExist = await utils.exists(packageFile);
if (!packageExist) {
await this._httpClient.download(model.downloadUrl, packageFile, this._outputChannel);
}
cmd = `"${this._rExecutable}" CMD INSTALL ${packageFile}`;
output = await this._processService.executeBufferedCommand(cmd, this._outputChannel);
} else if (model.repository) {
cmd = `"${this._rExecutable}" -e "install.packages('${model.name}', repos='${model.repository}')"`;
output = await this._processService.executeBufferedCommand(cmd, this._outputChannel);
}
return output;
}
}

View File

@@ -0,0 +1,136 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as nbExtensionApis from '../typings/notebookServices';
import { ApiWrapper } from '../common/apiWrapper';
import { ProcessService } from '../common/processService';
import { Config } from '../configurations/config';
import { SqlPackageManageProviderBase, ScriptMode } from './packageManageProviderBase';
import { HttpClient } from '../common/httpClient';
import * as utils from '../common/utils';
import { PackageManagementService } from './packageManagementService';
/**
* Manage Package Provider for python packages inside SQL server databases
*/
export class SqlPythonPackageManageProvider extends SqlPackageManageProviderBase implements nbExtensionApis.IPackageManageProvider {
public static ProviderId = 'sql_Python';
/**
* Creates new a instance
*/
constructor(
private _outputChannel: vscode.OutputChannel,
apiWrapper: ApiWrapper,
private _service: PackageManagementService,
private _processService: ProcessService,
private _config: Config,
private _httpClient: HttpClient) {
super(apiWrapper);
}
/**
* Returns provider Id
*/
public get providerId(): string {
return SqlPythonPackageManageProvider.ProviderId;
}
/**
* Returns package target
*/
public get packageTarget(): nbExtensionApis.IPackageTarget {
return { location: 'SQL', packageType: 'Python' };
}
/**
* Returns list of packages
*/
protected async fetchPackages(databaseName: string): Promise<nbExtensionApis.IPackageDetails[]> {
return await this._service.getPythonPackages(await this.getCurrentConnection(), databaseName);
}
/**
* Execute a script to install or uninstall a python package inside current SQL Server connection
* @param packageDetails Packages to install or uninstall
* @param scriptMode can be 'install' or 'uninstall'
*/
protected async executeScripts(scriptMode: ScriptMode, packageDetails: nbExtensionApis.IPackageDetails, databaseName: string): Promise<void> {
let connection = await this.getCurrentConnection();
let credentials = await this._apiWrapper.getCredentials(connection.connectionId);
if (connection) {
let port = '1433';
let server = connection.serverName;
let database = databaseName ? `, database="${databaseName}"` : '';
let index = connection.serverName.indexOf(',');
if (index > 0) {
port = connection.serverName.substring(index + 1);
server = connection.serverName.substring(0, index);
}
let pythonConnectionParts = `server="${server}", port=${port}, uid="${connection.userName}", pwd="${credentials[azdata.ConnectionOptionSpecialType.password]}"${database})`;
let pythonCommandScript = scriptMode === ScriptMode.Install ?
`pkgmanager.install(package="${packageDetails.name}", version="${packageDetails.version}")` :
`pkgmanager.uninstall(package_name="${packageDetails.name}")`;
let scripts: string[] = [
'import sqlmlutils',
`connection = sqlmlutils.ConnectionInfo(driver="ODBC Driver 17 for SQL Server", ${pythonConnectionParts}`,
'pkgmanager = sqlmlutils.SQLPackageManager(connection)',
pythonCommandScript
];
let pythonExecutable = this._config.pythonExecutable;
await this._processService.execScripts(pythonExecutable, scripts, [], this._outputChannel);
}
}
/**
* Returns true if the provider can be used
*/
async canUseProvider(): Promise<boolean> {
if (!this._config.pythonEnabled) {
return false;
}
let connection = await this.getCurrentConnection();
if (connection && await this._service.isPythonInstalled(connection)) {
return true;
}
return false;
}
private getPackageLink(packageName: string): string {
return `https://pypi.org/pypi/${packageName}/json`;
}
protected async fetchPackage(packageName: string): Promise<nbExtensionApis.IPackageOverview> {
let body = await this._httpClient.fetch(this.getPackageLink(packageName));
let packagesJson = JSON.parse(body);
let versionNums: string[] = [];
let packageSummary = '';
if (packagesJson) {
if (packagesJson.releases) {
let versionKeys = Object.keys(packagesJson.releases);
versionKeys = versionKeys.filter(versionKey => {
let releaseInfo = packagesJson.releases[versionKey];
return Array.isArray(releaseInfo) && releaseInfo.length > 0;
});
versionNums = utils.sortPackageVersions(versionKeys, false);
}
if (packagesJson.info && packagesJson.info.summary) {
packageSummary = packagesJson.info.summary;
}
}
return {
name: packageName,
versions: versionNums,
summary: packageSummary
};
}
}

View File

@@ -0,0 +1,123 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as nbExtensionApis from '../typings/notebookServices';
import { ApiWrapper } from '../common/apiWrapper';
import { ProcessService } from '../common/processService';
import { Config } from '../configurations/config';
import { SqlPackageManageProviderBase, ScriptMode } from './packageManageProviderBase';
import { HttpClient } from '../common/httpClient';
import * as constants from '../common/constants';
import { PackageManagementService } from './packageManagementService';
/**
* Manage Package Provider for r packages inside SQL server databases
*/
export class SqlRPackageManageProvider extends SqlPackageManageProviderBase implements nbExtensionApis.IPackageManageProvider {
public static ProviderId = 'sql_R';
/**
* Creates new a instance
*/
constructor(
private _outputChannel: vscode.OutputChannel,
apiWrapper: ApiWrapper,
private _service: PackageManagementService,
private _processService: ProcessService,
private _config: Config,
private _httpClient: HttpClient) {
super(apiWrapper);
}
/**
* Returns provider Id
*/
public get providerId(): string {
return SqlRPackageManageProvider.ProviderId;
}
/**
* Returns package target
*/
public get packageTarget(): nbExtensionApis.IPackageTarget {
return { location: 'SQL', packageType: 'R' };
}
/**
* Returns list of packages
*/
protected async fetchPackages(databaseName: string): Promise<nbExtensionApis.IPackageDetails[]> {
return await this._service.getRPackages(await this.getCurrentConnection(), databaseName);
}
/**
* Execute a script to install or uninstall a r package inside current SQL Server connection
* @param packageDetails Packages to install or uninstall
* @param scriptMode can be 'install' or 'uninstall'
*/
protected async executeScripts(scriptMode: ScriptMode, packageDetails: nbExtensionApis.IPackageDetails, databaseName: string): Promise<void> {
let connection = await this.getCurrentConnection();
let credentials = await this._apiWrapper.getCredentials(connection.connectionId);
if (connection) {
let database = databaseName ? `, database="${databaseName}"` : '';
let connectionParts = `server="${connection.serverName}", uid="${connection.userName}", pwd="${credentials[azdata.ConnectionOptionSpecialType.password]}"${database}`;
let rCommandScript = scriptMode === ScriptMode.Install ? 'sql_install.packages' : 'sql_remove.packages';
let scripts: string[] = [
'formals(quit)$save <- formals(q)$save <- "no"',
'library(sqlmlutils)',
`connection <- connectionInfo(${connectionParts})`,
`r = getOption("repos")`,
`r["CRAN"] = "${this._config.rPackagesRepository}"`,
`options(repos = r)`,
`pkgs <- c("${packageDetails.name}")`,
`${rCommandScript}(connectionString = connection, pkgs, scope = "PUBLIC")`,
'q()'
];
let rExecutable = this._config.rExecutable;
await this._processService.execScripts(`${rExecutable}`, scripts, ['--vanilla'], this._outputChannel);
}
}
/**
* Returns true if the provider can be used
*/
async canUseProvider(): Promise<boolean> {
if (!this._config.rEnabled) {
return false;
}
let connection = await this.getCurrentConnection();
if (connection && await this._service.isRInstalled(connection)) {
return true;
}
return false;
}
private getPackageLink(packageName: string): string {
return `${this._config.rPackagesRepository}/web/packages/${packageName}`;
}
/**
* Returns package overview for given name
* @param packageName Package Name
*/
protected async fetchPackage(packageName: string): Promise<nbExtensionApis.IPackageOverview> {
let packagePreview: nbExtensionApis.IPackageOverview = {
name: packageName,
versions: [constants.latestVersion],
summary: ''
};
await this._httpClient.fetch(this.getPackageLink(packageName));
return packagePreview;
}
}