From 28fff9ace8ced472790866f5eb025c4ec107fa51 Mon Sep 17 00:00:00 2001 From: Leila Lali Date: Mon, 27 Jan 2020 11:10:52 -0800 Subject: [PATCH] MLS R package management bug fixing (#8920) * MLS R package management bug fixing --- .../src/common/constants.ts | 1 + .../src/common/queryRunner.ts | 15 -------------- .../src/configurations/config.json | 4 +++- .../src/configurations/config.ts | 9 ++++++++- .../src/packageManagement/packageManager.ts | 4 ++-- .../sqlRPackageManageProvider.ts | 20 +++++++++++++------ .../sqlRPackageManageProvider.test.ts | 16 ++++++++------- 7 files changed, 37 insertions(+), 32 deletions(-) diff --git a/extensions/machine-learning-services/src/common/constants.ts b/extensions/machine-learning-services/src/common/constants.ts index a64f4b9fa4..33ddda939b 100644 --- a/extensions/machine-learning-services/src/common/constants.ts +++ b/extensions/machine-learning-services/src/common/constants.ts @@ -59,6 +59,7 @@ export const pythonConfigError = localize('mls.pythonConfigError', "Python execu export const rConfigError = localize('mls.rConfigError', "R executable is not configured"); export const installingDependencies = localize('mls.installingDependencies', "Installing dependencies ..."); export const resourceNotFoundError = localize('mls.resourceNotFound', "Could not find the specified resource"); +export const latestVersion = localize('mls.latestVersion', "Latest"); export function httpGetRequestError(code: number, message: string): string { return localize('mls.httpGetRequestError', "Package info request failed with error: {0} {1}", code, diff --git a/extensions/machine-learning-services/src/common/queryRunner.ts b/extensions/machine-learning-services/src/common/queryRunner.ts index 10aba9f151..c1495d8e20 100644 --- a/extensions/machine-learning-services/src/common/queryRunner.ts +++ b/extensions/machine-learning-services/src/common/queryRunner.ts @@ -25,13 +25,6 @@ EXEC sp_execute_external_script OutputDataSet <- as.data.frame(installed.packages()[,c(1,3)])' `; -const listRAvailablePackagesQuery = ` -EXEC sp_execute_external_script -@language=N'R', -@script=N' -OutputDataSet <- as.data.frame(installed.packages()[,c(1,3)])' -`; - const checkMlInstalledQuery = ` Declare @tablevar table(name NVARCHAR(MAX), min INT, max INT, config_value bit, run_value bit) insert into @tablevar(name, min, max, config_value, run_value) exec sp_configure @@ -82,14 +75,6 @@ export class QueryRunner { return this.getPackages(connection, listRPackagesQuery); } - /** - * Returns python packages installed in SQL server instance - * @param connection SQL Connection - */ - public async getRAvailablePackages(connection: azdata.connection.ConnectionProfile): Promise { - return this.getPackages(connection, listRAvailablePackagesQuery); - } - private async getPackages(connection: azdata.connection.ConnectionProfile, script: string): Promise { let packages: nbExtensionApis.IPackageDetails[] = []; let result: azdata.SimpleExecuteResult | undefined = undefined; diff --git a/extensions/machine-learning-services/src/configurations/config.json b/extensions/machine-learning-services/src/configurations/config.json index f28814be74..a9e3d622ec 100644 --- a/extensions/machine-learning-services/src/configurations/config.json +++ b/extensions/machine-learning-services/src/configurations/config.json @@ -7,5 +7,7 @@ "requiredRPackages": [ { "name": "RODBCext", "repository": "https://cran.microsoft.com" }, { "name": "sqlmlutils", "fileName": "sqlmlutils_0.7.1.zip", "downloadUrl": "https://github.com/microsoft/sqlmlutils/blob/master/R/dist/sqlmlutils_0.7.1.zip?raw=true"} - ] + ], + + "rPackagesRepository": "https://cran.r-project.org" } diff --git a/extensions/machine-learning-services/src/configurations/config.ts b/extensions/machine-learning-services/src/configurations/config.ts index 7c5b128795..90428681f2 100644 --- a/extensions/machine-learning-services/src/configurations/config.ts +++ b/extensions/machine-learning-services/src/configurations/config.ts @@ -30,7 +30,7 @@ export class Config { * Loads the config values */ public async load(): Promise { - const rawConfig = await fs.readFile(path.join(this._root, 'src', 'configurations', configFileName)); + const rawConfig = await fs.readFile(path.join(this._root, 'out', 'configurations', configFileName)); this._configValues = JSON.parse(rawConfig.toString()); } @@ -48,6 +48,13 @@ export class Config { return this._configValues.requiredRPackages; } + /** + * Returns r packages repository + */ + public get rPackagesRepository(): string { + return this._configValues.rPackagesRepository; + } + /** * Returns python path from user settings */ diff --git a/extensions/machine-learning-services/src/packageManagement/packageManager.ts b/extensions/machine-learning-services/src/packageManagement/packageManager.ts index bdc48d6268..5a597e98a3 100644 --- a/extensions/machine-learning-services/src/packageManagement/packageManager.ts +++ b/extensions/machine-learning-services/src/packageManagement/packageManager.ts @@ -38,7 +38,7 @@ export class PackageManager { private _config: Config, private _httpClient: HttpClient) { this._sqlPythonPackagePackageManager = new SqlPythonPackageManageProvider(this._outputChannel, this._apiWrapper, this._queryRunner, this._processService, this._config, this._httpClient); - this._sqlRPackageManager = new SqlRPackageManageProvider(this._outputChannel, this._apiWrapper, this._queryRunner, this._processService, this._config); + this._sqlRPackageManager = new SqlRPackageManageProvider(this._outputChannel, this._apiWrapper, this._queryRunner, this._processService, this._config, this._httpClient); } /** @@ -205,7 +205,7 @@ export class PackageManager { output = await this._processService.executeBufferedCommand(cmd, this._outputChannel); } else if (model.repository) { cmd = `"${this._rExecutable}" -e "install.packages('${model.name}', repos='${model.repository}')"`; - output = output + await this._processService.executeBufferedCommand(cmd, this._outputChannel); + output = await this._processService.executeBufferedCommand(cmd, this._outputChannel); } return output; } diff --git a/extensions/machine-learning-services/src/packageManagement/sqlRPackageManageProvider.ts b/extensions/machine-learning-services/src/packageManagement/sqlRPackageManageProvider.ts index 006e863e04..d51c6bf22b 100644 --- a/extensions/machine-learning-services/src/packageManagement/sqlRPackageManageProvider.ts +++ b/extensions/machine-learning-services/src/packageManagement/sqlRPackageManageProvider.ts @@ -12,6 +12,8 @@ import { ApiWrapper } from '../common/apiWrapper'; import { ProcessService } from '../common/processService'; import { Config } from '../configurations/config'; import { SqlPackageManageProviderBase, ScriptMode } from './SqlPackageManageProviderBase'; +import { HttpClient } from '../common/httpClient'; +import * as constants from '../common/constants'; @@ -30,7 +32,8 @@ export class SqlRPackageManageProvider extends SqlPackageManageProviderBase impl apiWrapper: ApiWrapper, private _queryRunner: QueryRunner, private _processService: ProcessService, - private _config: Config) { + private _config: Config, + private _httpClient: HttpClient) { super(apiWrapper); } @@ -73,6 +76,9 @@ export class SqlRPackageManageProvider extends SqlPackageManageProviderBase impl 'formals(quit)$save <- formals(q)$save <- "no"', 'library(sqlmlutils)', `connection <- connectionInfo(${connectionParts})`, + `r = getOption("repos")`, + `r["CRAN"] = "${this._config.rPackagesRepository}"`, + `options(repos = r)`, `pkgs <- c("${packageDetails.name}")`, `${rCommandScript}(connectionString = connection, pkgs, scope = "PUBLIC")`, 'q()' @@ -93,6 +99,10 @@ export class SqlRPackageManageProvider extends SqlPackageManageProviderBase impl return false; } + private getPackageLink(packageName: string): string { + return `${this._config.rPackagesRepository}/web/packages/${packageName}`; + } + /** * Returns package overview for given name * @param packageName Package Name @@ -100,13 +110,11 @@ export class SqlRPackageManageProvider extends SqlPackageManageProviderBase impl protected async fetchPackage(packageName: string): Promise { let packagePreview: nbExtensionApis.IPackageOverview = { name: packageName, - versions: [], + versions: [constants.latestVersion], summary: '' }; - let connection = await this.getCurrentConnection(); - let availablePackages = await this._queryRunner.getRAvailablePackages(connection); - let versions = availablePackages.filter(x => x.name === packageName).map(x => x.version); - packagePreview.versions = versions; + + await this._httpClient.fetch(this.getPackageLink(packageName)); return packagePreview; } } diff --git a/extensions/machine-learning-services/src/test/packageManagement/sqlRPackageManageProvider.test.ts b/extensions/machine-learning-services/src/test/packageManagement/sqlRPackageManageProvider.test.ts index 1871e93d28..926398daac 100644 --- a/extensions/machine-learning-services/src/test/packageManagement/sqlRPackageManageProvider.test.ts +++ b/extensions/machine-learning-services/src/test/packageManagement/sqlRPackageManageProvider.test.ts @@ -247,14 +247,14 @@ describe('SQL R Package Manager', () => { let testContext = createContext(); let packagePreview = { 'name': 'a-name', - 'versions': ['1.1.2'], + 'versions': ['Latest'], 'summary': '' }; - let allPackages = [{ - 'name': 'a-name', - 'version': '1.1.2' - }]; - testContext.queryRunner.setup(x => x.getRAvailablePackages(TypeMoq.It.isAny())).returns(() => Promise.resolve(allPackages)); + + testContext.httpClient.setup(x => x.fetch(TypeMoq.It.isAny())).returns(() => { + return Promise.resolve(``); + }); + let provider = createProvider(testContext); let actual = await provider.getPackageOverview('a-name'); @@ -301,11 +301,13 @@ describe('SQL R Package Manager', () => { function createProvider(testContext: TestContext): SqlRPackageManageProvider { testContext.config.setup(x => x.rExecutable).returns(() => 'r'); + testContext.config.setup(x => x.rPackagesRepository).returns(() => 'http://cran.r-project.org'); return new SqlRPackageManageProvider( testContext.outputChannel, testContext.apiWrapper.object, testContext.queryRunner.object, testContext.processService.object, - testContext.config.object); + testContext.config.object, + testContext.httpClient.object); } });