ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 18:46:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove all Big Data Cluster features (#21369)

Browse Source
This commit is contained in:
Cory Rivera
2022-12-07 12:28:17 -08:00
committed by GitHub
parent bb1f5bfffe
commit e2327c393a
213 changed files with 346 additions and 46800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
extensions/notebook/src/jupyter/jupyterController.ts
View File

@@ -6,12 +6,10 @@
import * as path from 'path';
import * as azdata from 'azdata';
import * as vscode from 'vscode';
import * as os from 'os';
import * as nls from 'vscode-nls';
const localize = nls.loadMessageBundle();
import * as constants from '../common/constants';
import * as localizedConstants from '../common/localizedConstants';
import { JupyterServerInstallation } from './jupyterServerInstallation';
import * as utils from '../common/utils';
import { IPrompter, IQuestion, QuestionTypes } from '../prompts/question';
@@ -66,9 +64,6 @@ export class JupyterController {
vscode.commands.registerCommand(constants.jupyterNewNotebookCommand, (explorerContext: azdata.ObjectExplorerContext) => {
return this.saveProfileAndCreateNotebook(explorerContext ? explorerContext.connectionProfile : undefined);
});
vscode.commands.registerCommand(constants.jupyterAnalyzeCommand, (explorerContext: azdata.ObjectExplorerContext) => {
return this.saveProfileAndAnalyzeNotebook(explorerContext);
});
vscode.commands.registerCommand(constants.jupyterReinstallDependenciesCommand, () => { return this.handleDependenciesReinstallation(); });
vscode.commands.registerCommand(constants.jupyterManagePackages, async (args) => { return this.doManagePackages(args); });
@@ -93,11 +88,7 @@ export class JupyterController {
}
private saveProfileAndCreateNotebook(profile: azdata.IConnectionProfile): Promise<void> {
return this.handleNewNotebookTask(undefined, profile);
}
private saveProfileAndAnalyzeNotebook(oeContext: azdata.ObjectExplorerContext): Promise<void> {
return this.handleNewNotebookTask(oeContext, oeContext.connectionProfile);
return this.handleNewNotebookTask(profile);
}
// EVENT HANDLERS //////////////////////////////////////////////////////
@@ -130,34 +121,17 @@ export class JupyterController {
}
}
private async handleNewNotebookTask(oeContext?: azdata.ObjectExplorerContext, profile?: azdata.IConnectionProfile): Promise<void> {
let editor = await azdata.nb.showNotebookDocument(vscode.Uri.from({ scheme: 'untitled' }), {
private async handleNewNotebookTask(profile?: azdata.IConnectionProfile): Promise<void> {
await azdata.nb.showNotebookDocument(vscode.Uri.from({ scheme: 'untitled' }), {
connectionProfile: profile,
providerId: constants.jupyterNotebookProviderId,
preview: false,
defaultKernel: {
name: 'pysparkkernel',
display_name: 'PySpark',
name: 'python3',
display_name: 'Python 3',
language: 'python'
}
});
if (oeContext && oeContext.nodeInfo && oeContext.nodeInfo.nodePath) {
// Get the file path after '/HDFS'
let hdfsPath: string = oeContext.nodeInfo.nodePath.substring(oeContext.nodeInfo.nodePath.indexOf('/HDFS') + '/HDFS'.length);
if (hdfsPath.length > 0) {
let analyzeCommand = '#' + localizedConstants.msgSampleCodeDataFrame + os.EOL + 'df = (spark.read.option(\"inferSchema\", \"true\")'
+ os.EOL + '.option(\"header\", \"true\")' + os.EOL + '.csv(\'{0}\'))' + os.EOL + 'df.show(10)';
// TODO re-enable insert into document once APIs are finalized.
// editor.document.cells[0].source = [analyzeCommand.replace('{0}', hdfsPath)];
await editor.edit(editBuilder => {
editBuilder.replace(0, {
cell_type: 'code',
source: analyzeCommand.replace('{0}', hdfsPath)
});
});
}
}
}
private async handleDependenciesReinstallation(): Promise<void> {

6
extensions/notebook/src/jupyter/jupyterKernel.ts
View File

@@ -60,12 +60,6 @@ export class JupyterKernel implements nb.IKernel {
return true;
}
public get requiresConnection(): boolean {
// TODO would be good to have a smarter way to do this.
// for now only Spark kernels need a connection
return !!(this.kernelImpl.name && this.kernelImpl.name.toLowerCase().indexOf('spark') > -1);
}
public get isReady(): boolean {
return this.kernelImpl.isReady;
}

21
extensions/notebook/src/jupyter/jupyterServerInstallation.ts
View File

@@ -88,22 +88,6 @@ export const requiredPowershellPkg: PythonPkgDetails = {
version: '0.1.4'
};
export const requiredSparkPackages: PythonPkgDetails[] = [
requiredJupyterPkg,
{
name: 'cryptography',
version: '3.2.1',
installExactVersion: true
},
{
name: 'sparkmagic',
version: '0.12.9'
}, {
name: 'pandas',
version: '0.24.2'
}
];
export class JupyterServerInstallation implements IJupyterServerInstallation {
public extensionPath: string;
public pythonBinPath: string;
@@ -162,11 +146,8 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
this._requiredKernelPackages.set(constants.ipykernelDisplayName, [requiredJupyterPkg]);
this._requiredKernelPackages.set(constants.python3DisplayName, [requiredJupyterPkg]);
this._requiredKernelPackages.set(constants.powershellDisplayName, [requiredJupyterPkg, requiredPowershellPkg]);
this._requiredKernelPackages.set(constants.pysparkDisplayName, requiredSparkPackages);
this._requiredKernelPackages.set(constants.sparkScalaDisplayName, requiredSparkPackages);
this._requiredKernelPackages.set(constants.sparkRDisplayName, requiredSparkPackages);
let allPackages = requiredSparkPackages.concat(requiredPowershellPkg);
let allPackages = [requiredJupyterPkg, requiredPowershellPkg];
this._requiredKernelPackages.set(constants.allKernelsName, allPackages);
this._requiredPackagesSet = new Set<string>();

237
extensions/notebook/src/jupyter/jupyterSessionManager.ts
View File

@@ -3,61 +3,16 @@
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { nb, ServerInfo, connection, IConnectionProfile, credentials } from 'azdata';
import { nb, IConnectionProfile } from 'azdata';
import { Session, Kernel } from '@jupyterlab/services';
import * as fs from 'fs-extra';
import * as nls from 'vscode-nls';
import * as vscode from 'vscode';
import * as path from 'path';
import * as utils from '../common/utils';
const localize = nls.loadMessageBundle();
import { JupyterKernel } from './jupyterKernel';
import { Deferred } from '../common/promise';
import { JupyterServerInstallation } from './jupyterServerInstallation';
import * as bdc from 'bdc';
import { noBDCConnectionError, providerNotValidError } from '../common/localizedConstants';
import { SQL_PROVIDER, CONTROLLER_ENDPOINT, KNOX_ENDPOINT_GATEWAY, KNOX_ENDPOINT_SERVER, KNOX_ENDPOINT_PORT } from '../common/constants';
import CodeAdapter from '../prompts/adapter';
import { IQuestion, QuestionTypes } from '../prompts/question';
import { ExtensionContextHelper } from '../common/extensionContextHelper';
import Logger from '../common/logger';
const configBase = {
'kernel_python_credentials': {
'url': ''
},
'kernel_scala_credentials': {
'url': ''
},
'kernel_r_credentials': {
'url': ''
},
'livy_session_startup_timeout_seconds': 100,
'logging_config': {
'version': 1,
'formatters': {
'magicsFormatter': {
'format': '%(asctime)s\t%(levelname)s\t%(message)s',
'datefmt': ''
}
},
'handlers': {
'magicsHandler': {
'class': 'hdijupyterutils.filehandler.MagicsFileHandler',
'formatter': 'magicsFormatter',
'home_path': ''
}
},
'loggers': {
'magicsLogger': {
'handlers': ['magicsHandler'],
'level': 'DEBUG',
'propagate': 0
}
}
}
};
export class JupyterSessionManager implements nb.SessionManager, vscode.Disposable {
private _ready: Deferred<void>;
@@ -114,10 +69,6 @@ export class JupyterSessionManager implements nb.SessionManager, vscode.Disposab
return kernel;
});
// For now, need to remove PySpark3, as it's been deprecated
// May want to have a formalized deprecated kernels mechanism in the future
kernels = kernels.filter(k => k.name !== 'pyspark3kernel');
let allKernels: nb.IAllKernels = {
defaultKernel: specs.default,
kernels: kernels
@@ -263,117 +214,12 @@ export class JupyterSession implements nb.ISession {
});
}
public async configureKernel(): Promise<void> {
let sparkmagicConfDir = path.join(utils.getUserHome(), '.sparkmagic');
await utils.ensureDir(sparkmagicConfDir);
// Default to localhost in config file.
let creds: ICredentials = {
'url': 'http://localhost:8088'
};
let config: ISparkMagicConfig = Object.assign({}, configBase);
this.updateConfig(config, creds, sparkmagicConfDir);
let configFilePath = path.join(sparkmagicConfDir, 'config.json');
await fs.writeFile(configFilePath, JSON.stringify(config));
configureKernel(kernelInfo: nb.IKernelSpec): Thenable<void> {
return Promise.resolve();
}
public async configureConnection(connectionProfile: IConnectionProfile): Promise<void> {
if (connectionProfile && connectionProfile.providerName && utils.isSparkKernel(this.sessionImpl.kernel.name)) {
Logger.log(`Configuring Spark connection`);
// %_do_not_call_change_endpoint is a SparkMagic command that lets users change endpoint options,
// such as user/profile/host name/auth type
let knoxUsername = connectionProfile.userName || 'root';
let knoxPassword: string = '';
//Update server info with bigdata endpoint - Unified Connection
if (connectionProfile.providerName === SQL_PROVIDER) {
const serverInfo: ServerInfo = await connection.getServerInfo(connectionProfile.id);
if (!serverInfo?.options['isBigDataCluster']) {
throw new Error(noBDCConnectionError);
}
const endpoints = utils.getClusterEndpoints(serverInfo);
const controllerEndpoint = endpoints.find(ep => ep.name.toLowerCase() === CONTROLLER_ENDPOINT);
Logger.log(`Found controller endpoint ${controllerEndpoint.endpoint}`);
// root is the default username for pre-CU5 instances, so while we prefer to use the connection username
// as a default now we'll still fall back to root if it's empty for some reason. (but the calls below should
// get the actual correct value regardless)
let clusterController: bdc.IClusterController | undefined = undefined;
if (!utils.isIntegratedAuth(connectionProfile)) {
// See if the controller creds have been saved already, otherwise fall back to using
// SQL creds as a default
const credentialProvider = await credentials.getProvider('notebook.bdc.password');
const usernameKey = `notebook.bdc.username::${connectionProfile.id}`;
const savedUsername = ExtensionContextHelper.extensionContext.globalState.get<string>(usernameKey) || connectionProfile.userName;
const connectionCreds = await connection.getCredentials(connectionProfile.id);
const savedPassword = (await credentialProvider.readCredential(connectionProfile.id)).password || connectionCreds.password;
clusterController = await getClusterController(controllerEndpoint.endpoint, 'basic', savedUsername, savedPassword);
// Now that we know that the username/password are valid store them for use later on with the same connection
await credentialProvider.saveCredential(connectionProfile.id, clusterController.password);
await ExtensionContextHelper.extensionContext.globalState.update(usernameKey, clusterController.username);
knoxPassword = clusterController.password;
try {
knoxUsername = await clusterController.getKnoxUsername(clusterController.username);
} catch (err) {
knoxUsername = clusterController.username;
console.log(`Unexpected error getting Knox username for Spark kernel: ${err}`);
}
} else {
clusterController = await getClusterController(controllerEndpoint.endpoint, 'integrated');
}
let gatewayEndpoint: bdc.IEndpointModel = endpoints?.find(ep => ep.name.toLowerCase() === KNOX_ENDPOINT_GATEWAY);
if (!gatewayEndpoint) {
Logger.log(`Querying controller for knox gateway endpoint`);
// User doesn't have permission to see the gateway endpoint from the DMV so we need to query the controller instead
const allEndpoints = (await clusterController.getEndPoints()).endPoints;
gatewayEndpoint = allEndpoints?.find(ep => ep.name.toLowerCase() === KNOX_ENDPOINT_GATEWAY);
if (!gatewayEndpoint) {
throw new Error(localize('notebook.couldNotFindKnoxGateway', "Could not find Knox gateway endpoint"));
}
}
Logger.log(`Got Knox gateway ${gatewayEndpoint.endpoint}`);
let gatewayHostAndPort = utils.getHostAndPortFromEndpoint(gatewayEndpoint.endpoint);
Logger.log(`Parsed knox host and port ${JSON.stringify(gatewayHostAndPort)}`);
connectionProfile.options[KNOX_ENDPOINT_SERVER] = gatewayHostAndPort.host;
connectionProfile.options[KNOX_ENDPOINT_PORT] = gatewayHostAndPort.port;
}
else {
throw new Error(providerNotValidError);
}
utils.setHostAndPort(':', connectionProfile);
utils.setHostAndPort(',', connectionProfile);
let server = vscode.Uri.parse(utils.getLivyUrl(connectionProfile.options[KNOX_ENDPOINT_SERVER], connectionProfile.options[KNOX_ENDPOINT_PORT])).toString();
let doNotCallChangeEndpointParams: string;
let doNotCallChangeEndpointLogMessage: string;
if (utils.isIntegratedAuth(connectionProfile)) {
doNotCallChangeEndpointParams = `%_do_not_call_change_endpoint --server=${server} --auth=Kerberos`;
doNotCallChangeEndpointLogMessage = doNotCallChangeEndpointParams;
} else {
doNotCallChangeEndpointParams = `%_do_not_call_change_endpoint --username=${knoxUsername} --server=${server} --auth=Basic_Access`;
doNotCallChangeEndpointLogMessage = doNotCallChangeEndpointParams + ` --password=${'*'.repeat(knoxPassword.length)}`;
doNotCallChangeEndpointParams += ` --password=${knoxPassword}`;
}
Logger.log(`Change endpoint command '${doNotCallChangeEndpointLogMessage}'`);
let future = this.sessionImpl.kernel.requestExecute({
code: doNotCallChangeEndpointParams
}, true);
await future.done;
}
}
private updateConfig(config: ISparkMagicConfig, creds: ICredentials, homePath: string): void {
config.kernel_python_credentials = creds;
config.kernel_scala_credentials = creds;
config.kernel_r_credentials = creds;
config.logging_config.handlers.magicsHandler.home_path = homePath;
config.ignore_ssl_errors = utils.getIgnoreSslVerificationConfigSetting();
configureConnection(connection: IConnectionProfile): Thenable<void> {
return Promise.resolve();
}
private async setEnvironmentVars(skip: boolean = false): Promise<void> {
@@ -404,76 +250,3 @@ export class JupyterSession implements nb.ISession {
this._messagesComplete.resolve();
}
}
async function getClusterController(controllerEndpoint: string, authType: bdc.AuthType, username?: string, password?: string): Promise<bdc.IClusterController | undefined> {
Logger.log(`Getting cluster controller ${controllerEndpoint}. Auth=${authType} Username=${username} password=${'*'.repeat(password?.length ?? 0)}`);
const bdcApi = <bdc.IExtension>await vscode.extensions.getExtension(bdc.constants.extensionName).activate();
const controller = bdcApi.getClusterController(
controllerEndpoint,
authType,
username,
password);
try {
Logger.log(`Fetching endpoints for ${controllerEndpoint} to test connection...`);
// We just want to test the connection - so using getEndpoints since that is available to all users (not just admin)
await controller.getEndPoints();
return controller;
} catch (err) {
// Initial username/password failed so prompt user for username password until either user
// cancels out or we successfully connect
console.log(`Error connecting to cluster controller: ${err}`);
let errorMessage = '';
const prompter = new CodeAdapter();
while (true) {
const newUsername = await prompter.promptSingle<string>(<IQuestion>{
type: QuestionTypes.input,
name: 'inputPrompt',
message: localize('promptBDCUsername', "{0}Please provide the username to connect to the BDC Controller:", errorMessage),
default: username
});
if (!username) {
console.log(`User cancelled out of username prompt for BDC Controller`);
break;
}
const newPassword = await prompter.promptSingle<string>(<IQuestion>{
type: QuestionTypes.password,
name: 'passwordPrompt',
message: localize('promptBDCPassword', "Please provide the password to connect to the BDC Controller"),
default: ''
});
if (!password) {
console.log(`User cancelled out of password prompt for BDC Controller`);
break;
}
const controller = bdcApi.getClusterController(controllerEndpoint, authType, newUsername, newPassword);
try {
// We just want to test the connection - so using getEndpoints since that is available to all users (not just admin)
await controller.getEndPoints();
return controller;
} catch (err) {
errorMessage = localize('bdcConnectError', "Error: {0}. ", err.message ?? err);
}
}
throw new Error(localize('clusterControllerConnectionRequired', "A connection to the cluster controller is required to run Spark jobs"));
}
}
interface ICredentials {
'url': string;
}
interface ISparkMagicConfig {
kernel_python_credentials: ICredentials;
kernel_scala_credentials: ICredentials;
kernel_r_credentials: ICredentials;
ignore_ssl_errors?: boolean;
logging_config: {
handlers: {
magicsHandler: {
home_path: string;
class?: string;
formatter?: string
}
}
};
}