From 747b8e84a89d418e380a949f16c41ac3a09930b3 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 5 Oct 2021 20:23:38 -0700 Subject: [PATCH] Add BDC spark connection logging (#17277) * Add BDC spark connection logging * fix test failures --- extensions/notebook/src/common/logger.ts | 28 +++++++++++++++++++ extensions/notebook/src/extension.ts | 3 +- .../src/jupyter/jupyterSessionManager.ts | 15 +++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 extensions/notebook/src/common/logger.ts diff --git a/extensions/notebook/src/common/logger.ts b/extensions/notebook/src/common/logger.ts new file mode 100644 index 0000000000..98774fd427 --- /dev/null +++ b/extensions/notebook/src/common/logger.ts @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as vscode from 'vscode'; + +class LoggerImpl { + private _output: vscode.OutputChannel | undefined; + + constructor() { + } + + initialize(channel: vscode.OutputChannel) { + this._output = channel; + } + + log(msg: string): void { + this._output?.appendLine(`[${new Date().toISOString()}] ${msg}`); + } + + show(): void { + this._output?.show(true); + } +} + +const Logger = new LoggerImpl(); +export default Logger; diff --git a/extensions/notebook/src/extension.ts b/extensions/notebook/src/extension.ts index a20ae2bc2c..cc40d5d47f 100644 --- a/extensions/notebook/src/extension.ts +++ b/extensions/notebook/src/extension.ts @@ -19,6 +19,7 @@ import { RemoteBookDialogModel } from './dialog/remoteBookDialogModel'; import { IconPathHelper } from './common/iconHelper'; import { ExtensionContextHelper } from './common/extensionContextHelper'; import { BookTreeItem } from './book/bookTreeItem'; +import Logger from './common/logger'; const localize = nls.loadMessageBundle(); @@ -30,7 +31,7 @@ export async function activate(extensionContext: vscode.ExtensionContext): Promi IconPathHelper.setExtensionContext(extensionContext); const appContext = new AppContext(extensionContext); - + Logger.initialize(appContext.outputChannel); // TODO: Notebook doesn't work without root setting enabled in web mode. Once we start using non-root containers, we can remove this code. const config = vscode.workspace.getConfiguration('notebook'); if (vscode.env.uiKind === vscode.UIKind.Web) { diff --git a/extensions/notebook/src/jupyter/jupyterSessionManager.ts b/extensions/notebook/src/jupyter/jupyterSessionManager.ts index c9b0a2a164..a0e80f16d9 100644 --- a/extensions/notebook/src/jupyter/jupyterSessionManager.ts +++ b/extensions/notebook/src/jupyter/jupyterSessionManager.ts @@ -21,6 +21,7 @@ import { SQL_PROVIDER, CONTROLLER_ENDPOINT, KNOX_ENDPOINT_GATEWAY, KNOX_ENDPOINT import CodeAdapter from '../prompts/adapter'; import { IQuestion, QuestionTypes } from '../prompts/question'; import { ExtensionContextHelper } from '../common/extensionContextHelper'; +import Logger from '../common/logger'; const configBase = { 'kernel_python_credentials': { @@ -280,6 +281,7 @@ export class JupyterSession implements nb.ISession { public async configureConnection(connectionProfile: IConnectionProfile): Promise { if (connectionProfile && connectionProfile.providerName && utils.isSparkKernel(this.sessionImpl.kernel.name)) { + Logger.log(`Configuring Spark connection`); // %_do_not_call_change_endpoint is a SparkMagic command that lets users change endpoint options, // such as user/profile/host name/auth type @@ -295,6 +297,7 @@ export class JupyterSession implements nb.ISession { const endpoints = utils.getClusterEndpoints(serverInfo); const controllerEndpoint = endpoints.find(ep => ep.name.toLowerCase() === CONTROLLER_ENDPOINT); + Logger.log(`Found controller endpoint ${controllerEndpoint.endpoint}`); // root is the default username for pre-CU5 instances, so while we prefer to use the connection username // as a default now we'll still fall back to root if it's empty for some reason. (but the calls below should // get the actual correct value regardless) @@ -324,6 +327,7 @@ export class JupyterSession implements nb.ISession { let gatewayEndpoint: bdc.IEndpointModel = endpoints?.find(ep => ep.name.toLowerCase() === KNOX_ENDPOINT_GATEWAY); if (!gatewayEndpoint) { + Logger.log(`Querying controller for knox gateway endpoint`); // User doesn't have permission to see the gateway endpoint from the DMV so we need to query the controller instead const allEndpoints = (await clusterController.getEndPoints()).endPoints; gatewayEndpoint = allEndpoints?.find(ep => ep.name.toLowerCase() === KNOX_ENDPOINT_GATEWAY); @@ -331,7 +335,9 @@ export class JupyterSession implements nb.ISession { throw new Error(localize('notebook.couldNotFindKnoxGateway', "Could not find Knox gateway endpoint")); } } + Logger.log(`Got Knox gateway ${gatewayEndpoint.endpoint}`); let gatewayHostAndPort = utils.getHostAndPortFromEndpoint(gatewayEndpoint.endpoint); + Logger.log(`Parsed knox host and port ${JSON.stringify(gatewayHostAndPort)}`); connectionProfile.options[KNOX_ENDPOINT_SERVER] = gatewayHostAndPort.host; connectionProfile.options[KNOX_ENDPOINT_PORT] = gatewayHostAndPort.port; } @@ -343,11 +349,16 @@ export class JupyterSession implements nb.ISession { let server = vscode.Uri.parse(utils.getLivyUrl(connectionProfile.options[KNOX_ENDPOINT_SERVER], connectionProfile.options[KNOX_ENDPOINT_PORT])).toString(); let doNotCallChangeEndpointParams: string; + let doNotCallChangeEndpointLogMessage: string; if (utils.isIntegratedAuth(connectionProfile)) { doNotCallChangeEndpointParams = `%_do_not_call_change_endpoint --server=${server} --auth=Kerberos`; + doNotCallChangeEndpointLogMessage = doNotCallChangeEndpointParams; } else { - doNotCallChangeEndpointParams = `%_do_not_call_change_endpoint --username=${knoxUsername} --password=${knoxPassword} --server=${server} --auth=Basic_Access`; + doNotCallChangeEndpointParams = `%_do_not_call_change_endpoint --username=${knoxUsername} --server=${server} --auth=Basic_Access`; + doNotCallChangeEndpointLogMessage = doNotCallChangeEndpointParams + ` --password=${'*'.repeat(knoxPassword.length)}`; + doNotCallChangeEndpointParams += ` --password=${knoxPassword}`; } + Logger.log(`Change endpoint command '${doNotCallChangeEndpointLogMessage}'`); let future = this.sessionImpl.kernel.requestExecute({ code: doNotCallChangeEndpointParams }, true); @@ -393,6 +404,7 @@ export class JupyterSession implements nb.ISession { } async function getClusterController(controllerEndpoint: string, authType: bdc.AuthType, username?: string, password?: string): Promise { + Logger.log(`Getting cluster controller ${controllerEndpoint}. Auth=${authType} Username=${username} password=${'*'.repeat(password?.length ?? 0)}`); const bdcApi = await vscode.extensions.getExtension(bdc.constants.extensionName).activate(); const controller = bdcApi.getClusterController( controllerEndpoint, @@ -400,6 +412,7 @@ async function getClusterController(controllerEndpoint: string, authType: bdc.Au username, password); try { + Logger.log(`Fetching endpoints for ${controllerEndpoint} to test connection...`); // We just want to test the connection - so using getEndpoints since that is available to all users (not just admin) await controller.getEndPoints(); return controller;