Fix HDFS support for CU5+ BDC instances (#10577)

* Fix HDFS node auth for non-root username

* more changes

extensions/big-data-cluster/src/bdc.d.ts

@@ -0,0 +1,17 @@
+declare module 'bdc' {
+
+	export const enum constants {
+		extensionName = 'Microsoft.big-data-cluster'
+	}
+
+	export interface IExtension {
+		getClusterController(url: string, authType: AuthType, username?: string, password?: string): IClusterController;
+	}
+
+	export type AuthType = 'integrated' | 'basic';
+
+	export interface IClusterController {
+		getClusterConfig(): Promise<any>;
+		getKnoxUsername(clusterUsername: string): Promise<string>;
+	}
+}

extensions/big-data-cluster/src/bigDataCluster/constants.ts

@@ -74,7 +74,5 @@ export namespace cssStyles {
 	export const errorText = { ...text, 'color': 'red' };
 }
 
-export type AuthType = 'integrated' | 'basic';
-
 export const clusterEndpointsProperty = 'clusterEndpoints';
 export const controllerEndpointName = 'controller';

extensions/big-data-cluster/src/bigDataCluster/controller/apiGenerated.ts

@@ -859,22 +859,12 @@ export class BdcRouterApi {
      * @param connection
      * @param {*} [options] Override http request options.
      */
-    public getCluster (xRequestId: string, connection: string, options: any = {}) : Promise<{ response: http.IncomingMessage; body: any;  }> {
+    public getCluster (xRequestId?: string, connection?: string, options: any = {}) : Promise<{ response: http.IncomingMessage; body: any;  }> {
         const localVarPath = this.basePath + '/api/v1/bdc/';
         let localVarQueryParameters: any = {};
         let localVarHeaderParams: any = (<any>Object).assign({}, this.defaultHeaders);
         let localVarFormParams: any = {};
 
-        // verify required parameter 'xRequestId' is not null or undefined
-        if (xRequestId === null || xRequestId === undefined) {
-            throw new Error('Required parameter xRequestId was null or undefined when calling getCluster.');
-        }
-
-        // verify required parameter 'connection' is not null or undefined
-        if (connection === null || connection === undefined) {
-            throw new Error('Required parameter connection was null or undefined when calling getCluster.');
-        }
-
         localVarHeaderParams['X-Request-Id'] = ObjectSerializer.serialize(xRequestId, "string");
         localVarHeaderParams['Connection'] = ObjectSerializer.serialize(connection, "string");
         (<any>Object).assign(localVarHeaderParams, options.headers);

extensions/big-data-cluster/src/bigDataCluster/controller/clusterControllerApi.ts

@@ -7,13 +7,15 @@ import * as request from 'request';
 import { authenticateKerberos, getHostAndPortFromEndpoint } from '../auth';
 import { BdcRouterApi, Authentication, EndpointModel, BdcStatusModel, DefaultApi } from './apiGenerated';
 import { TokenRouterApi } from './clusterApiGenerated2';
-import { AuthType } from '../constants';
 import * as nls from 'vscode-nls';
 import { ConnectControllerDialog, ConnectControllerModel } from '../dialog/connectControllerDialog';
 import { getIgnoreSslVerificationConfigSetting } from '../utils';
+import { IClusterController, AuthType } from 'bdc';
 
 const localize = nls.loadMessageBundle();
 
+const DEFAULT_KNOX_USERNAME = 'root';
+
 class SslAuth implements Authentication {
 	constructor() { }
 
@@ -84,7 +86,7 @@ class DefaultApiWrapper extends DefaultApi {
 	}
 }
 
-export class ClusterController {
+export class ClusterController implements IClusterController {
 
 	private _authPromise: Promise<Authentication>;
 	private _url: string;
@@ -171,6 +173,42 @@ export class ClusterController {
 		}
 	}
 
+	public async getKnoxUsername(sqlLogin: string): Promise<string> {
+		try {
+			// This all is necessary because prior to CU5 BDC deployments all had the same default username for
+			// accessing the Knox gateway. But in the allowRunAsRoot setting was added and defaulted to false - so
+			// if that exists and is false then we use the username instead.
+			// Note that the SQL username may not necessarily be correct here either - but currently this is what
+			// we're requiring to run Notebooks in a BDC
+			const config = await this.getClusterConfig();
+			return config.spec?.spec?.security?.allowRunAsRoot === false ? sqlLogin : DEFAULT_KNOX_USERNAME;
+		} catch (err) {
+			console.log(`Unexpected error fetching cluster config for getKnoxUsername ${err}`);
+			// Optimistically fall back to SQL login since root shouldn't be typically used going forward
+			return sqlLogin;
+		}
+
+	}
+
+	public async getClusterConfig(promptConnect: boolean = false): Promise<any> {
+		return await this.withConnectRetry<IEndPointsResponse>(
+			this.getClusterConfigImpl,
+			promptConnect,
+			localize('bdc.error.getClusterConfig', "Error retrieving cluster config from {0}", this._url));
+	}
+
+	private async getClusterConfigImpl(self: ClusterController): Promise<any> {
+		let auth = await self._authPromise;
+		let endPointApi = new BdcApiWrapper(self._username, self._password, self._url, auth);
+		let options: any = {};
+
+		let result = await endPointApi.getCluster(options);
+		return {
+			response: result.response as IHttpResponse,
+			spec: JSON.parse(result.body.spec)
+		};
+	}
+
 	public async getEndPoints(promptConnect: boolean = false): Promise<IEndPointsResponse> {
 		return await this.withConnectRetry<IEndPointsResponse>(
 			this.getEndpointsImpl,

extensions/big-data-cluster/src/bigDataCluster/dialog/addControllerDialog.ts

@@ -7,11 +7,11 @@ import * as azdata from 'azdata';
 import * as vscode from 'vscode';
 import { ClusterController, ControllerError } from '../controller/clusterControllerApi';
 import { ControllerTreeDataProvider } from '../tree/controllerTreeDataProvider';
-import { AuthType } from '../constants';
 import { BdcDashboardOptions } from './bdcDashboardModel';
 import { ControllerNode } from '../tree/controllerTreeNode';
 import { ManageControllerCommand } from '../../commands';
 import * as loc from '../localizedConstants';
+import { AuthType } from 'bdc';
 
 function getAuthCategory(name: AuthType): azdata.CategoryValue {
 	if (name === 'basic') {

extensions/big-data-cluster/src/bigDataCluster/dialog/bdcDashboardModel.ts

@@ -8,9 +8,9 @@ import * as vscode from 'vscode';
 import { ClusterController } from '../controller/clusterControllerApi';
 import { EndpointModel, BdcStatusModel } from '../controller/apiGenerated';
 import { Endpoint, Service } from '../utils';
-import { AuthType } from '../constants';
 import { ConnectControllerDialog, ConnectControllerModel } from './connectControllerDialog';
 import { ControllerTreeDataProvider } from '../tree/controllerTreeDataProvider';
+import { AuthType } from 'bdc';
 
 export type BdcDashboardOptions = { url: string, auth: AuthType, username: string, password: string, rememberPassword: boolean };
 

extensions/big-data-cluster/src/bigDataCluster/dialog/hdfsDialogBase.ts

@@ -5,9 +5,9 @@
 
 import * as azdata from 'azdata';
 import { ClusterController, ControllerError, IEndPointsResponse } from '../controller/clusterControllerApi';
-import { AuthType } from '../constants';
 import { Deferred } from '../../common/promise';
 import * as loc from '../localizedConstants';
+import { AuthType } from 'bdc';
 
 function getAuthCategory(name: AuthType): azdata.CategoryValue {
 	if (name === 'basic') {

extensions/big-data-cluster/src/bigDataCluster/tree/controllerTreeDataProvider.ts

@@ -12,7 +12,7 @@ import { AddControllerNode } from './addControllerNode';
 import { ControllerRootNode, ControllerNode } from './controllerTreeNode';
 import { showErrorMessage } from '../utils';
 import { LoadingControllerNode } from './loadingControllerNode';
-import { AuthType } from '../constants';
+import { AuthType } from 'bdc';
 
 const localize = nls.loadMessageBundle();
 

extensions/big-data-cluster/src/bigDataCluster/tree/controllerTreeNode.ts

@@ -7,7 +7,8 @@ import * as vscode from 'vscode';
 import * as azdata from 'azdata';
 import { IControllerTreeChangeHandler } from './controllerTreeChangeHandler';
 import { TreeNode } from './treeNode';
-import { IconPathHelper, BdcItemType, IconPath, AuthType } from '../constants';
+import { IconPathHelper, BdcItemType, IconPath } from '../constants';
+import { AuthType } from 'bdc';
 
 abstract class ControllerTreeNode extends TreeNode {
 

extensions/big-data-cluster/src/extension.ts

@@ -17,6 +17,8 @@ import { MountHdfsDialogModel as MountHdfsModel, MountHdfsProperties, MountHdfsD
 import { getControllerEndpoint } from './bigDataCluster/utils';
 import * as commands from './commands';
 import { HdfsDialogCancelledError } from './bigDataCluster/dialog/hdfsDialogBase';
+import { IExtension, AuthType, IClusterController } from 'bdc';
+import { ClusterController } from './bigDataCluster/controller/clusterControllerApi';
 
 const localize = nls.loadMessageBundle();
 
@@ -24,11 +26,16 @@ const endpointNotFoundError = localize('mount.error.endpointNotFound', "Controll
 
 let throttleTimers: { [key: string]: any } = {};
 
-export function activate(extensionContext: vscode.ExtensionContext) {
+export function activate(extensionContext: vscode.ExtensionContext): IExtension {
 	IconPathHelper.setExtensionContext(extensionContext);
 	let treeDataProvider = new ControllerTreeDataProvider(extensionContext.globalState);
 	registerTreeDataProvider(treeDataProvider);
 	registerCommands(extensionContext, treeDataProvider);
+	return {
+		getClusterController(url: string, authType: AuthType, username?: string, password?: string): IClusterController {
+			return new ClusterController(url, authType, username, password);
+		}
+	};
 }
 
 export function deactivate() {

extensions/mssql/src/objectExplorerNodeProvider/connection.ts

@@ -76,6 +76,12 @@ export class SqlClusterConnection {
 		return authType && authType.toLowerCase() === constants.integratedAuth;
 	}
 
+	public updateUsername(username: string): void {
+		if (username) {
+			this._user = username;
+		}
+	}
+
 	public updatePassword(password: string): void {
 		if (password) {
 			this._password = password;

extensions/mssql/src/objectExplorerNodeProvider/objectExplorerNodeProvider.ts

@@ -117,10 +117,19 @@ export class MssqlObjectExplorerNodeProvider extends ProviderBase implements azd
 					// Only child returned when failure happens : When failed with 'Unauthorized' error, prompt for password.
 					if (children.length === 1 && this.hasExpansionError(children)) {
 						if (children[0].errorStatusCode === 401) {
-							//Prompt for password
-							let password: string = await this.promptPassword(localize('prmptPwd', "Please provide the password to connect to HDFS:"));
-							if (password && password.length > 0) {
-								session.sqlClusterConnection.updatePassword(password);
+							// First prompt for username (defaulting to existing username)
+							let username: string = await this.promptInput(localize('promptUsername', "Please provide the username to connect to HDFS:"), session.sqlClusterConnection.user);
+							// Only update the username if it's different than the original (the update functions ignore falsy values)
+							if (username === session.sqlClusterConnection.user) {
+								username = '';
+							}
+							session.sqlClusterConnection.updateUsername(username);
+
+							// And then prompt for password
+							const password: string = await this.promptPassword(localize('prmptPwd', "Please provide the password to connect to HDFS:"));
+							session.sqlClusterConnection.updatePassword(password);
+
+							if (username || password) {
 								await node.updateFileSource(session.sqlClusterConnection);
 								children = await node.getChildren(true);
 							}
@@ -141,6 +150,15 @@ export class MssqlObjectExplorerNodeProvider extends ProviderBase implements azd
 		this.expandCompleteEmitter.fire(expandResult);
 	}
 
+	private async promptInput(promptMsg: string, defaultValue: string): Promise<string> {
+		return await this.prompter.promptSingle(<IQuestion>{
+			type: QuestionTypes.input,
+			name: 'inputPrompt',
+			message: promptMsg,
+			default: defaultValue
+		}).then(confirmed => <string>confirmed);
+	}
+
 	private async promptPassword(promptMsg: string): Promise<string> {
 		return await this.prompter.promptSingle(<IQuestion>{
 			type: QuestionTypes.password,

extensions/mssql/src/sqlClusterLookUp.ts

@@ -4,6 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as azdata from 'azdata';
+import * as bdc from 'bdc';
+import * as vscode from 'vscode';
 import * as constants from './constants';
 import * as UUID from 'vscode-languageclient/lib/utils/uuid';
 import { AppContext } from './appContext';
@@ -94,8 +96,16 @@ async function createSqlClusterConnInfo(sqlConnInfo: azdata.IConnectionProfile |
 	clusterConnInfo.options[constants.knoxPortPropName] = hostAndIp.port || constants.defaultKnoxPort;
 	let authType = clusterConnInfo.options[constants.authenticationTypePropName] = sqlConnInfo.options[constants.authenticationTypePropName];
 	if (authType && authType.toLowerCase() !== constants.integratedAuth) {
-		clusterConnInfo.options[constants.userPropName] = 'root'; //should be the same user as sql master
+		clusterConnInfo.options[constants.userPropName] = sqlConnInfo.options[constants.userPropName]; //should be the same user as sql master
 		clusterConnInfo.options[constants.passwordPropName] = credentials.password;
+		try {
+			const bdcApi = <bdc.IExtension>await vscode.extensions.getExtension(bdc.constants.extensionName).activate();
+			const controllerEndpoint = endpoints.find(ep => ep.serviceName.toLowerCase() === 'controller');
+			const controller = bdcApi.getClusterController(controllerEndpoint.endpoint, 'basic', sqlConnInfo.options[constants.userPropName], credentials.password);
+			clusterConnInfo.options[constants.userPropName] = await controller.getKnoxUsername(sqlConnInfo.options[constants.userPropName]);
+		} catch (err) {
+			console.log(`Unexpected error getting Knox username for SQL Cluster connection: ${err}`);
+		}
 	}
 	clusterConnInfo = connToConnectionParam(clusterConnInfo);
 

extensions/mssql/src/typings/refs.d.ts

@@ -6,3 +6,4 @@
 /// <reference path='../../../../src/sql/azdata.d.ts'/>
 /// <reference path='../../../../src/sql/azdata.proposed.d.ts'/>
 /// <reference path='../../../../src/vs/vscode.d.ts'/>
+/// <reference path='../../../big-data-cluster/src/bdc.d.ts'/>

extensions/notebook/src/common/utils.ts

@@ -194,7 +194,7 @@ export function isEditorTitleFree(title: string): boolean {
 	return !hasTextDoc && !hasNotebookDoc;
 }
 
-export function getClusterEndpoints(serverInfo: azdata.ServerInfo): IEndpoint[] | undefined {
+export function getClusterEndpoints(serverInfo: azdata.ServerInfo): IEndpoint[] {
 	let endpoints: RawEndpoint[] = serverInfo.options['clusterEndpoints'];
 	if (!endpoints || endpoints.length === 0) { return []; }
 

extensions/notebook/src/jupyter/jupyterSessionManager.ts

@@ -16,6 +16,7 @@ const localize = nls.loadMessageBundle();
 import { JupyterKernel } from './jupyterKernel';
 import { Deferred } from '../common/promise';
 import { JupyterServerInstallation } from './jupyterServerInstallation';
+import * as bdc from 'bdc';
 
 const configBase = {
 	'kernel_python_credentials': {
@@ -56,11 +57,11 @@ const configBase = {
 const KNOX_ENDPOINT_SERVER = 'host';
 const KNOX_ENDPOINT_PORT = 'knoxport';
 const KNOX_ENDPOINT_GATEWAY = 'gateway';
+const CONTROLLER_ENDPOINT = 'controller';
 const SQL_PROVIDER = 'MSSQL';
 const USER = 'user';
 const AUTHTYPE = 'authenticationType';
 const INTEGRATED_AUTH = 'integrated';
-const DEFAULT_CLUSTER_USER_NAME = 'root';
 
 export class JupyterSessionManager implements nb.SessionManager {
 	private _ready: Deferred<void>;
@@ -272,34 +273,50 @@ export class JupyterSession implements nb.ISession {
 		await fs.writeFile(configFilePath, JSON.stringify(config));
 	}
 
-	public async configureConnection(connection: IConnectionProfile): Promise<void> {
-		if (connection && connection.providerName && this.isSparkKernel(this.sessionImpl.kernel.name)) {
-			// TODO may need to reenable a way to get the credential
-			// await this._connection.getCredential();
+	public async configureConnection(connectionProfile: IConnectionProfile): Promise<void> {
+		if (connectionProfile && connectionProfile.providerName && this.isSparkKernel(this.sessionImpl.kernel.name)) {
 			// %_do_not_call_change_endpoint is a SparkMagic command that lets users change endpoint options,
 			// such as user/profile/host name/auth type
 
 			//Update server info with bigdata endpoint - Unified Connection
-			if (connection.providerName === SQL_PROVIDER) {
-				let clusterEndpoint: utils.IEndpoint = await this.getClusterEndpoint(connection.id, KNOX_ENDPOINT_GATEWAY);
-				if (!clusterEndpoint) {
+			if (connectionProfile.providerName === SQL_PROVIDER) {
+				const endpoints = await this.getClusterEndpoints(connectionProfile.id);
+				const gatewayEndpoint: utils.IEndpoint = endpoints?.find(ep => ep.serviceName.toLowerCase() === KNOX_ENDPOINT_GATEWAY);
+				if (!gatewayEndpoint) {
 					return Promise.reject(new Error(localize('connectionNotValid', "Spark kernels require a connection to a SQL Server Big Data Cluster master instance.")));
 				}
-				let hostAndPort = utils.getHostAndPortFromEndpoint(clusterEndpoint.endpoint);
-				connection.options[KNOX_ENDPOINT_SERVER] = hostAndPort.host;
-				connection.options[KNOX_ENDPOINT_PORT] = hostAndPort.port;
-				connection.options[USER] = DEFAULT_CLUSTER_USER_NAME;
+				let gatewayHostAndPort = utils.getHostAndPortFromEndpoint(gatewayEndpoint.endpoint);
+				connectionProfile.options[KNOX_ENDPOINT_SERVER] = gatewayHostAndPort.host;
+				connectionProfile.options[KNOX_ENDPOINT_PORT] = gatewayHostAndPort.port;
+				// root is the default username for pre-CU5 instances, so while we prefer to use the connection username
+				// as a default now we'll still fall back to root if it's empty for some reason. (but the calls below should
+				// get the actual correct value regardless)
+				connectionProfile.options[USER] = connectionProfile.userName || 'root';
+
+				try {
+					const bdcApi = <bdc.IExtension>await vscode.extensions.getExtension(bdc.constants.extensionName).activate();
+					const controllerEndpoint = endpoints.find(ep => ep.serviceName.toLowerCase() === CONTROLLER_ENDPOINT);
+					const controller = bdcApi.getClusterController(controllerEndpoint.endpoint, 'basic', connectionProfile.userName, connectionProfile.password);
+					connectionProfile.options[USER] = await controller.getKnoxUsername(connectionProfile.userName);
+				} catch (err) {
+					console.log(`Unexpected error getting Knox username for Spark kernel: ${err}`);
+				}
+
 			}
 			else {
-				connection.options[KNOX_ENDPOINT_PORT] = this.getKnoxPortOrDefault(connection);
+				connectionProfile.options[KNOX_ENDPOINT_PORT] = this.getKnoxPortOrDefault(connectionProfile);
 			}
-			this.setHostAndPort(':', connection);
-			this.setHostAndPort(',', connection);
+			this.setHostAndPort(':', connectionProfile);
+			this.setHostAndPort(',', connectionProfile);
 
-			let server = vscode.Uri.parse(utils.getLivyUrl(connection.options[KNOX_ENDPOINT_SERVER], connection.options[KNOX_ENDPOINT_PORT])).toString();
-			let doNotCallChangeEndpointParams = this.isIntegratedAuth(connection) ?
-				`%_do_not_call_change_endpoint --server=${server} --auth=Kerberos`
-				: `%_do_not_call_change_endpoint --username=${connection.options[USER]} --password=${connection.options['password']} --server=${server} --auth=Basic_Access`;
+			let server = vscode.Uri.parse(utils.getLivyUrl(connectionProfile.options[KNOX_ENDPOINT_SERVER], connectionProfile.options[KNOX_ENDPOINT_PORT])).toString();
+			let doNotCallChangeEndpointParams: string;
+			if (this.isIntegratedAuth(connectionProfile)) {
+				doNotCallChangeEndpointParams = `%_do_not_call_change_endpoint --server=${server} --auth=Kerberos`;
+			} else {
+				const credentials = await connection.getCredentials(connectionProfile.id);
+				doNotCallChangeEndpointParams = `%_do_not_call_change_endpoint --username=${connectionProfile.options[USER]} --password=${credentials.password} --server=${server} --auth=Basic_Access`;
+			}
 			let future = this.sessionImpl.kernel.requestExecute({
 				code: doNotCallChangeEndpointParams
 			}, true);
@@ -348,16 +365,12 @@ export class JupyterSession implements nb.ISession {
 		return port;
 	}
 
-	private async getClusterEndpoint(profileId: string, serviceName: string): Promise<utils.IEndpoint> {
+	private async getClusterEndpoints(profileId: string): Promise<utils.IEndpoint[]> {
 		let serverInfo: ServerInfo = await connection.getServerInfo(profileId);
 		if (!serverInfo || !serverInfo.options) {
-			return undefined;
+			return [];
 		}
-		let endpoints: utils.IEndpoint[] = utils.getClusterEndpoints(serverInfo);
-		if (!endpoints || endpoints.length === 0) {
-			return undefined;
-		}
-		return endpoints.find(ep => ep.serviceName.toLowerCase() === serviceName.toLowerCase());
+		return utils.getClusterEndpoints(serverInfo);
 	}
 
 	private async setEnvironmentVars(skip: boolean = false): Promise<void> {

extensions/notebook/src/typings/refs.d.ts

@@ -6,4 +6,5 @@
 /// <reference path='../../../../src/sql/azdata.d.ts'/>
 /// <reference path='../../../../src/sql/azdata.proposed.d.ts'/>
 /// <reference path='../../../../src/vs/vscode.d.ts'/>
+/// <reference path='../../../big-data-cluster/src/bdc.d.ts'/>
 /// <reference types='@types/node'/>