Add config for ignoring SSL errors on BDC queries (#8169)

* Add config for ignoring SSL errors on BDC queries

* Fix error handling in write stream

* Disable tslint check

* Handle promise appropriately

* PR comments

* Change defaults to true

extensions/big-data-cluster/package.json

@@ -139,7 +139,18 @@
 				"command": "bigDataClusters.command.deletemount",
 				"title": "%command.deletemount.title%"
 			}
-		]
+		],
+		"configuration": {
+			"type": "object",
+			"title": "%bdc.configuration.title%",
+			"properties": {
+				"bigDataCluster.ignoreSslVerification": {
+					"type": "boolean",
+					"default": true,
+					"description": "%bdc.ignoreSslVerification.desc%"
+				}
+			}
+		}
 	},
 	"dependencies": {
 		"kerberos": "^1.1.3",

extensions/big-data-cluster/package.nls.json

@@ -7,5 +7,7 @@
 	"command.manageController.title" : "Manage",
 	"command.mount.title" : "Mount HDFS",
 	"command.refreshmount.title" : "Refresh Mount",
-	"command.deletemount.title" : "Delete Mount"
+	"command.deletemount.title" : "Delete Mount",
+	"bdc.configuration.title" : "Big Data Cluster",
+	"bdc.ignoreSslVerification.desc" : "Ignore SSL verification errors against SQL Server Big Data Cluster endpoints such as HDFS, Spark, and Controller if true"
 }

extensions/big-data-cluster/src/bigDataCluster/constants.ts

@@ -3,8 +3,6 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-'use strict';
-
 import * as vscode from 'vscode';
 
 export enum BdcItemType {

extensions/big-data-cluster/src/bigDataCluster/controller/clusterControllerApi.ts

@@ -4,32 +4,30 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as request from 'request';
-
 import { authenticateKerberos, getHostAndPortFromEndpoint } from '../auth';
 import { BdcRouterApi, Authentication, EndpointModel, BdcStatusModel, DefaultApi } from './apiGenerated';
 import { TokenRouterApi } from './clusterApiGenerated2';
 import { AuthType } from '../constants';
 import * as nls from 'vscode-nls';
 import { ConnectControllerDialog, ConnectControllerModel } from '../dialog/connectControllerDialog';
+import { getIgnoreSslVerificationConfigSetting } from '../utils';
 
 const localize = nls.loadMessageBundle();
 
 class SslAuth implements Authentication {
-
-	constructor(private _ignoreSslVerification: boolean) {
-	}
+	constructor() { }
 
 	applyToRequest(requestOptions: request.Options): void {
 		requestOptions['agentOptions'] = {
-			rejectUnauthorized: !this._ignoreSslVerification
+			rejectUnauthorized: !getIgnoreSslVerificationConfigSetting()
 		};
 	}
 }
 
 export class KerberosAuth extends SslAuth implements Authentication {
 
-	constructor(public kerberosToken: string, ignoreSslVerification: boolean) {
-		super(ignoreSslVerification);
+	constructor(public kerberosToken: string) {
+		super();
 	}
 
 	applyToRequest(requestOptions: request.Options): void {
@@ -41,8 +39,8 @@ export class KerberosAuth extends SslAuth implements Authentication {
 	}
 }
 export class BasicAuth extends SslAuth implements Authentication {
-	constructor(public username: string, public password: string, ignoreSslVerification: boolean) {
-		super(ignoreSslVerification);
+	constructor(public username: string, public password: string) {
+		super();
 	}
 
 	applyToRequest(requestOptions: request.Options): void {
@@ -96,17 +94,16 @@ export class ClusterController {
 	constructor(url: string,
 		private _authType: AuthType,
 		private _username?: string,
-		private _password?: string,
-		ignoreSslVerification?: boolean
+		private _password?: string
 	) {
 		if (!url || (_authType === 'basic' && (!_username || !_password))) {
 			throw new Error('Missing required inputs for Cluster controller API (URL, username, password)');
 		}
 		this._url = adjustUrl(url);
 		if (this._authType === 'basic') {
-			this._authPromise = Promise.resolve(new BasicAuth(_username, _password, !!ignoreSslVerification));
+			this._authPromise = Promise.resolve(new BasicAuth(_username, _password));
 		} else {
-			this._authPromise = this.requestTokenUsingKerberos(ignoreSslVerification);
+			this._authPromise = this.requestTokenUsingKerberos();
 		}
 		this._dialog = new ConnectControllerDialog(new ConnectControllerModel(
 			{
@@ -133,8 +130,8 @@ export class ClusterController {
 		return this._password;
 	}
 
-	private async requestTokenUsingKerberos(ignoreSslVerification?: boolean): Promise<Authentication> {
-		let supportsKerberos = await this.verifyKerberosSupported(ignoreSslVerification);
+	private async requestTokenUsingKerberos(): Promise<Authentication> {
+		let supportsKerberos = await this.verifyKerberosSupported();
 		if (!supportsKerberos) {
 			throw new Error(localize('error.no.activedirectory', "This cluster does not support Windows authentication"));
 		}
@@ -145,9 +142,9 @@ export class ClusterController {
 			let host = getHostAndPortFromEndpoint(this._url).host;
 			let kerberosToken = await authenticateKerberos(host);
 			let tokenApi = new TokenRouterApi(this._url);
-			tokenApi.setDefaultAuthentication(new KerberosAuth(kerberosToken, !!ignoreSslVerification));
+			tokenApi.setDefaultAuthentication(new KerberosAuth(kerberosToken));
 			let result = await tokenApi.apiV1TokenPost();
-			let auth = new OAuthWithSsl(ignoreSslVerification);
+			let auth = new OAuthWithSsl();
 			auth.accessToken = result.body.accessToken;
 			return auth;
 		} catch (error) {
@@ -160,9 +157,9 @@ export class ClusterController {
 		}
 	}
 
-	private async verifyKerberosSupported(ignoreSslVerification: boolean): Promise<boolean> {
+	private async verifyKerberosSupported(): Promise<boolean> {
 		let tokenApi = new TokenRouterApi(this._url);
-		tokenApi.setDefaultAuthentication(new SslAuth(!!ignoreSslVerification));
+		tokenApi.setDefaultAuthentication(new SslAuth());
 		try {
 			await tokenApi.apiV1TokenPost();
 			// If we get to here, the route for endpoints doesn't require auth so state this is false

extensions/big-data-cluster/src/bigDataCluster/dialog/addControllerDialog.ts

@@ -70,7 +70,7 @@ export class AddControllerDialogModel {
 				}
 			}
 			// We pre-fetch the endpoints here to verify that the information entered is correct (the user is able to connect)
-			let controller = new ClusterController(url, auth, username, password, true);
+			let controller = new ClusterController(url, auth, username, password);
 			let response = await controller.getEndPoints();
 			if (response && response.endPoints) {
 				if (this._canceled) {

extensions/big-data-cluster/src/bigDataCluster/dialog/bdcDashboardModel.ts

@@ -31,9 +31,9 @@ export class BdcDashboardModel {
 	public onDidUpdateBdcStatus = this._onDidUpdateBdcStatus.event;
 	public onBdcError = this._onBdcError.event;
 
-	constructor(private _options: BdcDashboardOptions, private _treeDataProvider: ControllerTreeDataProvider, ignoreSslVerification = true) {
+	constructor(private _options: BdcDashboardOptions, private _treeDataProvider: ControllerTreeDataProvider) {
 		try {
-			this._clusterController = new ClusterController(_options.url, _options.auth, _options.username, _options.password, ignoreSslVerification);
+			this._clusterController = new ClusterController(_options.url, _options.auth, _options.username, _options.password);
 			// tslint:disable-next-line:no-floating-promises
 			this.refresh();
 		} catch {

extensions/big-data-cluster/src/bigDataCluster/dialog/hdfsDialogBase.ts

@@ -80,7 +80,7 @@ export abstract class HdfsDialogModelBase<T extends HdfsDialogProperties, R> {
 	}
 
 	protected createController(): ClusterController {
-		return new ClusterController(this.props.url, this.props.auth, this.props.username, this.props.password, true);
+		return new ClusterController(this.props.url, this.props.auth, this.props.username, this.props.password);
 	}
 
 	protected async createAndVerifyControllerConnection(): Promise<ClusterController> {

extensions/big-data-cluster/src/bigDataCluster/utils.ts

@@ -3,13 +3,11 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-'use strict';
-
 import * as vscode from 'vscode';
 import * as azdata from 'azdata';
 import * as nls from 'vscode-nls';
-const localize = nls.loadMessageBundle();
 import * as constants from './constants';
+const localize = nls.loadMessageBundle();
 
 export enum Endpoint {
 	gateway = 'gateway',
@@ -273,3 +271,21 @@ export function getControllerEndpoint(serverInfo: azdata.ServerInfo): string | u
 export function getBdcStatusErrorMessage(error: Error): string {
 	return localize('endpointsError', "Unexpected error retrieving BDC Endpoints: {0}", error.message);
 }
+
+export const bdcConfigSectionName = 'bigDataCluster';
+export const ignoreSslConfigName = 'ignoreSslVerification';
+
+/**
+ * Retrieves the current setting for whether to ignore SSL verification errors
+ */
+export function getIgnoreSslVerificationConfigSetting(): boolean {
+	try {
+		const config = vscode.workspace.getConfiguration(bdcConfigSectionName);
+		return config.get<boolean>(ignoreSslConfigName) || true;
+	} catch (error) {
+		console.error(`Unexpected error retrieving ${bdcConfigSectionName}.${ignoreSslConfigName} setting : ${error}`);
+	}
+	return true;
+}
+
+

extensions/big-data-cluster/src/extension.ts

@@ -75,7 +75,7 @@ function registerCommands(context: vscode.ExtensionContext, treeDataProvider: Co
 				info.rememberPassword);
 			await treeDataProvider.saveControllers();
 		}
-		const dashboard: BdcDashboard = new BdcDashboard(title, new BdcDashboardModel(info, treeDataProvider, /*ignoreSslVerification*/true));
+		const dashboard: BdcDashboard = new BdcDashboard(title, new BdcDashboardModel(info, treeDataProvider));
 		dashboard.showDashboard();
 	});
 

extensions/mssql/src/hdfs/webhdfs.ts

@@ -780,10 +780,6 @@ export class WebHDFS {
 				stream.pipe(upload);
 				stream.resume();
 			}
-			if (error && !response) {
-				// request failed, and req is not accessible in this case.
-				throw this.parseError(undefined, undefined, error);
-			}
 			if (error || this.isError(response)) {
 				emitError(req, this.parseError(response, body, error));
 			}

extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts

@@ -18,6 +18,7 @@ import { WebHDFS, HdfsError } from '../hdfs/webhdfs';
 import { PermissionStatus } from '../hdfs/aclEntry';
 import { Mount, MountStatus } from '../hdfs/mount';
 import { FileStatus, hdfsFileTypeToFileType } from '../hdfs/fileStatus';
+import { getIgnoreSslVerificationConfigSetting } from '../util/auth';
 
 const localize = nls.loadMessageBundle();
 
@@ -143,12 +144,11 @@ export class FileSourceFactory {
 		options = options && options.host ? FileSourceFactory.removePortFromHost(options) : options;
 		let requestParams: IRequestParams = options.requestParams ? options.requestParams : {};
 		if (requestParams.auth || requestParams.isKerberos) {
-			// TODO Remove handling of unsigned cert once we have real certs in our Knox service
 			let agentOptions = {
 				host: options.host,
 				port: options.port,
 				path: constants.hdfsRootPath,
-				rejectUnauthorized: false
+				rejectUnauthorized: !getIgnoreSslVerificationConfigSetting()
 			};
 			let agent = new https.Agent(agentOptions);
 			requestParams['agent'] = agent;

extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkJobSubmissionService.ts

@@ -37,8 +37,7 @@ export class SparkJobSubmissionService {
 				uri: livyUrl,
 				method: 'POST',
 				json: true,
-				// TODO, change it back after service's authentication changed.
-				rejectUnauthorized: false,
+				rejectUnauthorized: !auth.getIgnoreSslVerificationConfigSetting(),
 				body: {
 					file: submissionArgs.sparkFile,
 					proxyUser: submissionArgs.user,
@@ -114,7 +113,7 @@ export class SparkJobSubmissionService {
 				uri: livyUrl,
 				method: 'GET',
 				json: true,
-				rejectUnauthorized: false,
+				rejectUnauthorized: !auth.getIgnoreSslVerificationConfigSetting(),
 				// authentication headers
 				headers: headers
 			};

extensions/mssql/src/util/auth.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as kerberos from 'kerberos';
+import * as vscode from 'vscode';
 
 export enum AuthType {
 	Integrated = 'integrated',
@@ -17,3 +18,20 @@ export async function authenticateKerberos(hostname: string): Promise<string> {
 	let response = await client.step('');
 	return response;
 }
+
+const bdcConfigSectionName = 'bigDataCluster';
+const ignoreSslConfigName = 'ignoreSslVerification';
+
+/**
+ * Retrieves the current setting for whether to ignore SSL verification errors
+ */
+export function getIgnoreSslVerificationConfigSetting(): boolean {
+	try {
+		const config = vscode.workspace.getConfiguration(bdcConfigSectionName);
+		return config.get<boolean>(ignoreSslConfigName) || true;
+	} catch (error) {
+		console.error(`Unexpected error retrieving ${bdcConfigSectionName}.${ignoreSslConfigName} setting : ${error}`);
+	}
+	return true;
+}
+

extensions/notebook/resources/jupyter_config/sparkmagic_config.json

@@ -11,7 +11,6 @@
 		"url": "",
 		"auth": "None"
 	},
-	"ignore_ssl_errors": true,
 	"livy_session_startup_timeout_seconds": 100,
 	"logging_config": {
 		"version": 1,

extensions/notebook/src/common/utils.ts

@@ -246,3 +246,19 @@ export async function exists(path: string): Promise<boolean> {
 		return false;
 	}
 }
+
+const bdcConfigSectionName = 'bigDataCluster';
+const ignoreSslConfigName = 'ignoreSslVerification';
+
+/**
+ * Retrieves the current setting for whether to ignore SSL verification errors
+ */
+export function getIgnoreSslVerificationConfigSetting(): boolean {
+	try {
+		const config = vscode.workspace.getConfiguration(bdcConfigSectionName);
+		return config.get<boolean>(ignoreSslConfigName) || true;
+	} catch (error) {
+		console.error(`Unexpected error retrieving ${bdcConfigSectionName}.${ignoreSslConfigName} setting : ${error}`);
+	}
+	return true;
+}

extensions/notebook/src/jupyter/jupyterSessionManager.ts

@@ -26,10 +26,6 @@ const configBase = {
 	'kernel_r_credentials': {
 		'url': ''
 	},
-
-	'ignore_ssl_errors': true,
-	'livy_session_startup_timeout_seconds': 100,
-
 	'logging_config': {
 		'version': 1,
 		'formatters': {
@@ -172,7 +168,12 @@ export class JupyterSession implements nb.ISession {
 	private _messagesComplete: Deferred<void> = new Deferred<void>();
 
 	constructor(private sessionImpl: Session.ISession, skipSettingEnvironmentVars?: boolean, private _pythonEnvVarPath?: string) {
-		this.setEnvironmentVars(skipSettingEnvironmentVars);
+		this.setEnvironmentVars(skipSettingEnvironmentVars).catch(error => {
+			console.error(`Unexpected exception setting Jupyter Session variables : ${error}`);
+			// We don't want callers to hang forever waiting - it's better to continue on even if we weren't
+			// able to set environment variables
+			this._messagesComplete.resolve();
+		});
 	}
 
 	public get canChangeKernels(): boolean {
@@ -309,6 +310,7 @@ export class JupyterSession implements nb.ISession {
 		config.kernel_scala_credentials = creds;
 		config.kernel_r_credentials = creds;
 		config.logging_config.handlers.magicsHandler.home_path = homePath;
+		config.ignore_ssl_errors = utils.getIgnoreSslVerificationConfigSetting();
 	}
 
 	private getKnoxPortOrDefault(connectionProfile: IConnectionProfile): string {