Alanren/fix4001 (#4011)

* fix for 4001

* Revert "fix for 4001"

This reverts commit 91fed44f063acb44b844a206a13e4074150d8118.

* fix for issue 4001

.vscode/launch.json

@@ -152,6 +152,16 @@
 			"args": [
 				"--extensionDevelopmentPath=${workspaceRoot}/extensions/debug-auto-launch"
 			]
+		},
+		{
+			"type": "node",
+			"request": "launch",
+			"name": "Launch Smoke Test",
+			"program": "${workspaceFolder}/test/smoke/test/index.js",
+			"cwd": "${workspaceFolder}/test/smoke",
+			"env": {
+				"BUILD_ARTIFACTSTAGINGDIRECTORY": "${workspaceFolder}"
+			}
 		}
 	],
 	"compounds": [

README.md

@@ -19,7 +19,12 @@ Linux DEB | https://go.microsoft.com/fwlink/?linkid=2050157
 
 Go to our [download page](https://aka.ms/azuredatastudio) for more specific instructions.
 
-Try out the latest insiders build from `master` at https://github.com/Microsoft/azuredatastudio/releases.
+Try out the latest insiders build from `master`:
+- [Windows User Installer - **Insiders build**](https://azuredatastudio-update.azurewebsites.net/latest/win32-x64-user/insider)
+- [Windows System Installer - **Insiders build**](https://azuredatastudio-update.azurewebsites.net/latest/win32-x64/insider)
+- [Windows ZIP - **Insiders build**](https://azuredatastudio-update.azurewebsites.net/latest/win32-x64-archive/insider)
+- [macOS ZIP - **Insiders build**](https://azuredatastudio-update.azurewebsites.net/latest/darwin/insider)
+- [Linux TAR.GZ - **Insiders build**](https://azuredatastudio-update.azurewebsites.net/latest/linux-x64/insider)
 
 See the [change log](https://github.com/Microsoft/azuredatastudio/blob/master/CHANGELOG.md) for additional details of what's in this release.
 

build/gulpfile.vscode.js

@@ -65,6 +65,8 @@ const excludedExtensions = [
 	'vscode-colorize-tests',
 	'ms-vscode.node-debug',
 	'ms-vscode.node-debug2',
+	// {{SQL CARBON EDIT}}
+	'integration-tests',
 ];
 
 // {{SQL CARBON EDIT}}
@@ -76,7 +78,7 @@ const sqlBuiltInExtensions = [
 	'import',
 	'profiler'
 ];
-var azureExtensions =  [ 'azurecore'];
+var azureExtensions =  [ 'azurecore', 'mssql'];
 
 const vscodeEntryPoints = _.flatten([
 	buildfile.entrypoint('vs/workbench/workbench.main'),
@@ -278,13 +280,12 @@ function packageBuiltInExtensions() {
 	});
 }
 
-// {{SQL CARBON EDIT}}
-function packageAzureCoreTask(platform, arch) {
+function packageExtensionTask(extensionName, platform, arch) {
 	var destination = path.join(path.dirname(root), 'azuredatastudio') + (platform ? '-' + platform : '') + (arch ? '-' + arch : '');
 	if (platform === 'darwin') {
-		destination = path.join(destination, 'Azure Data Studio.app', 'Contents', 'Resources', 'app', 'extensions', 'azurecore');
+		destination = path.join(destination, 'Azure Data Studio.app', 'Contents', 'Resources', 'app', 'extensions', extensionName);
 	} else {
-		destination = path.join(destination, 'resources', 'app', 'extensions', 'azurecore');
+		destination = path.join(destination, 'resources', 'app', 'extensions', extensionName);
 	}
 
 	platform = platform || process.platform;
@@ -297,7 +298,7 @@ function packageAzureCoreTask(platform, arch) {
 				const extensionName = path.basename(extensionPath);
 				return { name: extensionName, path: extensionPath };
 			})
-			.filter(({ name }) => azureExtensions.indexOf(name) > -1);
+			.filter(({ name }) => extensionName === name);
 
 		const localExtensions = es.merge(...localExtensionDescriptions.map(extension => {
 			return ext.fromLocal(extension.path);
@@ -498,9 +499,13 @@ function packageTask(platform, arch, opts) {
 const buildRoot = path.dirname(root);
 
 // {{SQL CARBON EDIT}}
-gulp.task('vscode-win32-x64-azurecore', ['optimize-vscode'], packageAzureCoreTask('win32', 'x64'));
-gulp.task('vscode-darwin-azurecore', ['optimize-vscode'], packageAzureCoreTask('darwin'));
-gulp.task('vscode-linux-x64-azurecore', ['optimize-vscode'], packageAzureCoreTask('linux', 'x64'));
+gulp.task('vscode-win32-x64-azurecore', ['optimize-vscode'], packageExtensionTask('azurecore', 'win32', 'x64'));
+gulp.task('vscode-darwin-azurecore', ['optimize-vscode'], packageExtensionTask('azurecore', 'darwin'));
+gulp.task('vscode-linux-x64-azurecore', ['optimize-vscode'], packageExtensionTask('azurecore', 'linux', 'x64'));
+
+gulp.task('vscode-win32-x64-mssql', ['vscode-linux-x64-azurecore', 'optimize-vscode'], packageExtensionTask('mssql', 'win32', 'x64'));
+gulp.task('vscode-darwin-mssql', ['vscode-linux-x64-azurecore', 'optimize-vscode'], packageExtensionTask('mssql', 'darwin'));
+gulp.task('vscode-linux-x64-mssql', ['vscode-linux-x64-azurecore', 'optimize-vscode'], packageExtensionTask('mssql', 'linux', 'x64'));
 
 gulp.task('clean-vscode-win32-ia32', util.rimraf(path.join(buildRoot, 'azuredatastudio-win32-ia32')));
 gulp.task('clean-vscode-win32-x64', util.rimraf(path.join(buildRoot, 'azuredatastudio-win32-x64')));
@@ -510,10 +515,10 @@ gulp.task('clean-vscode-linux-x64', util.rimraf(path.join(buildRoot, 'azuredatas
 gulp.task('clean-vscode-linux-arm', util.rimraf(path.join(buildRoot, 'azuredatastudio-linux-arm')));
 
 gulp.task('vscode-win32-ia32', ['optimize-vscode', 'clean-vscode-win32-ia32'], packageTask('win32', 'ia32'));
-gulp.task('vscode-win32-x64',  ['vscode-win32-x64-azurecore', 'optimize-vscode', 'clean-vscode-win32-x64'], packageTask('win32', 'x64'));
-gulp.task('vscode-darwin', ['vscode-darwin-azurecore', 'optimize-vscode', 'clean-vscode-darwin'], packageTask('darwin'));
+gulp.task('vscode-win32-x64',  ['vscode-win32-x64-azurecore', 'vscode-win32-x64-mssql', 'optimize-vscode', 'clean-vscode-win32-x64'], packageTask('win32', 'x64'));
+gulp.task('vscode-darwin', ['vscode-darwin-azurecore', 'vscode-darwin-mssql', 'optimize-vscode', 'clean-vscode-darwin'], packageTask('darwin'));
 gulp.task('vscode-linux-ia32', ['optimize-vscode', 'clean-vscode-linux-ia32'], packageTask('linux', 'ia32'));
-gulp.task('vscode-linux-x64', ['vscode-linux-x64-azurecore', 'optimize-vscode', 'clean-vscode-linux-x64'], packageTask('linux', 'x64'));
+gulp.task('vscode-linux-x64', ['vscode-linux-x64-azurecore', 'vscode-linux-x64-mssql', 'optimize-vscode', 'clean-vscode-linux-x64'], packageTask('linux', 'x64'));
 gulp.task('vscode-linux-arm', ['optimize-vscode', 'clean-vscode-linux-arm'], packageTask('linux', 'arm'));
 
 gulp.task('vscode-win32-ia32-min', ['minify-vscode', 'clean-vscode-win32-ia32'], packageTask('win32', 'ia32', { minified: true }));

extensions/agent/package.json

@@ -9,7 +9,7 @@
   "icon": "images/sqlserver.png",
   "aiKey": "AIF-5574968e-856d-40d2-af67-c89a14e76412",
   "engines": {
-      "vscode": "0.10.x"
+    "vscode": "0.10.x"
   },
   "activationEvents": [
     "*"
@@ -45,7 +45,7 @@
     "vscode-nls": "^3.2.1"
   },
   "devDependencies": {
-		"mocha-junit-reporter": "^1.17.0",
-		"mocha-multi-reporters": "^1.1.7"
-	}
+    "mocha-junit-reporter": "^1.17.0",
+    "mocha-multi-reporters": "^1.1.7"
+  }
 }

extensions/agent/src/data/jobData.ts

@@ -26,6 +26,7 @@ export class JobData implements IAgentDialogData {
 	private _operators: string[];
 	private _defaultOwner: string;
 	private _jobCompletionActionConditions: sqlops.CategoryValue[];
+	private _jobCategoryIdsMap: sqlops.AgentJobCategory[];
 
 	public dialogMode: AgentDialogMode = AgentDialogMode.CREATE;
 	public name: string;
@@ -46,6 +47,7 @@ export class JobData implements IAgentDialogData {
 	public alerts: sqlops.AgentAlertInfo[];
 	public jobId: string;
 	public startStepId: number;
+	public categoryType: number;
 
 	constructor(
 		ownerUri: string,
@@ -66,6 +68,8 @@ export class JobData implements IAgentDialogData {
 			this.alerts = jobInfo.alerts;
 			this.jobId = jobInfo.jobId;
 			this.startStepId = jobInfo.startStepId;
+			this.categoryId = jobInfo.categoryId;
+			this.categoryType = jobInfo.categoryType;
 		}
 	}
 
@@ -73,6 +77,10 @@ export class JobData implements IAgentDialogData {
 		return this._jobCategories;
 	}
 
+	public get jobCategoryIdsMap(): sqlops.AgentJobCategory[] {
+		return this._jobCategoryIdsMap;
+	}
+
 	public get operators(): string[] {
 		return this._operators;
 	}
@@ -96,7 +104,7 @@ export class JobData implements IAgentDialogData {
 			this._jobCategories = jobDefaults.categories.map((cat) => {
 				return cat.name;
 			});
-
+			this._jobCategoryIdsMap = jobDefaults.categories;
 			this._defaultOwner = jobDefaults.owner;
 
 			this._operators = ['', this._defaultOwner];
@@ -164,8 +172,8 @@ export class JobData implements IAgentDialogData {
 			hasSchedule: false,
 			hasStep: false,
 			runnable: true,
-			categoryId: 0,
-			categoryType: 1, // LocalJob, hard-coding the value, corresponds to the target tab in SSMS
+			categoryId: this.categoryId,
+			categoryType: this.categoryType,
 			lastRun: '',
 			nextRun: '',
 			jobId: this.jobId,

extensions/agent/src/dialogs/jobDialog.ts

@@ -94,6 +94,9 @@ export class JobDialog extends AgentDialog<JobData>  {
 	private editStepButton: sqlops.ButtonComponent;
 	private deleteStepButton: sqlops.ButtonComponent;
 
+	// Schedule tab controls
+	private removeScheduleButton: sqlops.ButtonComponent;
+
 	// Notifications tab controls
 	private notificationsTabTopLabel: sqlops.TextComponent;
 	private emailCheckBox: sqlops.CheckBoxComponent;
@@ -302,6 +305,7 @@ export class JobDialog extends AgentDialog<JobData>  {
 				this.stepsTable.data = this.convertStepsToData(this.steps);
 				this.steps[previousRow].id = previousStepId;
 				this.steps[rowNumber].id = currentStepId;
+				this.stepsTable.selectedRows = [previousRow];
 			});
 
 			this.moveStepDownButton.onDidClick(() => {
@@ -316,6 +320,7 @@ export class JobDialog extends AgentDialog<JobData>  {
 				this.stepsTable.data = this.convertStepsToData(this.steps);
 				this.steps[nextRow].id = nextStepId;
 				this.steps[rowNumber].id = currentStepId;
+				this.stepsTable.selectedRows = [nextRow];
 			});
 
 			this.editStepButton.onDidClick(() => {
@@ -346,20 +351,30 @@ export class JobDialog extends AgentDialog<JobData>  {
 				if (this.stepsTable.selectedRows.length === 1) {
 					let rowNumber = this.stepsTable.selectedRows[0];
 					AgentUtils.getAgentService().then((agentService) => {
-						let steps = this.model.jobSteps ? this.model.jobSteps : [];
-						let stepData = this.model.jobSteps[rowNumber];
-						agentService.deleteJobStep(this.ownerUri, stepData).then((result) => {
-							if (result && result.success) {
-								delete steps[rowNumber];
-								let data = this.convertStepsToData(steps);
-								this.stepsTable.data = data;
-								this.startStepDropdownValues = [];
-								this.steps.forEach((step) => {
-									this.startStepDropdownValues.push({ displayName: step.id + ': ' + step.stepName, name: step.id.toString() });
-								});
-								this.startStepDropdown.values = this.startStepDropdownValues;
-							}
-						});
+						let stepData = this.steps[rowNumber];
+						if (stepData.jobId) {
+							agentService.deleteJobStep(this.ownerUri, stepData).then((result) => {
+								if (result && result.success) {
+									this.steps.splice(rowNumber, 1);
+									let data = this.convertStepsToData(this.steps);
+									this.stepsTable.data = data;
+									this.startStepDropdownValues = [];
+									this.steps.forEach((step) => {
+										this.startStepDropdownValues.push({ displayName: step.id + ': ' + step.stepName, name: step.id.toString() });
+									});
+									this.startStepDropdown.values = this.startStepDropdownValues;
+								}
+							});
+						} else {
+							this.steps.splice(rowNumber, 1);
+							let data = this.convertStepsToData(this.steps);
+							this.stepsTable.data = data;
+							this.startStepDropdownValues = [];
+							this.steps.forEach((step) => {
+								this.startStepDropdownValues.push({ displayName: step.id + ': ' + step.stepName, name: step.id.toString() });
+							});
+							this.startStepDropdown.values = this.startStepDropdownValues;
+						}
 					});
 				}
 			});
@@ -468,7 +483,11 @@ export class JobDialog extends AgentDialog<JobData>  {
 				label: this.PickScheduleButtonString,
 				width: 80
 			}).component();
-			this.pickScheduleButton.onDidClick((e)=>{
+			this.removeScheduleButton = view.modelBuilder.button().withProperties({
+				label: 'Remove schedule',
+				width: 100
+			}).component();
+			this.pickScheduleButton.onDidClick(()=>{
 				let pickScheduleDialog = new PickScheduleDialog(this.model.ownerUri, this.model.name);
 				pickScheduleDialog.onSuccess((dialogModel) => {
 					let selectedSchedule = dialogModel.selectedSchedule;
@@ -483,12 +502,23 @@ export class JobDialog extends AgentDialog<JobData>  {
 				});
 				pickScheduleDialog.showDialog();
 			});
-
+			this.removeScheduleButton.onDidClick(() => {
+                if (this.schedulesTable.selectedRows.length === 1) {
+                    let selectedRow = this.schedulesTable.selectedRows[0];
+					let selectedScheduleName = this.schedulesTable.data[selectedRow][1];
+					for (let i = 0; i < this.schedules.length; i++) {
+						if (this.schedules[i].name === selectedScheduleName) {
+							this.schedules.splice(i, 1);
+						}
+					}
+					this.populateScheduleTable();
+				}
+			});
 			let formModel = view.modelBuilder.formContainer()
 				.withFormItems([{
 					component: this.schedulesTable,
 					title: this.SchedulesTopLabelString,
-					actions: [this.pickScheduleButton]
+					actions: [this.pickScheduleButton, this.removeScheduleButton]
 				}]).withLayout({ width: '100%' }).component();
 
 			await view.initializeModel(formModel);
@@ -499,10 +529,9 @@ export class JobDialog extends AgentDialog<JobData>  {
 
 	private populateScheduleTable() {
 		let data = this.convertSchedulesToData(this.schedules);
-		if (data.length > 0) {
-			this.schedulesTable.data = data;
-			this.schedulesTable.height = 750;
-		}
+		this.schedulesTable.data = data;
+		this.schedulesTable.height = 750;
+
 	}
 
 	private initializeNotificationsTab() {
@@ -674,5 +703,6 @@ export class JobDialog extends AgentDialog<JobData>  {
 			this.model.alerts = [];
 		}
 		this.model.alerts = this.alerts;
+		this.model.categoryId = +this.model.jobCategoryIdsMap.find(cat => cat.name === this.model.category).id;
 	}
 }

extensions/agent/src/dialogs/jobStepDialog.ts

@@ -29,11 +29,10 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 	private readonly AdvancedTabText: string = localize('jobStepDialog.advanced', 'Advanced');
 	private readonly OpenCommandText: string = localize('jobStepDialog.open', 'Open...');
 	private readonly ParseCommandText: string = localize('jobStepDialog.parse','Parse');
-	private readonly NextButtonText: string = localize('jobStepDialog.next', 'Next');
-	private readonly PreviousButtonText: string = localize('jobStepDialog.previous','Previous');
 	private readonly SuccessfulParseText: string = localize('jobStepDialog.successParse', 'The command was successfully parsed.');
 	private readonly FailureParseText: string = localize('jobStepDialog.failParse', 'The command failed.');
 	private readonly BlankStepNameErrorText: string = localize('jobStepDialog.blankStepName', 'The step name cannot be left blank');
+	private readonly ProcessExitCodeText: string = localize('jobStepDialog.processExitCode', 'Process exit code of a successful command:');
 
 	// General Control Titles
 	private readonly StepNameLabelString: string = localize('jobStepDialog.stepNameLabel', 'Step Name');
@@ -62,6 +61,8 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 
 	// Dropdown options
 	private readonly TSQLScript: string = localize('jobStepDialog.TSQL', 'Transact-SQL script (T-SQL)');
+	private readonly Powershell: string = localize('jobStepDialog.powershell', 'PowerShell');
+	private readonly CmdExec: string = localize('jobStepDialog.CmdExec', 'Operating system (CmdExec)');
 	private readonly AgentServiceAccount: string = localize('jobStepDialog.agentServiceAccount', 'SQL Server Agent Service Account');
 	private readonly NextStep: string = localize('jobStepDialog.nextStep', 'Go to the next step');
 	private readonly QuitJobReportingSuccess: string = localize('jobStepDialog.quitJobSuccess', 'Quit the job reporting success');
@@ -88,6 +89,7 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 	private outputFileNameBox: sqlops.InputBoxComponent;
 	private fileBrowserNameBox: sqlops.InputBoxComponent;
 	private userInputBox: sqlops.InputBoxComponent;
+	private processExitCodeBox: sqlops.InputBoxComponent;
 
 	// Dropdowns
 	private typeDropdown: sqlops.DropDownComponent;
@@ -100,8 +102,6 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 	// Buttons
 	private openButton: sqlops.ButtonComponent;
 	private parseButton: sqlops.ButtonComponent;
-	private nextButton: sqlops.ButtonComponent;
-	private previousButton: sqlops.ButtonComponent;
 	private outputFileBrowserButton: sqlops.ButtonComponent;
 
 	// Checkbox
@@ -179,18 +179,6 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 				inputType: 'text'
 			})
 			.component();
-		this.nextButton = view.modelBuilder.button()
-			.withProperties({
-				label: this.NextButtonText,
-				enabled: false,
-				width: '80px'
-			}).component();
-		this.previousButton = view.modelBuilder.button()
-			.withProperties({
-				label: this.PreviousButtonText,
-				enabled: false,
-				width: '80px'
-			}).component();
 	}
 
 	private createGeneralTab(databases: string[], queryProvider: sqlops.QueryProvider) {
@@ -208,7 +196,7 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 			this.typeDropdown = view.modelBuilder.dropDown()
 				.withProperties({
 					value: this.TSQLScript,
-					values: [this.TSQLScript]
+					values: [this.TSQLScript, this.CmdExec, this.Powershell]
 				})
 				.component();
 			this.runAsDropdown = view.modelBuilder.dropDown()
@@ -218,33 +206,20 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 				})
 				.component();
 			this.runAsDropdown.enabled = false;
-			this.typeDropdown.onValueChanged((type) => {
-				if (type.selected !== this.TSQLScript) {
-					this.runAsDropdown.value = this.AgentServiceAccount;
-					this.runAsDropdown.values = [this.runAsDropdown.value];
-				} else {
-					this.runAsDropdown.value = '';
-					this.runAsDropdown.values = [''];
-				}
-			});
 			this.databaseDropdown = view.modelBuilder.dropDown()
 				.withProperties({
 					value: databases[0],
 					values: databases
 				}).component();
 
+			this.processExitCodeBox = view.modelBuilder.inputBox()
+				.withProperties({
+				}).component();
+			this.processExitCodeBox.enabled = false;
+
 			// create the commands section
 			this.createCommands(view, queryProvider);
 
-			let buttonContainer = view.modelBuilder.flexContainer()
-			.withLayout({
-				flexFlow: 'row',
-				justifyContent: 'space-between',
-				width: 420
-			}).withItems([this.openButton, this.parseButton, this.previousButton, this.nextButton], {
-				flex: '1 1 50%'
-			}).component();
-
 			let formModel = view.modelBuilder.formContainer()
 				.withFormItems([{
 					component: this.nameTextBox,
@@ -258,14 +233,52 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 				}, {
 					component: this.databaseDropdown,
 					title: this.DatabaseLabelString
+				}, {
+					component: this.processExitCodeBox,
+					title: this.ProcessExitCodeText
 				}, {
 					component: this.commandTextBox,
 					title: this.CommandLabelString,
-					actions: [buttonContainer]
+					actions: [this.openButton, this.parseButton]
 				}], {
 						horizontal: false,
 						componentWidth: 420
 					}).component();
+			this.typeDropdown.onValueChanged((type) => {
+				switch (type.selected) {
+					case(this.TSQLScript):
+						this.runAsDropdown.value = '';
+						this.runAsDropdown.values = [''];
+						this.runAsDropdown.enabled = false;
+						this.databaseDropdown.enabled = true;
+						this.databaseDropdown.values = databases;
+						this.databaseDropdown.value = databases[0];
+						this.processExitCodeBox.value = '';
+						this.processExitCodeBox.enabled = false;
+						break;
+					case(this.Powershell):
+						this.runAsDropdown.value = this.AgentServiceAccount;
+						this.runAsDropdown.values = [this.runAsDropdown.value];
+						this.runAsDropdown.enabled = true;
+						this.databaseDropdown.enabled = false;
+						this.databaseDropdown.values = [''];
+						this.databaseDropdown.value = '';
+						this.processExitCodeBox.value = '';
+						this.processExitCodeBox.enabled = false;
+						break;
+					case(this.CmdExec):
+						this.databaseDropdown.enabled = false;
+						this.databaseDropdown.values = [''];
+						this.databaseDropdown.value = '';
+						this.runAsDropdown.value = this.AgentServiceAccount;
+						this.runAsDropdown.values = [this.runAsDropdown.value];
+						this.runAsDropdown.enabled = true;
+						this.processExitCodeBox.enabled = true;
+						this.processExitCodeBox.value = '0';
+						break;
+
+				}
+			});
 			let formWrapper = view.modelBuilder.loadingComponent().withItem(formModel).component();
 			formWrapper.loading = false;
 			await view.initializeModel(formWrapper);
@@ -524,6 +537,7 @@ export class JobStepDialog extends AgentDialog<JobStepData> {
 		this.model.outputFileName = this.outputFileNameBox.value;
 		this.model.appendToLogFile = this.appendToExistingFileCheckbox.checked;
 		this.model.command = this.commandTextBox.value ? this.commandTextBox.value : '';
+		this.model.commandExecutionSuccessCode = this.processExitCodeBox.value ? +this.processExitCodeBox.value : 0;
 	}
 
 	public async initializeDialog() {

extensions/agent/yarn.lock

@@ -25,11 +25,11 @@ debug@^2.2.0:
     ms "2.0.0"
 
 debug@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
-  integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==
+  version "3.2.6"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b"
+  integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==
   dependencies:
-    ms "2.0.0"
+    ms "^2.1.1"
 
 is-buffer@~1.1.1:
   version "1.1.6"
@@ -37,9 +37,9 @@ is-buffer@~1.1.1:
   integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
 
 lodash@^4.16.4:
-  version "4.17.10"
-  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7"
-  integrity sha512-UejweD1pDoXu+AD825lWwp4ZGtSwgnpZxb3JDViD7StjQz+Nb/6l093lx4OQ0foGWNRoc19mWy7BzL+UAK2iVg==
+  version "4.17.11"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
+  integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
 
 md5@^2.1.0:
   version "2.2.1"
@@ -63,9 +63,9 @@ mkdirp@~0.5.1:
     minimist "0.0.8"
 
 mocha-junit-reporter@^1.17.0:
-  version "1.17.0"
-  resolved "https://registry.yarnpkg.com/mocha-junit-reporter/-/mocha-junit-reporter-1.17.0.tgz#2e5149ed40fc5d2e3ca71e42db5ab1fec9c6d85c"
-  integrity sha1-LlFJ7UD8XS48px5C21qx/snG2Fw=
+  version "1.18.0"
+  resolved "https://registry.yarnpkg.com/mocha-junit-reporter/-/mocha-junit-reporter-1.18.0.tgz#9209a3fba30025ae3ae5e6bfe7f9c5bc3c2e8ee2"
+  integrity sha512-y3XuqKa2+HRYtg0wYyhW/XsLm2Ps+pqf9HaTAt7+MVUAKFJaNAHOrNseTZo9KCxjfIbxUWwckP5qCDDPUmjSWA==
   dependencies:
     debug "^2.2.0"
     md5 "^2.1.0"
@@ -86,6 +86,11 @@ ms@2.0.0:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
   integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=
 
+ms@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a"
+  integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==
+
 strip-ansi@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f"
@@ -94,9 +99,9 @@ strip-ansi@^4.0.0:
     ansi-regex "^3.0.0"
 
 vscode-nls@^3.2.1:
-  version "3.2.2"
-  resolved "https://registry.yarnpkg.com/vscode-nls/-/vscode-nls-3.2.2.tgz#3817eca5b985c2393de325197cf4e15eb2aa5350"
-  integrity sha512-/Ur1+tgazwd51+ncRyoy0UIu4dvMdVXS9XMUULQlZIBoNGEwOhwEx9x+hHWoUjldMrOQ32t2CGKo0u6D4R6/hg==
+  version "3.2.5"
+  resolved "https://registry.yarnpkg.com/vscode-nls/-/vscode-nls-3.2.5.tgz#25520c1955108036dec607c85e00a522f247f1a4"
+  integrity sha512-ITtoh3V4AkWXMmp3TB97vsMaHRgHhsSFPsUdzlueSL+dRZbSNTZeOmdQv60kjCV306ghPxhDeoNUEm3+EZMuyw==
 
 xml@^1.0.0:
   version "1.0.1"

extensions/azurecore/package.json

@@ -122,6 +122,14 @@
 				}
 			]
 		},
+		"dataExplorer": {
+			"azureResource": [
+				{
+					"id": "azureResourceExplorer",
+					"name": "%azure.resource.explorer.title%"
+				}
+			]
+		},
 		"menus": {
 			"view/title": [
 				{
@@ -166,7 +174,7 @@
 		"@types/node": "^8.0.24",
 		"mocha": "^5.2.0",
 		"should": "^13.2.1",
-		"vscode": "^1.1.26",
-		"typemoq": "^2.1.0"
+		"typemoq": "^2.1.0",
+		"vscode": "^1.1.26"
 	}
 }

extensions/azurecore/src/azureResource/tree/accountTreeNode.ts

@@ -90,12 +90,9 @@ export class AzureResourceAccountTreeNode extends AzureResourceContainerTreeNode
 			}
 		} catch (error) {
 			if (error instanceof AzureResourceCredentialError) {
-				this.appContext.apiWrapper.showErrorMessage(error.message);
-
 				this.appContext.apiWrapper.executeCommand('azure.resource.signin');
-			} else {
-				return [AzureResourceMessageTreeNode.create(AzureResourceErrorMessageUtil.getErrorMessage(error), this)];
 			}
+			return [AzureResourceMessageTreeNode.create(AzureResourceErrorMessageUtil.getErrorMessage(error), this)];
 		}
 	}
 

extensions/azurecore/src/azureResource/tree/treeProvider.ts

@@ -32,7 +32,7 @@ export class AzureResourceTreeProvider implements TreeDataProvider<TreeNode>, IA
 			return element.getChildren(true);
 		}
 
-		if (!this.isSystemInitialized) {
+		if (!this.isSystemInitialized && !this._loadingTimer) {
 			this._loadingTimer = setInterval(async () => {
 				try {
 					// Call sqlops.accounts.getAllAccounts() to determine whether the system has been initialized.

extensions/azurecore/src/azureResource/utils.ts

@@ -40,4 +40,57 @@ export function generateGuid(): string {
     let clockSequenceHi: string = hexValues[8 + (Math.random() * 4) | 0];
     return oct.substr(0, 8) + '-' + oct.substr(9, 4) + '-4' + oct.substr(13, 3) + '-' + clockSequenceHi + oct.substr(16, 3) + '-' + oct.substr(19, 12);
     /* tslint:enable:no-bitwise */
+}
+
+export function equals(one: any, other: any): boolean {
+    if (one === other) {
+        return true;
+    }
+    if (one === null || one === undefined || other === null || other === undefined) {
+        return false;
+    }
+    if (typeof one !== typeof other) {
+        return false;
+    }
+    if (typeof one !== 'object') {
+        return false;
+    }
+    if ((Array.isArray(one)) !== (Array.isArray(other))) {
+        return false;
+    }
+
+    let i: number;
+    let key: string;
+
+    if (Array.isArray(one)) {
+        if (one.length !== other.length) {
+            return false;
+        }
+        for (i = 0; i < one.length; i++) {
+            if (!equals(one[i], other[i])) {
+                return false;
+            }
+        }
+    } else {
+        const oneKeys: string[] = [];
+
+        for (key in one) {
+            oneKeys.push(key);
+        }
+        oneKeys.sort();
+        const otherKeys: string[] = [];
+        for (key in other) {
+            otherKeys.push(key);
+        }
+        otherKeys.sort();
+        if (!equals(oneKeys, otherKeys)) {
+            return false;
+        }
+        for (i = 0; i < oneKeys.length; i++) {
+            if (!equals(one[oneKeys[i]], other[oneKeys[i]])) {
+                return false;
+            }
+        }
+    }
+    return true;
 }

extensions/azurecore/src/controllers/azureResourceController.ts

@@ -25,6 +25,7 @@ import { AzureResourceTenantService } from '../azureResource/services/tenantServ
 
 import { registerAzureResourceDatabaseServerCommands } from '../azureResource/providers/databaseServer/commands';
 import { registerAzureResourceDatabaseCommands } from '../azureResource/providers/database/commands';
+import { equals } from '../azureResource/utils';
 
 export default class AzureResourceController extends ControllerBase {
 	public activate(): Promise<boolean> {
@@ -37,7 +38,16 @@ export default class AzureResourceController extends ControllerBase {
 		const azureResourceTree = new AzureResourceTreeProvider(this.appContext);
 		this.extensionContext.subscriptions.push(this.apiWrapper.registerTreeDataProvider('azureResourceExplorer', azureResourceTree));
 
-		this.appContext.getService<IAzureResourceAccountService>(AzureResourceServiceNames.accountService).onDidChangeAccounts((e: DidChangeAccountsParams) => { azureResourceTree.notifyNodeChanged(undefined); });
+		let previousAccounts = undefined;
+		this.appContext.getService<IAzureResourceAccountService>(AzureResourceServiceNames.accountService).onDidChangeAccounts((e: DidChangeAccountsParams) => {
+			// the onDidChangeAccounts event will trigger in many cases where the accounts didn't actually change
+			// the notifyNodeChanged event triggers a refresh which triggers a getChildren which can trigger this callback
+			// this below check short-circuits the infinite callback loop
+			if (!equals(e.accounts, previousAccounts)) {
+				azureResourceTree.notifyNodeChanged(undefined);
+			}
+			previousAccounts = e.accounts;
+		});
 
 		registerAzureResourceCommands(this.appContext, azureResourceTree);
 

extensions/azurecore/tsconfig.json

@@ -14,7 +14,7 @@
 		"emitDecoratorMetadata": true,
 		"experimentalDecorators": true,
 		"moduleResolution": "node",
-		"declaration": true
+		"declaration": false
 	},
 	"exclude": [
 		"node_modules"

extensions/import/README.md

@@ -15,9 +15,8 @@ This experience is currently in its initial preview. Please report issues and fe
  * This wizard requires an active connection to a SQL Server instance to start.
  * This wizard only works on .txt and .csv files.
 
- ### How do I start the Import Flat File wizard?
- * The main entry point for the wizard is to right click a database in the Object Explorer, and click **Import wizard**.
- * If a user is connected to a SQL Server instance, the user can also press **Ctrl**+**I** to start the wizard.
+ ## How do I start the Flat File Import wizard?
+ * In Azure Data Studio, press **Ctrl**+**I** to start the wizard.
 
  ### Why would I use the Import Flat File wizard?
 This wizard was created to improve the current import experience leveraging an intelligent framework known as Program Synthesis using Examples ([PROSE](https://microsoft.github.io/prose/)). For a user without specialized domain knowledge, importing data can often be a complex, error prone, and tedious task. This wizard streamlines the import process as simple as selecting an input file and unique table name, and the PROSE framework handles the rest.

extensions/import/package.json

@@ -67,7 +67,7 @@
     }
   },
   "dependencies": {
-    "dataprotocol-client": "github:Microsoft/sqlops-dataprotocolclient#0.2.10",
+    "dataprotocol-client": "github:Microsoft/sqlops-dataprotocolclient#0.2.15",
     "opener": "^1.4.3",
     "service-downloader": "github:anthonydresser/service-downloader#0.1.5",
     "vscode-extension-telemetry": "0.0.18",

extensions/import/src/wizard/api/models.ts

@@ -45,4 +45,6 @@ export interface DacFxDataModel extends BaseDataModel {
 	filePath: string;
 	version: string;
 	upgradeExisting: boolean;
+	scriptFilePath: string;
+	generateScriptAndDeploy: boolean;
 }

extensions/import/src/wizard/dataTierApplicationWizard.ts

@@ -8,6 +8,8 @@ import * as nls from 'vscode-nls';
 import * as sqlops from 'sqlops';
 import { SelectOperationPage } from './pages/selectOperationpage';
 import { DeployConfigPage } from './pages/deployConfigPage';
+import { DeployPlanPage } from './pages/deployPlanPage';
+import { DeployActionPage } from './pages/deployActionPage';
 import { DacFxSummaryPage } from './pages/dacFxSummaryPage';
 import { ExportConfigPage } from './pages/exportConfigPage';
 import { ExtractConfigPage } from './pages/extractConfigPage';
@@ -30,7 +32,40 @@ export enum Operation {
 	deploy,
 	extract,
 	import,
-	export
+	export,
+	generateDeployScript
+}
+
+export enum DeployOperationPath {
+	selectOperation,
+	deployOptions,
+	deployPlan,
+	deployAction,
+	summary
+}
+
+export enum DeployNewOperationPath {
+	selectOperation,
+	deployOptions,
+	summary
+}
+
+export enum ExtractOperationPath {
+	selectOperation,
+	options,
+	summary
+}
+
+export enum ImportOperationPath {
+	selectOperation,
+	options,
+	summary
+}
+
+export enum ExportOperationPath {
+	selectOperation,
+	options,
+	summary
 }
 
 export class DataTierApplicationWizard {
@@ -60,6 +95,8 @@ export class DataTierApplicationWizard {
 		this.wizard = sqlops.window.modelviewdialog.createWizard('Data-tier Application Wizard');
 		let selectOperationWizardPage = sqlops.window.modelviewdialog.createWizardPage(localize('dacFx.selectOperationPageName', 'Select an Operation'));
 		let deployConfigWizardPage = sqlops.window.modelviewdialog.createWizardPage(localize('dacFx.deployConfigPageName', 'Select Deploy Dacpac Settings'));
+		let deployPlanWizardPage = sqlops.window.modelviewdialog.createWizardPage(localize('dacFx.deployPlanPage', 'Review the deploy plan'));
+		let deployActionWizardPage = sqlops.window.modelviewdialog.createWizardPage(localize('dacFx.deployActionPageName', 'Select Action'));
 		let summaryWizardPage = sqlops.window.modelviewdialog.createWizardPage(localize('dacFx.summaryPageName', 'Summary'));
 		let extractConfigWizardPage = sqlops.window.modelviewdialog.createWizardPage(localize('dacFx.extractConfigPageName', 'Select Extract Dacpac Settings'));
 		let importConfigWizardPage = sqlops.window.modelviewdialog.createWizardPage(localize('dacFx.importConfigPageName', 'Select Import Bacpac Settings'));
@@ -67,6 +104,8 @@ export class DataTierApplicationWizard {
 
 		this.pages.set('selectOperation', new Page(selectOperationWizardPage));
 		this.pages.set('deployConfig', new Page(deployConfigWizardPage));
+		this.pages.set('deployPlan', new Page(deployPlanWizardPage));
+		this.pages.set('deployAction', new Page(deployActionWizardPage));
 		this.pages.set('extractConfig', new Page(extractConfigWizardPage));
 		this.pages.set('importConfig', new Page(importConfigWizardPage));
 		this.pages.set('exportConfig', new Page(exportConfigWizardPage));
@@ -87,6 +126,18 @@ export class DataTierApplicationWizard {
 			await deployConfigDacFxPage.start();
 		});
 
+		deployPlanWizardPage.registerContent(async (view) => {
+			let deployPlanDacFxPage = new DeployPlanPage(this, deployPlanWizardPage, this.model, view);
+			this.pages.get('deployPlan').dacFxPage = deployPlanDacFxPage;
+			await deployPlanDacFxPage.start();
+		});
+
+		deployActionWizardPage.registerContent(async (view) => {
+			let deployActionDacFxPage = new DeployActionPage(this, deployActionWizardPage, this.model, view);
+			this.pages.get('deployAction').dacFxPage = deployActionDacFxPage;
+			await deployActionDacFxPage.start();
+		});
+
 		extractConfigWizardPage.registerContent(async (view) => {
 			let extractConfigDacFxPage = new ExtractConfigPage(this, extractConfigWizardPage, this.model, view);
 			this.pages.get('extractConfig').dacFxPage = extractConfigDacFxPage;
@@ -113,39 +164,27 @@ export class DataTierApplicationWizard {
 
 		this.wizard.onPageChanged(async (event) => {
 			let idx = event.newPage;
-			let page: Page;
-
-			if (idx === 1) {
-				switch (this.selectedOperation) {
-					case Operation.deploy: {
-						page = this.pages.get('deployConfig');
-						break;
-					}
-					case Operation.extract: {
-						page = this.pages.get('extractConfig');
-						break;
-					}
-					case Operation.import: {
-						page = this.pages.get('importConfig');
-						break;
-					}
-					case Operation.export: {
-						page = this.pages.get('exportConfig');
-						break;
-					}
-				}
-			} else if (idx === 2) {
-				page = this.pages.get('summary');
-			}
+			let page = this.getPage(idx);
 
 			if (page !== undefined) {
 				page.dacFxPage.setupNavigationValidator();
 				page.dacFxPage.onPageEnter();
 			}
+
+			//do onPageLeave for summary page so that GenerateScript button only shows up if upgrading database
+			let idxLast = event.lastPage;
+
+			if (this.isSummaryPage(idxLast)) {
+				let lastPage = this.pages.get('summary');
+				if (lastPage) {
+					lastPage.dacFxPage.onPageLeave();
+				}
+			}
 		});
 
-		this.wizard.pages = [selectOperationWizardPage, deployConfigWizardPage, summaryWizardPage];
+		this.wizard.pages = [selectOperationWizardPage, deployConfigWizardPage, deployPlanWizardPage, deployActionWizardPage, summaryWizardPage];
 		this.wizard.generateScriptButton.hidden = true;
+		this.wizard.generateScriptButton.onClick(async () => await this.generateDeployScript());
 		this.wizard.doneButton.onClick(async () => await this.executeOperation());
 
 		this.wizard.open();
@@ -177,6 +216,15 @@ export class DataTierApplicationWizard {
 				this.selectedOperation = Operation.export;
 				break;
 			}
+			case Operation.generateDeployScript: {
+				this.wizard.doneButton.label = localize('dacFx.generateScriptButton', 'Generate Script');
+				this.selectedOperation = Operation.generateDeployScript;
+				break;
+			}
+		}
+
+		if (operation !== Operation.deploy && operation !== Operation.generateDeployScript) {
+			this.model.upgradeExisting = false;
 		}
 	}
 
@@ -198,6 +246,10 @@ export class DataTierApplicationWizard {
 				await this.export();
 				break;
 			}
+			case Operation.generateDeployScript: {
+				await this.generateDeployScript();
+				break;
+			}
 		}
 	}
 
@@ -245,6 +297,81 @@ export class DataTierApplicationWizard {
 		}
 	}
 
+	private async generateDeployScript() {
+		if (!this.model.scriptFilePath) {
+			return;
+		}
+
+		let service = await DataTierApplicationWizard.getService(this.model.server.providerName);
+		let ownerUri = await sqlops.connection.getUriForConnection(this.model.server.connectionId);
+		this.wizard.message = {
+			text: localize('dacfx.scriptGeneratingMessage', 'You can view the status of script generation in the Task History once the wizard is closed'),
+			level: sqlops.window.modelviewdialog.MessageLevel.Information,
+			description: ''
+		};
+
+		let result = await service.generateDeployScript(this.model.filePath, this.model.database, this.model.scriptFilePath, ownerUri, sqlops.TaskExecutionMode.execute);
+		if (!result || !result.success) {
+			vscode.window.showErrorMessage(
+				localize('alertData.deployErrorMessage', "Deploy failed '{0}'", result.errorMessage ? result.errorMessage : 'Unknown'));
+		}
+	}
+
+	private getPage(idx: number): Page {
+		let page: Page;
+
+		if (idx === 1) {
+			switch (this.selectedOperation) {
+				case Operation.deploy: {
+					page = this.pages.get('deployConfig');
+					break;
+				}
+				case Operation.extract: {
+					page = this.pages.get('extractConfig');
+					break;
+				}
+				case Operation.import: {
+					page = this.pages.get('importConfig');
+					break;
+				}
+				case Operation.export: {
+					page = this.pages.get('exportConfig');
+					break;
+				}
+			}
+		} else if (this.isSummaryPage(idx)) {
+			page = this.pages.get('summary');
+		} else if ((this.selectedOperation === Operation.deploy || this.selectedOperation === Operation.generateDeployScript) && idx === DeployOperationPath.deployPlan) {
+			page = this.pages.get('deployPlan');
+		} else if ((this.selectedOperation === Operation.deploy || this.selectedOperation === Operation.generateDeployScript) && idx === DeployOperationPath.deployAction) {
+			page = this.pages.get('deployAction');
+		}
+
+		return page;
+	}
+
+	private isSummaryPage(idx: number): boolean {
+		return this.selectedOperation === Operation.import && idx === ImportOperationPath.summary
+			|| this.selectedOperation === Operation.export && idx === ExportOperationPath.summary
+			|| this.selectedOperation === Operation.extract && idx === ExtractOperationPath.summary
+			|| this.selectedOperation === Operation.deploy && !this.model.upgradeExisting && idx === DeployNewOperationPath.summary
+			|| (this.selectedOperation === Operation.deploy || this.selectedOperation === Operation.generateDeployScript) && idx === DeployOperationPath.summary;
+	}
+
+	public async generateDeployPlan(): Promise<string> {
+		let service = await DataTierApplicationWizard.getService(this.model.server.providerName);
+		let ownerUri = await sqlops.connection.getUriForConnection(this.model.server.connectionId);
+
+		let result = await service.generateDeployPlan(this.model.filePath, this.model.database, ownerUri, sqlops.TaskExecutionMode.execute);
+
+		if (!result || !result.success) {
+			vscode.window.showErrorMessage(
+				localize('alertData.deployPlanErrorMessage', "Generating deploy plan failed '{0}'", result.errorMessage ? result.errorMessage : 'Unknown'));
+		}
+
+		return result.report;
+	}
+
 	private static async getService(providerName: string): Promise<sqlops.DacFxServicesProvider> {
 		let service = sqlops.dataprotocol.getProvider<sqlops.DacFxServicesProvider>(providerName, sqlops.DataProviderType.DacFxServicesProvider);
 		return service;

extensions/import/src/wizard/pages/dacFxSummaryPage.ts

@@ -49,6 +49,14 @@ export class DacFxSummaryPage extends BasePage {
 	async onPageEnter(): Promise<boolean> {
 		this.populateTable();
 		this.loader.loading = false;
+		if (this.model.upgradeExisting && this.model.generateScriptAndDeploy) {
+			this.instance.wizard.generateScriptButton.hidden = false;
+		}
+		return true;
+	}
+
+	async onPageLeave(): Promise<boolean> {
+		this.instance.wizard.generateScriptButton.hidden = true;
 		return true;
 	}
 
@@ -68,6 +76,10 @@ export class DacFxSummaryPage extends BasePage {
 		let sourceServer = localize('dacfx.sourceServerName', 'Source Server');
 		let sourceDatabase = localize('dacfx.sourceDatabaseName', 'Source Database');
 		let fileLocation = localize('dacfx.fileLocation', 'File Location');
+		let scriptLocation = localize('dacfx.scriptLocation', 'Deployment Script Location');
+		let action = localize('dacfx.action', 'Action');
+		let deploy = localize('dacfx.deploy', 'Deploy');
+		let generateScript = localize('dacfx.generateScript', 'Generate Deployment Script');
 
 		switch (this.instance.selectedOperation) {
 			case Operation.deploy: {
@@ -75,6 +87,13 @@ export class DacFxSummaryPage extends BasePage {
 					[targetServer, this.model.serverName],
 					[fileLocation, this.model.filePath],
 					[targetDatabase, this.model.database]];
+				if (this.model.generateScriptAndDeploy) {
+					data[3] = [scriptLocation, this.model.scriptFilePath];
+					data[4] = [action, generateScript + ', ' + deploy];
+				}
+				else {
+					data[3] = [action, deploy];
+				}
 				break;
 			}
 			case Operation.extract: {
@@ -99,12 +118,29 @@ export class DacFxSummaryPage extends BasePage {
 					[fileLocation, this.model.filePath]];
 				break;
 			}
+			case Operation.generateDeployScript: {
+				data = [
+					[targetServer, this.model.serverName],
+					[fileLocation, this.model.filePath],
+					[targetDatabase, this.model.database],
+					[scriptLocation, this.model.scriptFilePath],
+					[action, generateScript]];
+				break;
+			}
 		}
 
 		this.table.updateProperties({
 			data: data,
-			columns: ['Setting', 'Value'],
-			width: 600,
+			columns: [
+				{
+					value: localize('dacfx.settingColumn', 'Setting'),
+					cssClass: 'align-with-header'
+				},
+				{
+					value: localize('dacfx.valueColumn', 'Value'),
+					cssClass: 'align-with-header'
+				}],
+			width: 700,
 			height: 200
 		});
 	}

extensions/import/src/wizard/pages/deployActionPage.ts

@@ -0,0 +1,181 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+import * as sqlops from 'sqlops';
+import * as nls from 'vscode-nls';
+import * as vscode from 'vscode';
+import * as path from 'path';
+import * as os from 'os';
+import { DacFxDataModel } from '../api/models';
+import { DataTierApplicationWizard, Operation } from '../dataTierApplicationWizard';
+import { DacFxConfigPage } from '../api/dacFxConfigPage';
+
+const localize = nls.loadMessageBundle();
+
+export class DeployActionPage extends DacFxConfigPage {
+
+	protected readonly wizardPage: sqlops.window.modelviewdialog.WizardPage;
+	protected readonly instance: DataTierApplicationWizard;
+	protected readonly model: DacFxDataModel;
+	protected readonly view: sqlops.ModelView;
+	private deployRadioButton: sqlops.RadioButtonComponent;
+	private deployScriptRadioButton: sqlops.RadioButtonComponent;
+	private scriptRadioButton: sqlops.RadioButtonComponent;
+	private form: sqlops.FormContainer;
+
+	public constructor(instance: DataTierApplicationWizard, wizardPage: sqlops.window.modelviewdialog.WizardPage, model: DacFxDataModel, view: sqlops.ModelView) {
+		super(instance, wizardPage, model, view);
+	}
+
+	async start(): Promise<boolean> {
+		let deployComponent = await this.createDeployRadioButton();
+		let deployScriptComponent = await this.createDeployScriptRadioButton();
+		let scriptComponent = await this.createScriptRadioButton();
+		let fileBrowserComponent = await this.createFileBrowser();
+
+		this.form = this.view.modelBuilder.formContainer()
+			.withFormItems(
+				[
+					deployComponent,
+					scriptComponent,
+					deployScriptComponent,
+					fileBrowserComponent
+				]).component();
+		await this.view.initializeModel(this.form);
+
+		//default have the first radio button checked
+		this.deployRadioButton.checked = true;
+		this.toggleFileBrowser(false);
+
+		return true;
+	}
+
+	async onPageEnter(): Promise<boolean> {
+		// generate script file path in case the database changed since last time the page was entered
+		this.setDefaultScriptFilePath();
+		return true;
+	}
+
+	private async createDeployRadioButton(): Promise<sqlops.FormComponent> {
+		this.deployRadioButton = this.view.modelBuilder.radioButton()
+			.withProperties({
+				name: 'selectedDeployAction',
+				label: localize('dacFx.deployRadioButtonLabel', 'Deploy'),
+			}).component();
+
+		this.deployRadioButton.onDidClick(() => {
+			this.model.generateScriptAndDeploy = false;
+			this.instance.setDoneButton(Operation.deploy);
+			this.toggleFileBrowser(false);
+		});
+
+		return {
+			component: this.deployRadioButton,
+			title: ''
+		};
+	}
+
+	private async createDeployScriptRadioButton(): Promise<sqlops.FormComponent> {
+		this.deployScriptRadioButton = this.view.modelBuilder.radioButton()
+			.withProperties({
+				name: 'selectedDeployAction',
+				label: localize('dacFx.deployScriptRadioButtonLabel', 'Generate Deployment Script and Deploy'),
+			}).component();
+
+		this.deployScriptRadioButton.onDidClick(() => {
+			this.model.generateScriptAndDeploy = true;
+			this.instance.setDoneButton(Operation.deploy);
+			this.toggleFileBrowser(true);
+		});
+
+		return {
+			component: this.deployScriptRadioButton,
+			title: ''
+		};
+	}
+
+	private async createScriptRadioButton(): Promise<sqlops.FormComponent> {
+		this.scriptRadioButton = this.view.modelBuilder.radioButton()
+			.withProperties({
+				name: 'selectedDeployAction',
+				label: localize('dacFx.scriptRadioButtonLabel', 'Generate Deployment Script'),
+			}).component();
+
+		this.scriptRadioButton.onDidClick(() => {
+			this.model.generateScriptAndDeploy = false;
+			this.toggleFileBrowser(true);
+
+			//change button text and operation
+			this.instance.setDoneButton(Operation.generateDeployScript);
+		});
+
+		return {
+			component: this.scriptRadioButton,
+			title: ''
+		};
+	}
+
+	private async createFileBrowser(): Promise<sqlops.FormComponentGroup> {
+		this.createFileBrowserParts();
+
+		//default filepath
+		this.setDefaultScriptFilePath();
+		this.fileButton.onDidClick(async (click) => {
+			let fileUri = await vscode.window.showSaveDialog(
+				{
+					defaultUri: vscode.Uri.file(this.fileTextBox.value),
+					saveLabel: localize('dacfxDeployScript.saveFile', 'Save'),
+					filters: {
+						'SQL Files': ['sql'],
+					}
+				}
+			);
+
+			if (!fileUri) {
+				return;
+			}
+
+			this.fileTextBox.value = fileUri.fsPath;
+			this.model.scriptFilePath = fileUri.fsPath;
+		});
+
+		this.fileTextBox.onTextChanged(async () => {
+			this.model.scriptFilePath = this.fileTextBox.value;
+		});
+
+		return {
+			title: '',
+			components: [
+				{
+					title: localize('dacfx.generatedScriptLocation', 'Deployment Script Location'),
+					component: this.fileTextBox,
+					layout: {
+						horizontal: true,
+						componentWidth: 400
+					},
+					actions: [this.fileButton]
+				},],
+		};
+	}
+
+	private toggleFileBrowser(enable: boolean): void {
+		this.fileTextBox.enabled = enable;
+		this.fileButton.enabled = enable;
+	}
+
+	private setDefaultScriptFilePath(): void {
+		let now = new Date();
+		let datetime = now.getFullYear() + '-' + (now.getMonth() + 1) + '-' + now.getDate() + '-' + now.getHours() + '-' + now.getMinutes();
+		this.fileTextBox.value = path.join(os.homedir(), this.model.database + '_UpgradeDACScript_' + datetime + '.sql');
+		this.model.scriptFilePath = this.fileTextBox.value;
+	}
+
+	public setupNavigationValidator() {
+		this.instance.registerNavigationValidator(() => {
+			return true;
+		});
+	}
+}

extensions/import/src/wizard/pages/deployConfigPage.ts

@@ -10,7 +10,7 @@ import * as vscode from 'vscode';
 import * as path from 'path';
 import * as os from 'os';
 import { DacFxDataModel } from '../api/models';
-import { DataTierApplicationWizard } from '../dataTierApplicationWizard';
+import { DataTierApplicationWizard, DeployOperationPath, Operation } from '../dataTierApplicationWizard';
 import { DacFxConfigPage } from '../api/dacFxConfigPage';
 
 const localize = nls.loadMessageBundle();
@@ -122,6 +122,12 @@ export class DeployConfigPage extends DacFxConfigPage {
 			this.formBuilder.removeFormItem(this.databaseComponent);
 			this.formBuilder.addFormItem(this.databaseDropdownComponent, { horizontal: true, componentWidth: 400 });
 			this.model.database = (<sqlops.CategoryValue>this.databaseDropdown.value).name;
+
+			// add deploy plan and generate script pages
+			let deployPlanPage = this.instance.pages.get('deployPlan');
+			this.instance.wizard.addPage(deployPlanPage.wizardPage, DeployOperationPath.deployPlan);
+			let deployActionPage = this.instance.pages.get('deployAction');
+			this.instance.wizard.addPage(deployActionPage.wizardPage, DeployOperationPath.deployAction);
 		});
 
 		newRadioButton.onDidClick(() => {
@@ -129,6 +135,11 @@ export class DeployConfigPage extends DacFxConfigPage {
 			this.formBuilder.removeFormItem(this.databaseDropdownComponent);
 			this.formBuilder.addFormItem(this.databaseComponent, { horizontal: true, componentWidth: 400 });
 			this.model.database = this.databaseTextBox.value;
+			this.instance.setDoneButton(Operation.deploy);
+
+			// remove deploy plan and generate script pages
+			this.instance.wizard.removePage(DeployOperationPath.deployAction);
+			this.instance.wizard.removePage(DeployOperationPath.deployPlan);
 		});
 
 		//Initialize with upgrade existing true

extensions/import/src/wizard/pages/deployPlanPage.ts

@@ -0,0 +1,296 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+import * as sqlops from 'sqlops';
+import * as nls from 'vscode-nls';
+import * as parser from 'htmlparser2';
+import { DacFxDataModel } from '../api/models';
+import { DataTierApplicationWizard } from '../dataTierApplicationWizard';
+import { DacFxConfigPage } from '../api/dacFxConfigPage';
+
+const localize = nls.loadMessageBundle();
+
+export enum deployPlanXml {
+	AlertElement = 'Alert',
+	OperationElement = 'Operation',
+	ItemElement = 'Item',
+	NameAttribute = 'Name',
+	ValueAttribute = 'Value',
+	TypeAttribute = 'Type',
+	IdAttribute = 'Id',
+	DataIssueAttribute = 'DataIssue'
+}
+
+export class TableObject {
+	operation: string;
+	object: string;
+	type: string;
+	dataloss: boolean;
+}
+
+export class DeployPlanResult {
+	columnData: Array<Array<string>>;
+	dataLossAlerts: Set<string>;
+}
+
+export class DeployPlanPage extends DacFxConfigPage {
+	protected readonly wizardPage: sqlops.window.modelviewdialog.WizardPage;
+	protected readonly instance: DataTierApplicationWizard;
+	protected readonly model: DacFxDataModel;
+	protected readonly view: sqlops.ModelView;
+	private formBuilder: sqlops.FormBuilder;
+	private form: sqlops.FormContainer;
+	private table: sqlops.TableComponent;
+	private loader: sqlops.LoadingComponent;
+	private dataLossCheckbox: sqlops.CheckBoxComponent;
+	private dataLossText: sqlops.TextComponent;
+	private dataLossComponentGroup: sqlops.FormComponentGroup;
+	private noDataLossTextComponent: sqlops.FormComponent;
+
+	public constructor(instance: DataTierApplicationWizard, wizardPage: sqlops.window.modelviewdialog.WizardPage, model: DacFxDataModel, view: sqlops.ModelView) {
+		super(instance, wizardPage, model, view);
+	}
+
+	async start(): Promise<boolean> {
+		this.table = this.view.modelBuilder.table().component();
+		this.loader = this.view.modelBuilder.loadingComponent().withItem(this.table).component();
+		this.dataLossComponentGroup = await this.createDataLossComponents();
+		this.noDataLossTextComponent = await this.createNoDataLossText();
+
+		this.formBuilder = this.view.modelBuilder.formContainer()
+			.withFormItems(
+				[
+					{
+						component: this.loader,
+						title: ''
+					},
+					this.dataLossComponentGroup
+				], {
+					horizontal: true,
+				});
+		this.form = this.formBuilder.component();
+		await this.view.initializeModel(this.form);
+
+		return true;
+	}
+
+	async onPageEnter(): Promise<boolean> {
+		// reset checkbox settings
+		this.formBuilder.addFormItem(this.dataLossComponentGroup, { horizontal: true, componentWidth: 400 });
+		this.dataLossCheckbox.checked = false;
+		this.dataLossCheckbox.enabled = false;
+		this.formBuilder.removeFormItem(this.noDataLossTextComponent);
+
+		this.loader.loading = true;
+		this.table.data = [];
+		await this.populateTable();
+		this.loader.loading = false;
+		return true;
+	}
+
+	private async populateTable() {
+		let report = await this.instance.generateDeployPlan();
+		let result = this.parseXml(report);
+
+		this.table.updateProperties({
+			data: this.getColumnData(result),
+			columns: this.getTableColumns(result.dataLossAlerts.size > 0),
+			width: 875,
+			height: 300
+		});
+
+		if (result.dataLossAlerts.size > 0) {
+			// update message to list how many operations could result in data loss
+			this.dataLossText.updateProperties({
+				value: localize('dacfx.dataLossTextWithCount', '{0} of the deploy actions listed may result in data loss. Please ensure you have a backup or snapshot available in the event of an issue with the deployment.', result.dataLossAlerts.size)
+			});
+			this.dataLossCheckbox.enabled = true;
+		} else {
+			// check checkbox to enable Next button and remove checkbox because there won't be any possible data loss
+			this.dataLossCheckbox.checked = true;
+			this.formBuilder.removeFormItem(this.dataLossComponentGroup);
+			this.formBuilder.addFormItem(this.noDataLossTextComponent, { componentWidth: 300, horizontal: true });
+		}
+	}
+
+	private async createDataLossCheckbox(): Promise<sqlops.FormComponent> {
+		this.dataLossCheckbox = this.view.modelBuilder.checkBox()
+			.withValidation(component => component.checked === true)
+			.withProperties({
+				label: localize('dacFx.dataLossCheckbox', 'Proceed despite possible data loss'),
+			}).component();
+
+		return {
+			component: this.dataLossCheckbox,
+			title: '',
+			required: true
+		};
+	}
+
+	private async createNoDataLossText(): Promise<sqlops.FormComponent> {
+		let noDataLossText = this.view.modelBuilder.text()
+			.withProperties({
+				value: localize('dacfx.noDataLossText', 'No data loss will occur from the listed deploy actions.')
+			}).component();
+
+		return {
+			title: '',
+			component: noDataLossText
+		};
+	}
+
+	private async createDataLossComponents(): Promise<sqlops.FormComponentGroup> {
+		let dataLossComponent = await this.createDataLossCheckbox();
+		this.dataLossText = this.view.modelBuilder.text()
+			.withProperties({
+				value: localize('dacfx.dataLossText', 'The deploy actions may result in data loss. Please ensure you have a backup or snapshot available in the event of an issue with the deployment.')
+			}).component();
+
+		return {
+			title: '',
+			components: [
+				{
+					component: this.dataLossText,
+					layout: {
+						componentWidth: 400,
+						horizontal: true
+					},
+					title: ''
+				},
+				dataLossComponent
+			]
+		};
+	}
+
+	private getColumnData(result: DeployPlanResult): Array<Array<string>> {
+		// remove data loss column data if there aren't any alerts
+		let columnData = result.columnData;
+		if (result.dataLossAlerts.size === 0) {
+			columnData.forEach(entry => {
+				entry.shift();
+			});
+		}
+		return columnData;
+	}
+
+	private getTableColumns(dataloss: boolean): sqlops.TableColumn[] {
+		let columns: sqlops.TableColumn[] = [
+			{
+				value: localize('dacfx.operationColumn', 'Operation'),
+				width: 75,
+				cssClass: 'align-with-header',
+				toolTip: localize('dacfx.operationTooltip', 'Operation(Create, Alter, Delete) that will occur during deployment')
+			},
+			{
+				value: localize('dacfx.typeColumn', 'Type'),
+				width: 100,
+				cssClass: 'align-with-header',
+				toolTip: localize('dacfx.typeTooltip', 'Type of object that will be affected by deployment')
+			},
+			{
+				value: localize('dacfx.objectColumn', 'Object'),
+				width: 300,
+				cssClass: 'align-with-header',
+				toolTip: localize('dacfx.objecTooltip', 'Name of object that will be affected by deployment')
+			}];
+
+		if (dataloss) {
+			columns.unshift(
+				{
+					value: localize('dacfx.dataLossColumn', 'Data Loss'),
+					width: 50,
+					cssClass: 'center-align',
+					toolTip: localize('dacfx.dataLossTooltip', 'Operations that may result in data loss are marked with a warning sign')
+				});
+		}
+		return columns;
+	}
+
+	private parseXml(report: string): DeployPlanResult {
+		let operations = new Array<TableObject>();
+		let dataLossAlerts = new Set<string>();
+
+		let currentOperation = '';
+		let dataIssueAlert = false;
+		let currentReportSection: deployPlanXml;
+		let currentTableObj: TableObject;
+		let p = new parser.Parser({
+			onopentagname(name) {
+				if (name === deployPlanXml.AlertElement) {
+					currentReportSection = deployPlanXml.AlertElement;
+				} else if (name === deployPlanXml.OperationElement) {
+					currentReportSection = deployPlanXml.OperationElement;
+				} else if (name === deployPlanXml.ItemElement) {
+					currentTableObj = new TableObject();
+				}
+			},
+			onattribute: function (name, value) {
+				if (currentReportSection === deployPlanXml.AlertElement) {
+					switch (name) {
+						case deployPlanXml.NameAttribute: {
+							// only care about showing data loss alerts
+							if (value === deployPlanXml.DataIssueAttribute) {
+								dataIssueAlert = true;
+							}
+							break;
+						}
+						case deployPlanXml.IdAttribute: {
+							if (dataIssueAlert) {
+								dataLossAlerts.add(value);
+							}
+							break;
+						}
+					}
+				} else if (currentReportSection === deployPlanXml.OperationElement) {
+					switch (name) {
+						case deployPlanXml.NameAttribute: {
+							currentOperation = value;
+							break;
+						}
+						case deployPlanXml.ValueAttribute: {
+							currentTableObj.object = value;
+							break;
+						}
+						case deployPlanXml.TypeAttribute: {
+							currentTableObj.type = value;
+							break;
+						}
+						case deployPlanXml.IdAttribute: {
+							if (dataLossAlerts.has(value)) {
+								currentTableObj.dataloss = true;
+							}
+							break;
+						}
+					}
+				}
+			},
+			onclosetag: function (name) {
+				if (name === deployPlanXml.ItemElement) {
+					currentTableObj.operation = currentOperation;
+					operations.push(currentTableObj);
+				}
+			}
+		}, { xmlMode: true, decodeEntities: true });
+		p.parseChunk(report);
+
+		let data = new Array<Array<string>>();
+		operations.forEach(operation => {
+			let isDataLoss = operation.dataloss ? '⚠️' : '';
+			data.push([isDataLoss, operation.operation, operation.type, operation.object]);
+		});
+
+		return {
+			columnData: data,
+			dataLossAlerts: dataLossAlerts
+		};
+	}
+
+	public setupNavigationValidator() {
+		this.instance.registerNavigationValidator(() => {
+			return true;
+		});
+	}
+}

extensions/import/src/wizard/pages/selectOperationpage.ts

@@ -7,7 +7,7 @@
 import * as sqlops from 'sqlops';
 import * as nls from 'vscode-nls';
 import { DacFxDataModel } from '../api/models';
-import { DataTierApplicationWizard, Operation } from '../dataTierApplicationWizard';
+import { DataTierApplicationWizard, Operation, DeployOperationPath, ExtractOperationPath, ImportOperationPath, ExportOperationPath } from '../dataTierApplicationWizard';
 import { BasePage } from '../api/basePage';
 
 const localize = nls.loadMessageBundle();
@@ -58,11 +58,6 @@ export class SelectOperationPage extends BasePage {
 	}
 
 	async onPageEnter(): Promise<boolean> {
-		let numPages = this.instance.wizard.pages.length;
-		for (let i = numPages - 1; i > 2; --i) {
-			await this.instance.wizard.removePage(i);
-		}
-
 		return true;
 	}
 
@@ -74,12 +69,16 @@ export class SelectOperationPage extends BasePage {
 			}).component();
 
 		this.deployRadioButton.onDidClick(() => {
-			// remove the previous page
-			this.instance.wizard.removePage(1);
+			this.removePages();
 
-			// add deploy page
-			let page = this.instance.pages.get('deployConfig');
-			this.instance.wizard.addPage(page.wizardPage, 1);
+			//add deploy pages
+			let configPage = this.instance.pages.get('deployConfig');
+			this.instance.wizard.addPage(configPage.wizardPage, DeployOperationPath.deployOptions);
+			let deployPlanPage = this.instance.pages.get('deployPlan');
+			this.instance.wizard.addPage(deployPlanPage.wizardPage, DeployOperationPath.deployPlan);
+			let actionPage = this.instance.pages.get('deployAction');
+			this.instance.wizard.addPage(actionPage.wizardPage, DeployOperationPath.deployAction);
+			this.addSummaryPage(DeployOperationPath.summary);
 
 			// change button text and operation
 			this.instance.setDoneButton(Operation.deploy);
@@ -99,12 +98,12 @@ export class SelectOperationPage extends BasePage {
 			}).component();
 
 		this.extractRadioButton.onDidClick(() => {
-			// remove the previous pages
-			this.instance.wizard.removePage(1);
+			this.removePages();
 
 			// add the extract page
 			let page = this.instance.pages.get('extractConfig');
-			this.instance.wizard.addPage(page.wizardPage, 1);
+			this.instance.wizard.addPage(page.wizardPage, ExtractOperationPath.options);
+			this.addSummaryPage(ExtractOperationPath.summary);
 
 			// change button text and operation
 			this.instance.setDoneButton(Operation.extract);
@@ -124,12 +123,12 @@ export class SelectOperationPage extends BasePage {
 			}).component();
 
 		this.importRadioButton.onDidClick(() => {
-			// remove the  previous page
-			this.instance.wizard.removePage(1);
+			this.removePages();
 
 			// add the import page
 			let page = this.instance.pages.get('importConfig');
-			this.instance.wizard.addPage(page.wizardPage, 1);
+			this.instance.wizard.addPage(page.wizardPage, ImportOperationPath.options);
+			this.addSummaryPage(ImportOperationPath.summary);
 
 			// change button text and operation
 			this.instance.setDoneButton(Operation.import);
@@ -149,12 +148,12 @@ export class SelectOperationPage extends BasePage {
 			}).component();
 
 		this.exportRadioButton.onDidClick(() => {
-			// remove the 2 previous pages
-			this.instance.wizard.removePage(1);
+			this.removePages();
 
 			// add the export pages
 			let page = this.instance.pages.get('exportConfig');
-			this.instance.wizard.addPage(page.wizardPage, 1);
+			this.instance.wizard.addPage(page.wizardPage, ExportOperationPath.options);
+			this.addSummaryPage(ExportOperationPath.summary);
 
 			// change button text and operation
 			this.instance.setDoneButton(Operation.export);
@@ -166,6 +165,18 @@ export class SelectOperationPage extends BasePage {
 		};
 	}
 
+	private removePages() {
+		let numPages = this.instance.wizard.pages.length;
+		for (let i = numPages - 1; i > 0; --i) {
+			this.instance.wizard.removePage(i);
+		}
+	}
+
+	private addSummaryPage(index: number) {
+		let summaryPage = this.instance.pages.get('summary');
+		this.instance.wizard.addPage(summaryPage.wizardPage, index);
+	}
+
 	public setupNavigationValidator() {
 		this.instance.registerNavigationValidator(() => {
 			return true;

extensions/import/tsconfig.json

@@ -11,7 +11,7 @@
         "emitDecoratorMetadata": true,
         "experimentalDecorators": true,
         "moduleResolution": "node",
-        "declaration": true
+        "declaration": false
     },
     "exclude": [
         "node_modules"

extensions/import/yarn.lock

@@ -75,9 +75,9 @@ core-util-is@~1.0.0:
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
-"dataprotocol-client@github:Microsoft/sqlops-dataprotocolclient#0.2.10":
-  version "0.2.10"
-  resolved "https://codeload.github.com/Microsoft/sqlops-dataprotocolclient/tar.gz/4de3f7caf0eba54159911b977ddb4f5d7c0a9ca8"
+"dataprotocol-client@github:Microsoft/sqlops-dataprotocolclient#0.2.15":
+  version "0.2.15"
+  resolved "https://codeload.github.com/Microsoft/sqlops-dataprotocolclient/tar.gz/a2cd2db109de882f0959f7b6421c86afa585f460"
   dependencies:
     vscode-languageclient "3.5.1"
 

extensions/integration-tests/.vscode/launch.json

@@ -0,0 +1,17 @@
+// A launch configuration that compiles the extension and then opens it inside a new window
+{
+	"version": "0.1.0",
+	"configurations": [
+		{
+			"name": "Launch Tests",
+			"type": "extensionHost",
+			"request": "launch",
+			"runtimeExecutable": "${execPath}",
+			"args": ["${workspaceFolder}/../../", "${workspaceFolder}/test", "--extensionDevelopmentPath=${workspaceFolder}", "--extensionTestsPath=${workspaceFolder}/out" ],
+			"stopOnEntry": false,
+			"sourceMaps": true,
+			"outDir": "${workspaceFolder}/out",
+			"preLaunchTask": "npm"
+		}
+	]
+}

extensions/integration-tests/.vscode/tasks.json

@@ -0,0 +1,31 @@
+// Available variables which can be used inside of strings.
+// ${workspaceFolder}: the root folder of the team
+// ${file}: the current opened file
+// ${relativeFile}: the current opened file relative to cwd
+// ${fileBasename}: the current opened file's basename
+// ${fileDirname}: the current opened file's dirname
+// ${fileExtname}: the current opened file's extension
+// ${cwd}: the current working directory of the spawned process
+
+// A task runner that calls a custom npm script that compiles the extension.
+{
+	"version": "0.1.0",
+
+	// we want to run npm
+	"command": "npm",
+
+	// the command is a shell script
+	"isShellCommand": true,
+
+	// show the output window only if unrecognized errors occur.
+	"showOutput": "silent",
+
+	// we run the custom script "compile" as defined in package.json
+	"args": ["run", "compile", "--loglevel", "silent"],
+
+	// The tsc compiler is started in watching mode
+	"isWatching": true,
+
+	// use the standard tsc in watch mode problem matcher to find compile problems in the output.
+	"problemMatcher": "$tsc-watch"
+}

extensions/integration-tests/extensionInstallers/readme.md

@@ -0,0 +1 @@
+copy the extension installers to this folder

extensions/integration-tests/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "integration-tests",
+  "description": "Integration Tests",
+  "version": "0.0.1",
+  "publisher": "Microsoft",
+  "private": true,
+  "engines": {
+    "vscode": "*",
+    "sqlops": "*"
+  },
+  "activationEvents": [
+    "*"
+  ],
+  "main": "./out/main",
+  "extensionDependencies": [
+    "Microsoft.agent",
+    "Microsoft.import",
+    "Microsoft.profiler",
+    "Microsoft.mssql",
+    "Microsoft.notebook"
+  ],
+  "contributes": {
+    "configuration": {
+      "type": "object",
+      "title": "ADS Integration Test Configuration",
+      "properties": {
+        "test.testSetupCompleted": {
+          "type": "boolean",
+          "default": false,
+          "description": ""
+        }
+      }
+    },
+    "commands": [{
+        "command": "test.setupIntegrationTest",
+        "title": "Setup Integration Test",
+        "category": "Test"
+      },
+      {
+        "command": "test.waitForExtensionsToLoad",
+        "title": "Wait For Extensions To Load",
+        "category": "Test"
+      }
+    ]
+  },
+  "scripts": {
+    "vscode:prepublish": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:vscode-colorize-tests ./tsconfig.json",
+    "postinstall": "node ./node_modules/vscode/bin/install"
+  },
+  "devDependencies": {
+    "@types/node": "7.0.43",
+    "@types/chai": "3.4.34",
+    "mocha-junit-reporter": "^1.17.0",
+    "mocha-multi-reporters": "^1.1.7",
+    "vscode": "1.1.5",
+    "chai": "3.5.0"
+  }
+}

extensions/integration-tests/readme.md

@@ -0,0 +1,32 @@
+This integration-tests suite is based on the extension testing feature provided by VS Code, We can use this for:
+a.	Commands for setting up the environment for feature testing.
+b.	Adding test cases that do not need UI interaction or the test scenarios not supported by the UI automation framework (e.g. object explorer context menu – not html based)
+
+extensionInstallers folder: copy the VISX installers for the extensions we would like to run the tests with.
+src folder: this is where the test file for features should be added, name the file like this: feature.test.ts. e.g. objectExplorer.test.ts
+
+UI automation testing:
+the ADS UI automation test cases should be added under $root/test/smoke/src/sql folder. Each feature should create its own folder and add 2 files, one for accessing the feature and the other for the test cases. For example: objectExplorer.ts and objectExplorer.test.ts. 
+
+Setup step:
+1.	Launch ADS
+2.	Install extensions from /extensions/integration-tests/extensionInstallers by calling the test command in the integration-tests extension
+3.	Set configuration values. E.g. Enable preview features by calling the test command in the integration-tests extension
+
+For now this has only been tested for Windows platform
+
+How to run the test:
+1.	In the build pipeline: 
+The integration tests and UI automation tests have been added to ADS windows pipeline to run the test and report the results, you can find the test result under the test tab.
+
+2.	Local environment:
+Integration tests:
+test-integration.bat or test-integration.sh under scripts folder
+
+UI automation tests:
+navigate to test/smoke folder and run: node test/index.js
+You can also run UI automation from VSCode by selecting the launch option: Launch Smoke Test.
+
+ADS will be launched using new temp folders: extension folder and data folder so that your local dev environment won't be changed.
+
+

extensions/integration-tests/src/index.ts

@@ -0,0 +1,36 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import * as vscode from 'vscode';
+import { context } from './testContext';
+
+const path = require('path');
+const testRunner = require('vscode/lib/testrunner');
+
+const suite = 'Integration Tests';
+
+const options: any = {
+	ui: 'tdd',
+	useColors: true,
+	timeout: 600000
+};
+
+if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
+	options.reporter = 'mocha-multi-reporters';
+	options.reporterOptions = {
+		reporterEnabled: 'spec, mocha-junit-reporter',
+		mochaJunitReporterReporterOptions: {
+			testsuitesTitle: `${suite} ${process.platform}`,
+			mochaFile: path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${suite.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`)
+		}
+	};
+}
+
+if (!vscode.workspace.getConfiguration('test')['testSetupCompleted']) {
+	context.RunTest = false;
+}
+
+testRunner.configure(options);
+
+export = testRunner;

extensions/integration-tests/src/main.ts

@@ -0,0 +1,77 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+import * as vscode from 'vscode';
+import * as sqlops from 'sqlops';
+import { normalize, join } from 'path';
+import * as fs from 'fs';
+
+const TEST_SETUP_COMPLETED_TEXT: string = 'Test Setup Completed';
+const EXTENSION_LOADED_TEXT: string = 'Test Extension Loaded';
+const ALL_EXTENSION_LOADED_TEXT: string = 'All Extensions Loaded';
+
+var statusBarItemTimer: NodeJS.Timer;
+
+export function activate(context: vscode.ExtensionContext) {
+	var statusBarItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left);
+	vscode.commands.registerCommand('test.setupIntegrationTest', async () => {
+		let extensionInstallersFolder = normalize(join(__dirname, '../extensionInstallers'));
+		let installers = fs.readdirSync(extensionInstallersFolder);
+		for (let i = 0; i < installers.length; i++) {
+			if (installers[i].endsWith('.vsix')) {
+				let installerFullPath = join(extensionInstallersFolder, installers[i]);
+				await sqlops.extensions.install(installerFullPath);
+			}
+		}
+		await setConfiguration('workbench.enablePreviewFeatures', true);
+		await setConfiguration('workbench.showConnectDialogOnStartup', false);
+		await setConfiguration('test.testSetupCompleted', true);
+		showStatusBarItem(statusBarItem, TEST_SETUP_COMPLETED_TEXT);
+	});
+
+	vscode.commands.registerCommand('test.waitForExtensionsToLoad', async () => {
+		let expectedExtensions = ['Microsoft.agent', 'Microsoft.import', 'Microsoft.mssql', 'Microsoft.profiler'];
+		do {
+			let extensions = vscode.extensions.all.filter(ext => { return expectedExtensions.indexOf(ext.id) !== -1; });
+
+			let isReady = true;
+			for (let i = 0; i < extensions.length; i++) {
+				let extension = extensions[i];
+				isReady = isReady && extension.isActive;
+				if (!isReady) {
+					break;
+				}
+			}
+
+			if (isReady) {
+				showStatusBarItem(statusBarItem, ALL_EXTENSION_LOADED_TEXT);
+				break;
+			} else {
+				await new Promise(resolve => { setTimeout(resolve, 1000); });
+			}
+		}
+		while (true);
+	});
+	showStatusBarItem(statusBarItem, EXTENSION_LOADED_TEXT);
+}
+
+function showStatusBarItem(statusBarItem: vscode.StatusBarItem, text: string) {
+	statusBarItem.text = text;
+	statusBarItem.tooltip = text;
+	statusBarItem.show();
+	clearTimeout(statusBarItemTimer);
+	statusBarItemTimer = setTimeout(function () {
+		statusBarItem.hide();
+	}, 5000);
+}
+
+// this method is called when your extension is deactivated
+export function deactivate(): void {
+
+}
+
+async function setConfiguration(name: string, value: any) {
+	await vscode.workspace.getConfiguration().update(name, value, true);
+}

extensions/integration-tests/src/objectExplorer.test.ts

@@ -0,0 +1,29 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import 'mocha';
+import * as sqlops from 'sqlops';
+import { context } from './testContext';
+import { getDefaultTestingServer } from './testConfig';
+import { connectToServer } from './utils';
+import assert = require('assert');
+
+if (context.RunTest) {
+	suite('Object Explorer integration test suite', () => {
+		test('context menu test', async function () {
+			await connectToServer(await getDefaultTestingServer());
+			let nodes = <sqlops.objectexplorer.ObjectExplorerNode[]>await sqlops.objectexplorer.getActiveConnectionNodes();
+			assert(nodes.length === 1, `expecting 1 active connection, actual: ${nodes.length}`);
+			let actions = await sqlops.objectexplorer.getNodeActions(nodes[0].connectionId, nodes[0].nodePath);
+			const expectedActions = ['Manage', 'New Query', 'Disconnect', 'Delete Connection', 'Refresh', 'Launch Profiler'];
+
+			const expectedString = expectedActions.join(',');
+			const actualString = actions.join(',');
+			assert(expectedActions.length === actions.length && expectedString === actualString, `Expected actions: "${expectedString}", Actual actions: "${actualString}"`);
+		});
+	});
+}

extensions/integration-tests/src/setup.test.ts

@@ -0,0 +1,21 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import 'mocha';
+import * as vscode from 'vscode';
+import { context } from './testContext';
+
+if (!context.RunTest) {
+	suite('integration test setup', () => {
+		test('test setup', async function () {
+			//Prepare the environment and make it ready for testing
+			await vscode.commands.executeCommand('test.setupIntegrationTest');
+			//Reload the window, this is required for some changes made by the 'test.setupIntegrationTest' to work
+			await vscode.commands.executeCommand('workbench.action.reloadWindow');
+		});
+	});
+}

extensions/integration-tests/src/testConfig.ts

@@ -0,0 +1,92 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/*
+	TODO: Due to a runtime error, I duplicated this file at these 2 locations:
+	$/extensions/integration-test/src/testConfig.ts
+	$/test/smoke/src/sql/testConfig.ts
+	for now, make sure to keep both files in sync.
+*/
+
+interface ITestServerProfile {
+	serverName: string;
+	userName: string;
+	password: string;
+	authenticationType: AuthenticationType;
+	database: string;
+	provider: ConnectionProvider;
+	version: string;
+}
+
+interface INameDisplayNamePair {
+	name: string;
+	displayName: string;
+}
+
+export enum AuthenticationType {
+	Windows,
+	SqlLogin
+}
+
+export enum ConnectionProvider {
+	SQLServer
+}
+
+var connectionProviderMapping = {};
+var authenticationTypeMapping = {};
+connectionProviderMapping[ConnectionProvider.SQLServer] = { name: 'MSSQL', displayName: 'Microsoft SQL Server' };
+
+authenticationTypeMapping[AuthenticationType.SqlLogin] = { name: 'SqlLogin', displayName: 'SQL Login' };
+authenticationTypeMapping[AuthenticationType.Windows] = { name: 'Integrated', displayName: 'Windows Authentication' };
+
+export class TestServerProfile {
+	constructor(private _profile: ITestServerProfile) { }
+	public get serverName(): string { return this._profile.serverName; }
+	public get userName(): string { return this._profile.userName; }
+	public get password(): string { return this._profile.password; }
+	public get database(): string { return this._profile.database; }
+	public get version(): string { return this._profile.version; }
+	public get provider(): ConnectionProvider { return this._profile.provider; }
+	public get providerName(): string { return getEnumMappingEntry(connectionProviderMapping, this.provider).name; }
+	public get providerDisplayName(): string { return getEnumMappingEntry(connectionProviderMapping, this.provider).displayName; }
+	public get authenticationType(): AuthenticationType { return this._profile.authenticationType; }
+	public get authenticationTypeName(): string { return getEnumMappingEntry(authenticationTypeMapping, this.authenticationType).name; }
+	public get authenticationTypeDisplayName(): string { return getEnumMappingEntry(authenticationTypeMapping, this.authenticationType).displayName; }
+}
+
+var TestingServers: TestServerProfile[] = [
+	new TestServerProfile(
+		{
+			serverName: 'SQLTOOLS2017-3',
+			userName: '',
+			password: '',
+			authenticationType: AuthenticationType.Windows,
+			database: 'master',
+			provider: ConnectionProvider.SQLServer,
+			version: '2017'
+		})
+];
+
+function getEnumMappingEntry(mapping: any, enumValue: any): INameDisplayNamePair {
+	let entry = mapping[enumValue];
+	if (entry) {
+		return entry;
+	} else {
+		throw `Unknown enum type: ${enumValue.toString()}`;
+	}
+}
+
+export async function getDefaultTestingServer(): Promise<TestServerProfile> {
+	let servers = await getTestingServers();
+	return servers[0];
+}
+
+export async function getTestingServers(): Promise<TestServerProfile[]> {
+	let promise = new Promise<TestServerProfile[]>(resolve => {
+		resolve(TestingServers);
+	});
+	await promise;
+	return promise;
+}

extensions/integration-tests/src/testContext.ts

@@ -0,0 +1,8 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+export var context = {
+	RunTest: true
+};

extensions/integration-tests/src/typings/ref.d.ts

@@ -0,0 +1,8 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+/// <reference path='../../../../src/sql/sqlops.d.ts'/>
+/// <reference path='../../../../src/sql/sqlops.proposed.d.ts'/>
+/// <reference path='../../../../src/sql/sqlops.test.d.ts'/>
+/// <reference types='@types/node'/>

extensions/integration-tests/src/utils.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert = require('assert');
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+import { TestServerProfile } from './testConfig';
+
+export async function connectToServer(server: TestServerProfile) {
+	let connectionProfile: sqlops.IConnectionProfile = {
+		serverName: server.serverName,
+		databaseName: server.database,
+		authenticationType: server.authenticationTypeName,
+		providerName: server.providerName,
+		connectionName: '',
+		userName: server.userName,
+		password: server.password,
+		savePassword: false,
+		groupFullName: undefined,
+		saveProfile: true,
+		id: undefined,
+		groupId: undefined,
+		options: {}
+	};
+	await ensureConnectionViewOpened();
+	let result = <sqlops.ConnectionResult>await sqlops.connection.connect(connectionProfile);
+	assert(result.connected, `Failed to connect to "${connectionProfile.serverName}", error code: ${result.errorCode}, error message: ${result.errorMessage}`);
+
+	//workaround
+	//wait for OE to load
+	await new Promise(c => setTimeout(c, 3000));
+}
+
+export async function ensureConnectionViewOpened() {
+	await vscode.commands.executeCommand('workbench.view.connections');
+}

extensions/integration-tests/tsconfig.json

@@ -0,0 +1,15 @@
+{
+	"compilerOptions": {
+		"module": "commonjs",
+		"target": "ES5",
+		"outDir": "out",
+		"noUnusedLocals": true,
+		"lib": [
+			"es2015"
+		],
+		"sourceMap": true
+	},
+	"include": [
+		"src/**/*"
+	]
+}

extensions/mssql/package.json

@@ -18,13 +18,21 @@
     "update-grammar": "node ../../build/npm/update-grammar.js Microsoft/vscode-mssql syntaxes/SQL.plist ./syntaxes/sql.tmLanguage.json"
   },
   "dependencies": {
-    "dataprotocol-client": "github:Microsoft/sqlops-dataprotocolclient#0.2.10",
+    "bytes": "^3.1.0",
+    "clipboardy": "^1.2.3",
+    "dataprotocol-client": "github:Microsoft/sqlops-dataprotocolclient#0.2.15",
+    "error-ex": "^1.3.2",
+    "figures": "^2.0.0",
     "opener": "^1.4.3",
+    "request-promise": "^4.2.2",
     "service-downloader": "github:anthonydresser/service-downloader#0.1.5",
-    "vscode-extension-telemetry": "^0.0.15"
-  },
-  "devDependencies": {
+    "stream-meter": "^1.0.4",
+    "uri-js": "^4.2.2",
+    "vscode-extension-telemetry": "^0.0.15",
+    "vscode-nls": "^4.0.0",
+    "webhdfs": "^1.1.1"
   },
+  "devDependencies": {},
   "contributes": {
     "languages": [
       {
@@ -37,6 +45,80 @@
         ]
       }
     ],
+    "commands": [
+      {
+        "command": "mssqlCluster.uploadFiles",
+        "title": "%mssqlCluster.uploadFiles%"
+      },
+      {
+        "command": "mssqlCluster.mkdir",
+        "title": "%mssqlCluster.mkdir%"
+      },
+      {
+        "command": "mssqlCluster.deleteFiles",
+        "title": "%mssqlCluster.deleteFiles%"
+      },
+      {
+        "command": "mssqlCluster.previewFile",
+        "title": "%mssqlCluster.previewFile%"
+      },
+      {
+        "command": "mssqlCluster.saveFile",
+        "title": "%mssqlCluster.saveFile%"
+      },
+      {
+        "command": "mssqlCluster.copyPath",
+        "title": "%mssqlCluster.copyPath%"
+      },
+      {
+        "command": "mssqlCluster.task.newNotebook",
+        "title": "%notebook.command.new%",
+        "icon": {
+          "dark": "resources/dark/new_notebook_inverse.svg",
+          "light": "resources/light/new_notebook.svg"
+        }
+      },
+      {
+        "command": "mssqlCluster.task.openNotebook",
+        "title": "%notebook.command.open%",
+        "icon": {
+          "dark": "resources/dark/open_notebook_inverse.svg",
+          "light": "resources/light/open_notebook.svg"
+        }
+      },
+      {
+        "command": "mssqlCluster.livy.cmd.submitSparkJob",
+        "title": "%title.submitSparkJob%"
+      },
+      {
+        "command": "mssqlCluster.livy.task.submitSparkJob",
+        "title": "%title.newSparkJob%",
+        "icon": {
+          "dark": "resources/dark/new_spark_job_inverse.svg",
+          "light": "resources/light/new_spark_job.svg"
+        }
+      },
+      {
+        "command": "mssqlCluster.livy.task.openSparkHistory",
+        "title": "%title.openSparkHistory%",
+        "icon": {
+          "dark": "resources/dark/new_spark_job_inverse.svg",
+          "light": "resources/light/new_spark_job.svg"
+        }
+      },
+      {
+        "command": "mssqlCluster.livy.task.openYarnHistory",
+        "title": "%title.openYarnHistory%",
+        "icon": {
+          "dark": "resources/light/hadoop.svg",
+          "light": "resources/light/hadoop.svg"
+        }
+      },
+      {
+        "command": "mssqlCluster.livy.cmd.submitFileToSparkJob",
+        "title": "%title.submitSparkJob%"
+      }
+    ],
     "outputChannels": [
       "MSSQL"
     ],
@@ -126,6 +208,92 @@
         }
       }
     },
+    "menus": {
+      "commandPalette": [
+        {
+          "command": "mssqlCluster.uploadFiles",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.mkdir",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.deleteFiles",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.previewFile",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.saveFile",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.copyPath",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.task.newNotebook",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.task.openNotebook",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.livy.cmd.submitFileToSparkJob",
+          "when": "false"
+        },
+        {
+          "command": "mssqlCluster.livy.task.submitSparkJob",
+          "when": "false"
+        }
+      ],
+      "objectExplorer/item/context": [
+        {
+          "command": "mssqlCluster.uploadFiles",
+          "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file",
+          "group": "1mssqlCluster@1"
+        },
+        {
+          "command": "mssqlCluster.mkdir",
+          "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file",
+          "group": "1mssqlCluster@1"
+        },
+        {
+          "command": "mssqlCluster.saveFile",
+          "when": "nodeType == mssqlCluster:file",
+          "group": "1mssqlCluster@1"
+        },
+        {
+          "command": "mssqlCluster.previewFile",
+          "when": "nodeType == mssqlCluster:file",
+          "group": "1mssqlCluster@2"
+        },
+        {
+          "command": "mssqlCluster.copyPath",
+          "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:connection && nodeType != mssqlCluster:message",
+          "group": "1mssqlCluster@3"
+        },
+        {
+          "command": "mssqlCluster.deleteFiles",
+          "when": "nodeType=~/^mssqlCluster/ && viewItem != mssqlCluster:connection && nodeType != mssqlCluster:message",
+          "group": "1mssqlCluster@4"
+        },
+        {
+          "command": "mssqlCluster.livy.cmd.submitSparkJob",
+          "when": "connectionProvider == MSSQL && nodeType == dataservices",
+          "group": "1root@1"
+        },
+        {
+          "command": "mssqlCluster.livy.cmd.submitFileToSparkJob",
+          "when": "nodeType == mssqlCluster:file && nodeSubType == mssqlCluster:spark",
+          "group": "1mssqlCluster@6"
+        }
+      ]
+    },
     "dashboard": {
       "provider": "MSSQL",
       "flavors": [
@@ -221,6 +389,34 @@
         }
       ]
     },
+    "dashboard.tabs": [
+      {
+        "id": "mssql-big-data-cluster",
+        "description": "tab.bigDataClusterDescription",
+        "provider": "MSSQL",
+        "title": "%title.bigDataCluster%",
+        "when": "connectionProvider == 'MSSQL' && mssql:iscluster",
+        "container": {
+          "grid-container": [
+            {
+              "name": "%title.tasks%",
+              "row": 0,
+              "col": 0,
+              "colspan": 2,
+              "widget": {
+                "tasks-widget": [
+                  "mssqlCluster.task.newNotebook",
+                  "mssqlCluster.task.openNotebook",
+                  "mssqlCluster.livy.task.submitSparkJob",
+                  "mssqlCluster.livy.task.openSparkHistory",
+                  "mssqlCluster.livy.task.openYarnHistory"
+                ]
+              }
+            }
+          ]
+        }
+      }
+    ],
     "connectionProvider": {
       "providerId": "MSSQL",
       "displayName": "Microsoft SQL Server",

extensions/mssql/package.nls.json

@@ -4,5 +4,25 @@
 	"json.schemas.fileMatch.desc": "An array of file patterns to match against when resolving JSON files to schemas.",
 	"json.schemas.fileMatch.item.desc": "A file pattern that can contain '*' to match against when resolving JSON files to schemas.",
 	"json.schemas.schema.desc": "The schema definition for the given URL. The schema only needs to be provided to avoid accesses to the schema URL.",
-	"json.format.enable.desc": "Enable/disable default JSON formatter (requires restart)"
+	"json.format.enable.desc": "Enable/disable default JSON formatter (requires restart)",
+
+	"mssqlCluster.uploadFiles": "Upload files",
+	"mssqlCluster.mkdir": "New directory",
+	"mssqlCluster.deleteFiles": "Delete",
+	"mssqlCluster.previewFile": "Preview",
+	"mssqlCluster.saveFile": "Save",
+	"mssqlCluster.copyPath": "Copy Path",
+
+	"notebook.command.new": "New Notebook",
+	"notebook.command.open": "Open Notebook",
+
+	"tab.bigDataClusterDescription": "Tasks and information about your SQL Server Big Data Cluster",
+	"title.bigDataCluster": "SQL Server Big Data Cluster",
+    "title.submitSparkJob": "Submit Spark Job",
+    "title.newSparkJob": "New Spark Job",
+    "title.openSparkHistory": "View Spark History",
+    "title.openYarnHistory": "View Yarn History",
+    "title.tasks": "Tasks",
+    "title.installPackages": "Install Packages",
+    "title.configurePython": "Configure Python for Notebooks"
 }

extensions/mssql/resources/dark/cluster_inverse.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1{fill:#fff;}</style></defs><title>cluster_inverse</title><path class="cls-1" d="M14,7a1.94,1.94,0,0,1,.78.16,2,2,0,0,1,1.07,1.07,2,2,0,0,1,0,1.55,2,2,0,0,1-1.07,1.07,2,2,0,0,1-1.51,0,2.05,2.05,0,0,1-1.05-1,1.88,1.88,0,0,1-.2-.72L10.84,9a3,3,0,0,1-.56,1,3,3,0,0,1-.87.7L9.86,12H10a1.94,1.94,0,0,1,.78.16,2,2,0,0,1,1.07,1.07,2,2,0,0,1,0,1.55,2,2,0,0,1-1.07,1.07,2,2,0,0,1-1.55,0,2,2,0,0,1-1.07-1.07A2,2,0,0,1,8.25,13a2,2,0,0,1,.67-.72L8.46,11l-.23,0H8a3,3,0,0,1-1.36-.32,3,3,0,0,1-1.07-.9L4,10.58a2,2,0,0,1-.11,1.2,2,2,0,0,1-1.07,1.07A1.94,1.94,0,0,1,2,13a1.94,1.94,0,0,1-.78-.16A2,2,0,0,1,.16,11.78a2,2,0,0,1,0-1.55A2,2,0,0,1,1.22,9.16,1.94,1.94,0,0,1,2,9a2,2,0,0,1,.83.18,2,2,0,0,1,.68.51l1.63-.81A3,3,0,0,1,5.2,6.93,2.91,2.91,0,0,1,5.77,6L4.82,4.82A2,2,0,0,1,4,5a1.94,1.94,0,0,1-.78-.16A2,2,0,0,1,2.16,3.78a2,2,0,0,1,0-1.55A2,2,0,0,1,3.22,1.16a2,2,0,0,1,1.55,0A2,2,0,0,1,5.84,2.22,1.94,1.94,0,0,1,6,3a1.94,1.94,0,0,1-.4,1.2l.94,1.18a3.24,3.24,0,0,1,.71-.28A2.94,2.94,0,0,1,8,5a3,3,0,0,1,1.23.26l1.28-1.92a2,2,0,0,1-.37-.62A2,2,0,0,1,10,2a1.94,1.94,0,0,1,.16-.78A2,2,0,0,1,11.22.16a2,2,0,0,1,1.55,0,2,2,0,0,1,1.07,1.07A1.94,1.94,0,0,1,14,2a1.94,1.94,0,0,1-.16.78,2,2,0,0,1-1.07,1.07A1.94,1.94,0,0,1,12,4a2.06,2.06,0,0,1-.66-.11L10.05,5.82A3,3,0,0,1,11,8l1.17.2a2,2,0,0,1,.74-.86,2.14,2.14,0,0,1,.52-.24A1.92,1.92,0,0,1,14,7ZM2,12a1,1,0,0,0,.39-.08,1,1,0,0,0,.53-.53,1,1,0,0,0,0-.78,1,1,0,0,0-.53-.53,1,1,0,0,0-.78,0,1,1,0,0,0-.53.53,1,1,0,0,0,0,.78,1,1,0,0,0,.53.53A1,1,0,0,0,2,12ZM3,3a1,1,0,0,0,.08.39,1,1,0,0,0,.53.53,1,1,0,0,0,.78,0,1,1,0,0,0,.53-.53,1,1,0,0,0,0-.78,1,1,0,0,0-.53-.53,1,1,0,0,0-.78,0,1,1,0,0,0-.53.53A1,1,0,0,0,3,3Zm5,7a1.94,1.94,0,0,0,.78-.16A2,2,0,0,0,9.84,8.78a2,2,0,0,0,0-1.55A2,2,0,0,0,8.78,6.16a2,2,0,0,0-1.55,0A2,2,0,0,0,6.16,7.22a2,2,0,0,0,0,1.55A2,2,0,0,0,7.22,9.84,1.94,1.94,0,0,0,8,10Zm3,4a1,1,0,0,0-.08-.39,1,1,0,0,0-.53-.53,1,1,0,0,0-.78,0,1,1,0,0,0-.53.53,1,1,0,0,0,0,.78,1,1,0,0,0,.53.53,1,1,0,0,0,.78,0,1,1,0,0,0,.53-.53A1,1,0,0,0,11,14ZM12,1a1,1,0,0,0-.39.08,1,1,0,0,0-.53.53,1,1,0,0,0,0,.78,1,1,0,0,0,.53.53,1,1,0,0,0,.78,0,1,1,0,0,0,.53-.53,1,1,0,0,0,0-.78,1,1,0,0,0-.53-.53A1,1,0,0,0,12,1Zm2,9a1,1,0,0,0,.39-.08,1,1,0,0,0,.53-.53,1,1,0,0,0,0-.78,1,1,0,0,0-.53-.53,1,1,0,0,0-.78,0,1,1,0,0,0-.53.53,1,1,0,0,0,0,.78,1,1,0,0,0,.53.53A1,1,0,0,0,14,10Z"/></svg>

extensions/mssql/resources/dark/new_notebook_inverse.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1{fill:#fff;}.cls-2{fill:#388a34;}</style></defs><title>new_notebook_inverse</title><path class="cls-1" d="M11.87,1.24V.33H9.13A3.78,3.78,0,0,0,7.92.52a3.48,3.48,0,0,0-1.07.58A3.6,3.6,0,0,0,5.78.52,3.78,3.78,0,0,0,4.57.33H1.83v.91H0V13.1H9.67v-.91H7a4,4,0,0,1,.47-.39A2.39,2.39,0,0,1,8,11.52a2.2,2.2,0,0,1,.53-.18,2.93,2.93,0,0,1,.61-.06h2.74V2.15h.91V9h.91V1.24Zm-9.13,0H4.57a3,3,0,0,1,1,.17,2.58,2.58,0,0,1,.85.49v8.93a3.94,3.94,0,0,0-.88-.35,3.73,3.73,0,0,0-.94-.12H2.74Zm-1.82,11v-10h.91v9.13H4.57a2.93,2.93,0,0,1,.61.06,2.55,2.55,0,0,1,.53.18,2.68,2.68,0,0,1,.49.28,3.29,3.29,0,0,1,.46.39Zm8.21-1.83a3.73,3.73,0,0,0-.94.12,4.22,4.22,0,0,0-.89.35V1.9a2.74,2.74,0,0,1,.86-.49,2.91,2.91,0,0,1,1-.17H11v9.12ZM12.87,10v2.2h-2.2v.91h3V10Z"/><polygon class="cls-2" points="16 12.19 16 13.13 13.8 13.13 13.8 15.33 12.87 15.33 12.87 13.13 10.67 13.13 10.67 12.19 12.87 12.19 12.87 9.99 13.8 9.99 13.8 12.19 16 12.19"/><path class="cls-2" d="M13.8,12.19V10h-.93v2.2h-2.2v.94h2.2v2.2h.93v-2.2H16v-.94Z"/></svg>

extensions/mssql/resources/dark/open_notebook_inverse.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1{fill:#fff;}.cls-2{fill:#0095d7;}</style></defs><title>open_notebook_inverse</title><path class="cls-1" d="M12.55,4.21l-.08-.11h-.56l-.69.06a1.54,1.54,0,0,0-.23.29v8.69H9.18a3.32,3.32,0,0,0-.93.13,3.34,3.34,0,0,0-.87.34V4.76a2.88,2.88,0,0,1,.43-.31A5.58,5.58,0,0,1,8.29,3.3a2.63,2.63,0,0,0-.3.09A3.62,3.62,0,0,0,6.93,4a3.68,3.68,0,0,0-1.07-.57A3.58,3.58,0,0,0,4.67,3.2H2v.9H.15V15.85H13.72V5.48ZM2.86,4.1H4.67a2.61,2.61,0,0,1,1,.17,2.32,2.32,0,0,1,.86.49v8.85a3.27,3.27,0,0,0-.88-.34,3.22,3.22,0,0,0-.93-.13H2.86ZM1,15V5H2v9H4.67a3.94,3.94,0,0,1,.61.06,3.2,3.2,0,0,1,.52.18,4.19,4.19,0,0,1,.49.29,2.28,2.28,0,0,1,.45.39ZM12.8,15H7.11a2.7,2.7,0,0,1,.47-.39A2.83,2.83,0,0,1,8,14.28a3.42,3.42,0,0,1,.54-.18A3.81,3.81,0,0,1,9.18,14h2.73V5h.89Z"/><polygon class="cls-2" points="13.2 3.56 13.2 3.58 13.19 3.57 13.2 3.56"/><path class="cls-2" d="M13.19,3.57h0v0Z"/><polygon class="cls-2" points="13.2 3.56 13.2 3.58 13.19 3.57 13.2 3.56"/><polygon class="cls-2" points="14.21 1.65 14.19 1.65 14.19 1.63 14.21 1.65"/><path class="cls-2" d="M15.91,2.1,14.2,3.81l-.38.38-.62-.61v0l1-1H12.79a3.35,3.35,0,0,0-1.09.26h0a3.94,3.94,0,0,0-.86.52l-.24.21s0,0,0,0a3.3,3.3,0,0,0-.51.67,3.1,3.1,0,0,0-.26.47A3.41,3.41,0,0,0,9.5,6.11H8.6a4.68,4.68,0,0,1,.16-1.19A4.74,4.74,0,0,1,9,4.26a2.21,2.21,0,0,1,.2-.41,4.66,4.66,0,0,1,.36-.51c.1-.13.22-.26.34-.39a4.14,4.14,0,0,1,.66-.53,1.19,1.19,0,0,1,.23-.16,2.79,2.79,0,0,1,.34-.18l.31-.13.42-.14a4.32,4.32,0,0,1,1.19-.16h1.15l-1-1L13.82,0Z"/></svg>

extensions/mssql/resources/light/cluster.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><title>cluster</title><path d="M14,7a1.94,1.94,0,0,1,.78.16,2,2,0,0,1,1.07,1.07,2,2,0,0,1,0,1.55,2,2,0,0,1-1.07,1.07,2,2,0,0,1-1.51,0,2.05,2.05,0,0,1-1.05-1,1.88,1.88,0,0,1-.2-.72L10.84,9a3,3,0,0,1-.56,1,3,3,0,0,1-.87.7L9.86,12H10a1.94,1.94,0,0,1,.78.16,2,2,0,0,1,1.07,1.07,2,2,0,0,1,0,1.55,2,2,0,0,1-1.07,1.07,2,2,0,0,1-1.55,0,2,2,0,0,1-1.07-1.07A2,2,0,0,1,8.25,13a2,2,0,0,1,.67-.72L8.46,11l-.23,0H8a3,3,0,0,1-1.36-.32,3,3,0,0,1-1.07-.9L4,10.58a2,2,0,0,1-.11,1.2,2,2,0,0,1-1.07,1.07A1.94,1.94,0,0,1,2,13a1.94,1.94,0,0,1-.78-.16A2,2,0,0,1,.16,11.78a2,2,0,0,1,0-1.55A2,2,0,0,1,1.22,9.16,1.94,1.94,0,0,1,2,9a2,2,0,0,1,.83.18,2,2,0,0,1,.68.51l1.63-.81A3,3,0,0,1,5.2,6.93,2.91,2.91,0,0,1,5.77,6L4.82,4.82A2,2,0,0,1,4,5a1.94,1.94,0,0,1-.78-.16A2,2,0,0,1,2.16,3.78a2,2,0,0,1,0-1.55A2,2,0,0,1,3.22,1.16a2,2,0,0,1,1.55,0A2,2,0,0,1,5.84,2.22,1.94,1.94,0,0,1,6,3a1.94,1.94,0,0,1-.4,1.2l.94,1.18a3.24,3.24,0,0,1,.71-.28A2.94,2.94,0,0,1,8,5a3,3,0,0,1,1.23.26l1.28-1.92a2,2,0,0,1-.37-.62A2,2,0,0,1,10,2a1.94,1.94,0,0,1,.16-.78A2,2,0,0,1,11.22.16a2,2,0,0,1,1.55,0,2,2,0,0,1,1.07,1.07A1.94,1.94,0,0,1,14,2a1.94,1.94,0,0,1-.16.78,2,2,0,0,1-1.07,1.07A1.94,1.94,0,0,1,12,4a2.06,2.06,0,0,1-.66-.11L10.05,5.82A3,3,0,0,1,11,8l1.17.2a2,2,0,0,1,.74-.86,2.14,2.14,0,0,1,.52-.24A1.92,1.92,0,0,1,14,7ZM2,12a1,1,0,0,0,.39-.08,1,1,0,0,0,.53-.53,1,1,0,0,0,0-.78,1,1,0,0,0-.53-.53,1,1,0,0,0-.78,0,1,1,0,0,0-.53.53,1,1,0,0,0,0,.78,1,1,0,0,0,.53.53A1,1,0,0,0,2,12ZM3,3a1,1,0,0,0,.08.39,1,1,0,0,0,.53.53,1,1,0,0,0,.78,0,1,1,0,0,0,.53-.53,1,1,0,0,0,0-.78,1,1,0,0,0-.53-.53,1,1,0,0,0-.78,0,1,1,0,0,0-.53.53A1,1,0,0,0,3,3Zm5,7a1.94,1.94,0,0,0,.78-.16A2,2,0,0,0,9.84,8.78a2,2,0,0,0,0-1.55A2,2,0,0,0,8.78,6.16a2,2,0,0,0-1.55,0A2,2,0,0,0,6.16,7.22a2,2,0,0,0,0,1.55A2,2,0,0,0,7.22,9.84,1.94,1.94,0,0,0,8,10Zm3,4a1,1,0,0,0-.08-.39,1,1,0,0,0-.53-.53,1,1,0,0,0-.78,0,1,1,0,0,0-.53.53,1,1,0,0,0,0,.78,1,1,0,0,0,.53.53,1,1,0,0,0,.78,0,1,1,0,0,0,.53-.53A1,1,0,0,0,11,14ZM12,1a1,1,0,0,0-.39.08,1,1,0,0,0-.53.53,1,1,0,0,0,0,.78,1,1,0,0,0,.53.53,1,1,0,0,0,.78,0,1,1,0,0,0,.53-.53,1,1,0,0,0,0-.78,1,1,0,0,0-.53-.53A1,1,0,0,0,12,1Zm2,9a1,1,0,0,0,.39-.08,1,1,0,0,0,.53-.53,1,1,0,0,0,0-.78,1,1,0,0,0-.53-.53,1,1,0,0,0-.78,0,1,1,0,0,0-.53.53,1,1,0,0,0,0,.78,1,1,0,0,0,.53.53A1,1,0,0,0,14,10Z"/></svg>

extensions/mssql/resources/light/new_notebook.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1{fill:#388a34;}</style></defs><title>new_notebook</title><path d="M11.86,1.24V.33H9.13A3.78,3.78,0,0,0,7.91.52a3.48,3.48,0,0,0-1.07.58A3.6,3.6,0,0,0,5.78.52,3.78,3.78,0,0,0,4.57.33H1.83v.91H0V13.1H9.66v-.91H7a4,4,0,0,1,.47-.39A2.39,2.39,0,0,1,8,11.52a2.2,2.2,0,0,1,.53-.18,2.93,2.93,0,0,1,.61-.06h2.74V2.15h.91V9h.91V1.24Zm-9.13,0H4.57a3,3,0,0,1,1,.17,2.58,2.58,0,0,1,.85.49v8.93a3.94,3.94,0,0,0-.88-.35,3.73,3.73,0,0,0-.94-.12H2.73Zm-1.82,11v-10h.91v9.13H4.57a2.93,2.93,0,0,1,.61.06,2.55,2.55,0,0,1,.53.18,2.68,2.68,0,0,1,.49.28,3.29,3.29,0,0,1,.46.39Zm8.21-1.83a3.73,3.73,0,0,0-.94.12,4.22,4.22,0,0,0-.89.35V1.9a2.74,2.74,0,0,1,.86-.49,2.91,2.91,0,0,1,1-.17h1.82v9.12ZM12.86,10v2.2h-2.2v.91h3V10Z"/><polygon class="cls-1" points="15.99 12.19 15.99 13.13 13.79 13.13 13.79 15.33 12.87 15.33 12.87 13.13 10.66 13.13 10.66 12.19 12.87 12.19 12.87 9.99 13.79 9.99 13.79 12.19 15.99 12.19"/><path class="cls-1" d="M13.79,12.19V10h-.93v2.2h-2.2v.94h2.2v2.2h.93v-2.2H16v-.94Z"/></svg>

extensions/mssql/resources/light/new_spark_job.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 16 16"><defs><style>.cls-1,.cls-2{fill:none;}.cls-1{clip-rule:evenodd;}.cls-3{clip-path:url(#clip-path);}.cls-4{fill:#e25a1c;}.cls-5{clip-path:url(#clip-path-2);}.cls-6{fill:#3c3a3e;}.cls-7{clip-path:url(#clip-path-3);}.cls-8{clip-path:url(#clip-path-4);}.cls-9{clip-path:url(#clip-path-5);}.cls-10{clip-path:url(#clip-path-6);}.cls-11{clip-path:url(#clip-path-7);}</style><clipPath id="clip-path"><path class="cls-1" d="M14.58,6.89l0-.06L14,5.7a.07.07,0,0,1,0-.09l.95-1.11a.1.1,0,0,0,0,0l-.28.07-1.15.3a.05.05,0,0,1-.07,0l-.65-1.09a.15.15,0,0,0,0-.05l-.05.29-.18,1s0,.07,0,.11,0,0-.05.06l-1.35.43-.06,0L12.14,6l0,0-.69.45a.07.07,0,0,1-.08,0l-.83-.37a.85.85,0,0,1-.32-.23.43.43,0,0,1,.1-.68,1.23,1.23,0,0,1,.28-.13l1.33-.42A.08.08,0,0,0,12,4.62l.18-1a1.78,1.78,0,0,1,.14-.54.9.9,0,0,1,.12-.18.39.39,0,0,1,.61,0,1.15,1.15,0,0,1,.16.21l.61,1a.07.07,0,0,0,.09,0l1.48-.39a.7.7,0,0,1,.31,0,.3.3,0,0,1,.25.44.84.84,0,0,1-.16.26l-1,1.21a.07.07,0,0,0,0,.09l.62,1.17a.65.65,0,0,1,.09.3.48.48,0,0,1-.42.48.87.87,0,0,1-.39,0l-.93-.28a.05.05,0,0,1,0-.05c0-.22-.07-.44-.11-.65a.14.14,0,0,1,0,0l1.07.29"/></clipPath><clipPath id="clip-path-2"><path class="cls-1" d="M14,10.07h-.84a.08.08,0,0,1-.08,0l-1-1.51,0-.06-.21,1.6h-.73l0-.21.21-1.63.21-1.56a.07.07,0,0,1,0,0l.76-.49h0l-.23,1.74h0l1.2-1.33,0,.18c0,.17.06.33.09.5a.08.08,0,0,1,0,.08l-.77.8,0,0,0,0L13.95,10l0,0h0"/></clipPath><clipPath id="clip-path-3"><path class="cls-1" d="M3.39,9.86l-.06.47-.08.61s0,0,0,0H2.59l0-.29.13-1c.05-.39.09-.77.16-1.16A1.81,1.81,0,0,1,4.29,7.1a1.42,1.42,0,0,1,1.11.18A1.24,1.24,0,0,1,6,8.22a1.66,1.66,0,0,1-.55,1.43,1.7,1.7,0,0,1-.95.47,1.4,1.4,0,0,1-1-.23Zm1.93-1.4a1.71,1.71,0,0,0,0-.22.75.75,0,0,0-.91-.49,1,1,0,0,0-.8.9A.73.73,0,0,0,4,9.42a.86.86,0,0,0,.76-.09A1,1,0,0,0,5.32,8.46Z"/></clipPath><clipPath id="clip-path-4"><path class="cls-1" d="M3.06,6.64l-.66.49L2.3,7a.51.51,0,0,0-.38-.24.43.43,0,0,0-.36.14.25.25,0,0,0,0,.33c.09.12.19.23.29.33l.5.53a1.12,1.12,0,0,1,.3.57,1.16,1.16,0,0,1-.13.75,1.43,1.43,0,0,1-1.08.76,1.42,1.42,0,0,1-.63,0,.93.93,0,0,1-.59-.52c0-.09-.08-.19-.12-.28l.72-.38L.81,9a2.14,2.14,0,0,0,.12.24.49.49,0,0,0,.64.18.7.7,0,0,0,.17-.11.37.37,0,0,0,.07-.51,2.49,2.49,0,0,0-.23-.28c-.2-.22-.4-.43-.59-.65a1,1,0,0,1-.25-.53.91.91,0,0,1,.13-.62A1.34,1.34,0,0,1,2.13,6a1,1,0,0,1,.76.4l.17.23"/></clipPath><clipPath id="clip-path-5"><path class="cls-1" d="M8.4,9.14l-.11.81a.05.05,0,0,1,0,0A1.45,1.45,0,0,1,6.56,9.7a1.31,1.31,0,0,1-.33-1A1.8,1.8,0,0,1,7.79,7.08,1.33,1.33,0,0,1,9,7.52a1.24,1.24,0,0,1,.31.9c0,.22,0,.44-.07.67s-.08.63-.12.94v0H8.48l0-.21c0-.36.1-.72.14-1.09a1.16,1.16,0,0,0-.09-.66A.64.64,0,0,0,8,7.74a1,1,0,0,0-1.09.79.75.75,0,0,0,.3.81A.82.82,0,0,0,8,9.4a1,1,0,0,0,.37-.26"/></clipPath><clipPath id="clip-path-6"><path class="cls-1" d="M11.15,7.14l-.09.68h-.41a.26.26,0,0,0-.24.17.71.71,0,0,0,0,.12l-.2,1.56-.05.39H9.45l0-.29.13-1c0-.29.07-.58.12-.87a.94.94,0,0,1,.84-.75h.57"/></clipPath><clipPath id="clip-path-7"><path class="cls-2" d="M14.28,9.68v.38h-.06V9.68h-.11V9.62h.27v.06Zm.5.38V9.69h0l-.11.37h0l-.11-.37h0v.37h-.06V9.62h.09l.1.34.1-.34h.09v.44Z"/></clipPath></defs><title>new_spark_job</title><g class="cls-3"><rect class="cls-4" x="5.65" y="-1.69" width="14.7" height="13.77"/></g><g class="cls-5"><rect class="cls-6" x="6.67" y="1.7" width="11.72" height="12.78"/></g><g class="cls-7"><rect class="cls-6" x="-1.83" y="2.64" width="12.23" height="12.75"/></g><g class="cls-8"><rect class="cls-6" x="-4.35" y="1.59" width="11.83" height="12.99"/></g><g class="cls-9"><rect class="cls-6" x="1.82" y="2.64" width="11.93" height="11.91"/></g><g class="cls-10"><rect class="cls-6" x="5.03" y="2.72" width="10.53" height="11.76"/></g><g class="cls-11"><rect class="cls-6" x="9.7" y="5.2" width="9.55" height="9.27"/></g></svg>

extensions/mssql/resources/light/open_notebook.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1{fill:#00539c;}</style></defs><title>open_notebook</title><path d="M12.4,4.21l-.08-.11h-.56l-.69.06a1.54,1.54,0,0,0-.23.29v8.69H9a3.32,3.32,0,0,0-.93.13,3.34,3.34,0,0,0-.87.34V4.76a2.88,2.88,0,0,1,.43-.31A5.58,5.58,0,0,1,8.14,3.3a2.63,2.63,0,0,0-.3.09A3.62,3.62,0,0,0,6.78,4a3.68,3.68,0,0,0-1.07-.57A3.58,3.58,0,0,0,4.52,3.2H1.81v.9H0V15.85H13.57V5.48ZM2.71,4.1H4.52a2.61,2.61,0,0,1,1,.17,2.32,2.32,0,0,1,.86.49v8.85a3.27,3.27,0,0,0-.88-.34,3.22,3.22,0,0,0-.93-.13H2.71ZM.9,15V5h.91v9H4.52a3.94,3.94,0,0,1,.61.06,3.2,3.2,0,0,1,.52.18,4.19,4.19,0,0,1,.49.29,2.28,2.28,0,0,1,.45.39Zm11.75,0H7a2.7,2.7,0,0,1,.47-.39,2.83,2.83,0,0,1,.47-.29,3.42,3.42,0,0,1,.54-.18A3.81,3.81,0,0,1,9,14h2.73V5h.89Z"/><polygon class="cls-1" points="13.05 3.56 13.05 3.58 13.04 3.57 13.05 3.56"/><path class="cls-1" d="M13,3.57h0v0Z"/><polygon class="cls-1" points="13.05 3.56 13.05 3.58 13.04 3.57 13.05 3.56"/><polygon class="cls-1" points="14.06 1.65 14.04 1.65 14.04 1.63 14.06 1.65"/><path class="cls-1" d="M15.76,2.1,14,3.81l-.38.38L13,3.58v0l1-1H12.64a3.35,3.35,0,0,0-1.09.26h0a3.94,3.94,0,0,0-.86.52l-.24.21s0,0,0,0a3.3,3.3,0,0,0-.51.67,3.1,3.1,0,0,0-.26.47,3.41,3.41,0,0,0-.27,1.39h-.9a4.68,4.68,0,0,1,.16-1.19,4.74,4.74,0,0,1,.25-.66,2.21,2.21,0,0,1,.2-.41,4.66,4.66,0,0,1,.36-.51c.1-.13.22-.26.34-.39a4.14,4.14,0,0,1,.66-.53,1.19,1.19,0,0,1,.23-.16A2.79,2.79,0,0,1,11,2.08l.31-.13.42-.14a4.32,4.32,0,0,1,1.19-.16h1.15l-1-1L13.67,0Z"/></svg>

extensions/mssql/src/api/mssqlapis.d.ts

@@ -0,0 +1,65 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+// This is the place for extensions to expose APIs.
+
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+
+/**
+* The APIs provided by Mssql extension
+*
+* @export
+* @interface MssqlExtensionApi
+*/
+export interface MssqlExtensionApi {
+	/**
+	 * Gets the object explorer API that supports querying over the connections supported by this extension
+	 *
+	 * @returns {IMssqlObjectExplorerBrowser}
+	 * @memberof IMssqlExtensionApi
+	 */
+	getMssqlObjectExplorerBrowser(): MssqlObjectExplorerBrowser;
+}
+
+/**
+ * A browser supporting actions over the object explorer connections provided by this extension.
+ * Currently this is the
+ *
+ * @export
+ * @interface MssqlObjectExplorerBrowser
+ */
+export interface MssqlObjectExplorerBrowser {
+	/**
+	 * Gets the matching node given a context object, e.g. one from a right-click on a node in Object Explorer
+	 *
+	 * @param {sqlops.ObjectExplorerContext} objectExplorerContext
+	 * @returns {Promise<T>}
+	 */
+	getNode<T extends ITreeNode>(objectExplorerContext: sqlops.ObjectExplorerContext): Promise<T>;
+}
+
+/**
+ * A tree node in the object explorer tree
+ *
+ * @export
+ * @interface ITreeNode
+ */
+export interface ITreeNode {
+	getNodeInfo(): sqlops.NodeInfo;
+	getChildren(refreshChildren: boolean): ITreeNode[] | Promise<ITreeNode[]>;
+}
+
+/**
+ * A HDFS file node. This is a leaf node in the object explorer tree, and its contents
+ * can be queried
+ *
+ * @export
+ * @interface IFileNode
+ * @extends {ITreeNode}
+ */
+export interface IFileNode extends ITreeNode {
+	getFileContentsAsString(maxBytes?: number): Promise<string>;
+}

extensions/mssql/src/apiWrapper.ts

@@ -0,0 +1,119 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as vscode from 'vscode';
+import * as sqlops from 'sqlops';
+
+/**
+ * Wrapper class to act as a facade over VSCode and Data APIs and allow us to test / mock callbacks into
+ * this API from our code
+ *
+ * @export
+ * @class ApiWrapper
+ */
+export class ApiWrapper {
+	// Data APIs
+	public registerConnectionProvider(provider: sqlops.ConnectionProvider): vscode.Disposable {
+		return sqlops.dataprotocol.registerConnectionProvider(provider);
+	}
+
+	public registerObjectExplorerNodeProvider(provider: sqlops.ObjectExplorerNodeProvider): vscode.Disposable {
+		return sqlops.dataprotocol.registerObjectExplorerNodeProvider(provider);
+	}
+
+	public registerTaskServicesProvider(provider: sqlops.TaskServicesProvider): vscode.Disposable {
+		return sqlops.dataprotocol.registerTaskServicesProvider(provider);
+	}
+
+	public registerFileBrowserProvider(provider: sqlops.FileBrowserProvider): vscode.Disposable {
+		return sqlops.dataprotocol.registerFileBrowserProvider(provider);
+	}
+
+	public createDialog(title: string): sqlops.window.modelviewdialog.Dialog {
+		return sqlops.window.modelviewdialog.createDialog(title);
+	}
+
+	public openDialog(dialog: sqlops.window.modelviewdialog.Dialog): void {
+		return sqlops.window.modelviewdialog.openDialog(dialog);
+	}
+
+	public closeDialog(dialog: sqlops.window.modelviewdialog.Dialog): void {
+		return sqlops.window.modelviewdialog.closeDialog(dialog);
+	}
+
+	public registerTaskHandler(taskId: string, handler: (profile: sqlops.IConnectionProfile) => void): void {
+		sqlops.tasks.registerTask(taskId, handler);
+	}
+
+	public startBackgroundOperation(operationInfo: sqlops.BackgroundOperationInfo): void {
+		sqlops.tasks.startBackgroundOperation(operationInfo);
+	}
+
+	public getActiveConnections(): Thenable<sqlops.connection.Connection[]> {
+		return sqlops.connection.getActiveConnections();
+	}
+
+	// VSCode APIs
+	public executeCommand(command: string, ...rest: any[]): Thenable<any> {
+		return vscode.commands.executeCommand(command, ...rest);
+	}
+
+	public registerCommand(command: string, callback: (...args: any[]) => any, thisArg?: any): vscode.Disposable {
+		return vscode.commands.registerCommand(command, callback, thisArg);
+	}
+
+	public showErrorMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+		return vscode.window.showErrorMessage(message, ...items);
+	}
+
+	public showWarningMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+		return vscode.window.showWarningMessage(message, ...items);
+	}
+
+	public showInformationMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+		return vscode.window.showInformationMessage(message, ...items);
+	}
+
+	public showOpenDialog(options: vscode.OpenDialogOptions): Thenable<vscode.Uri[] | undefined> {
+		return vscode.window.showOpenDialog(options);
+	}
+
+	public showSaveDialog(options: vscode.SaveDialogOptions): Thenable<vscode.Uri> {
+		return vscode.window.showSaveDialog(options);
+	}
+
+	public openTextDocument(uri: vscode.Uri): Thenable<vscode.TextDocument>;
+	public openTextDocument(options: { language?: string; content?: string; }): Thenable<vscode.TextDocument>;
+	public openTextDocument(uriOrOptions): Thenable<vscode.TextDocument> {
+		return vscode.workspace.openTextDocument(uriOrOptions);
+	}
+
+	public showTextDocument(document: vscode.TextDocument, column?: vscode.ViewColumn, preserveFocus?: boolean, preview?: boolean): Thenable<vscode.TextEditor> {
+		let options: vscode.TextDocumentShowOptions = {
+			viewColumn: column,
+			preserveFocus: preserveFocus,
+			preview: preview
+		};
+		return vscode.window.showTextDocument(document, options);
+	}
+
+	public get workspaceFolders(): vscode.WorkspaceFolder[] {
+		return vscode.workspace.workspaceFolders;
+	}
+
+	public createStatusBarItem(alignment?: vscode.StatusBarAlignment, priority?: number): vscode.StatusBarItem {
+		return vscode.window.createStatusBarItem(alignment, priority);
+	}
+
+	public createOutputChannel(name: string): vscode.OutputChannel {
+		return vscode.window.createOutputChannel(name);
+	}
+
+	public createTab(title: string): sqlops.window.modelviewdialog.DialogTab {
+		return sqlops.window.modelviewdialog.createTab(title);
+	}
+}

extensions/mssql/src/appContext.ts

@@ -0,0 +1,28 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as vscode from 'vscode';
+import { ApiWrapper } from './apiWrapper';
+
+/**
+ * Global context for the application
+ */
+export class AppContext {
+
+	private serviceMap: Map<string, any> = new Map();
+	constructor(public readonly extensionContext: vscode.ExtensionContext, public readonly apiWrapper: ApiWrapper) {
+		this.apiWrapper = apiWrapper || new ApiWrapper();
+	}
+
+	public getService<T>(serviceName: string): T {
+		return this.serviceMap.get(serviceName) as T;
+	}
+
+	public registerService<T>(serviceName: string, service: T): void {
+		this.serviceMap.set(serviceName, service);
+	}
+}

extensions/mssql/src/config.json

@@ -1,6 +1,6 @@
 {
 	"downloadUrl": "https://github.com/Microsoft/sqltoolsservice/releases/download/v{#version#}/microsoft.sqltools.servicelayer-{#fileName#}",
-	"version": "1.5.0-alpha.63",
+	"version": "1.5.0-alpha.71",
 	"downloadFileNames": {
 		"Windows_86": "win-x86-netcoreapp2.2.zip",
 		"Windows_64": "win-x64-netcoreapp2.2.zip",

extensions/mssql/src/constants.ts

@@ -10,3 +10,65 @@ export const serviceCrashMessage = 'SQL Tools Service component exited unexpecte
 export const serviceCrashButton = 'View Known Issues';
 export const serviceCrashLink = 'https://github.com/Microsoft/vscode-mssql/wiki/SqlToolsService-Known-Issues';
 export const extensionConfigSectionName = 'mssql';
+
+// DATA PROTOCOL VALUES ///////////////////////////////////////////////////////////
+export const mssqlClusterProviderName = 'mssqlCluster';
+export const hadoopKnoxEndpointName = 'Knox';
+export const protocolVersion = '1.0';
+export const hostPropName = 'host';
+export const userPropName = 'user';
+export const knoxPortPropName = 'knoxport';
+export const passwordPropName = 'password';
+export const groupIdPropName = 'groupId';
+export const defaultKnoxPort = '30443';
+export const groupIdName = 'groupId';
+export const sqlProviderName = 'MSSQL';
+export const dataService = 'Data Services';
+
+export const hdfsHost = 'host';
+export const hdfsUser = 'user';
+export const UNTITLED_SCHEMA = 'untitled';
+
+export const hadoopConnectionTimeoutSeconds = 15;
+export const hdfsRootPath = '/';
+
+export const clusterEndpointsProperty = 'clusterEndpoints';
+export const isBigDataClusterProperty = 'isBigDataCluster';
+
+// SERVICE NAMES //////////////////////////////////////////////////////////
+export const ObjectExplorerService = 'objectexplorer';
+export const objectExplorerPrefix: string = 'objectexplorer://';
+export const ViewType = 'view';
+
+export enum BuiltInCommands {
+	SetContext = 'setContext'
+}
+
+export enum CommandContext {
+	WizardServiceEnabled = 'wizardservice:enabled'
+}
+
+export enum MssqlClusterItems {
+	Connection = 'mssqlCluster:connection',
+	Folder = 'mssqlCluster:folder',
+	File = 'mssqlCluster:file',
+	Error = 'mssqlCluster:error'
+}
+
+export enum MssqlClusterItemsSubType {
+	Spark = 'mssqlCluster:spark'
+}
+
+// SPARK JOB SUBMISSION //////////////////////////////////////////////////////////
+export const mssqlClusterNewNotebookTask = 'mssqlCluster.task.newNotebook';
+export const mssqlClusterOpenNotebookTask = 'mssqlCluster.task.openNotebook';
+export const mssqlClusterLivySubmitSparkJobCommand = 'mssqlCluster.livy.cmd.submitSparkJob';
+export const mssqlClusterLivySubmitSparkJobFromFileCommand = 'mssqlCluster.livy.cmd.submitFileToSparkJob';
+export const mssqlClusterLivySubmitSparkJobTask = 'mssqlCluster.livy.task.submitSparkJob';
+export const mssqlClusterLivyOpenSparkHistory = 'mssqlCluster.livy.task.openSparkHistory';
+export const mssqlClusterLivyOpenYarnHistory = 'mssqlCluster.livy.task.openYarnHistory';
+export const mssqlClusterLivySubmitPath = '/gateway/default/livy/v1/batches';
+export const mssqlClusterLivyTimeInMSForCheckYarnApp = 1000;
+export const mssqlClusterLivyRetryTimesForCheckYarnApp = 20;
+export const mssqlClusterSparkJobFileSelectorButtonWidth = '30px';
+export const mssqlClusterSparkJobFileSelectorButtonHeight = '30px';

extensions/mssql/src/contextProvider.ts

@@ -7,6 +7,7 @@ import * as vscode from 'vscode';
 import * as sqlops from 'sqlops';
 
 import * as types from './types';
+import * as Constants from './constants';
 
 export enum BuiltInCommands {
 	SetContext = 'setContext',
@@ -14,7 +15,9 @@ export enum BuiltInCommands {
 
 export enum ContextKeys {
 	ISCLOUD = 'mssql:iscloud',
-	EDITIONID = 'mssql:engineedition'
+	EDITIONID = 'mssql:engineedition',
+	ISCLUSTER = 'mssql:iscluster',
+	SERVERMAJORVERSION = 'mssql:servermajorversion'
 }
 
 const isCloudEditions = [
@@ -37,6 +40,8 @@ export default class ContextProvider {
 	public onDashboardOpen(e: sqlops.DashboardDocument): void {
 		let iscloud: boolean;
 		let edition: number;
+		let isCluster: boolean = false;
+		let serverMajorVersion: number;
 		if (e.profile.providerName.toLowerCase() === 'mssql' && !types.isUndefinedOrNull(e.serverInfo) && !types.isUndefinedOrNull(e.serverInfo.engineEditionId)) {
 			if (isCloudEditions.some(i => i === e.serverInfo.engineEditionId)) {
 				iscloud = true;
@@ -45,6 +50,14 @@ export default class ContextProvider {
 			}
 
 			edition = e.serverInfo.engineEditionId;
+
+			if (!types.isUndefinedOrNull(e.serverInfo.options)) {
+				let isBigDataCluster = e.serverInfo.options[Constants.isBigDataClusterProperty];
+				if (isBigDataCluster) {
+					isCluster = isBigDataCluster;
+				}
+			}
+			serverMajorVersion = e.serverInfo.serverMajorVersion;
 		}
 
 		if (iscloud === true || iscloud === false) {
@@ -54,6 +67,14 @@ export default class ContextProvider {
 		if (!types.isUndefinedOrNull(edition)) {
 			setCommandContext(ContextKeys.EDITIONID, edition);
 		}
+
+		if (!types.isUndefinedOrNull(isCluster)) {
+			setCommandContext(ContextKeys.ISCLUSTER, isCluster);
+		}
+
+		if (!types.isUndefinedOrNull(serverMajorVersion)) {
+			setCommandContext(ContextKeys.SERVERMAJORVERSION, serverMajorVersion);
+		}
 	}
 
 	dispose(): void {

extensions/mssql/src/contracts.ts

@@ -331,6 +331,21 @@ export interface DeployParams {
 	taskExecutionMode: TaskExecutionMode;
 }
 
+export interface GenerateDeployScriptParams {
+	packageFilePath: string;
+	databaseName: string;
+	scriptFilePath: string;
+	ownerUri: string;
+	taskExecutionMode: TaskExecutionMode;
+}
+
+export interface GenerateDeployPlanParams {
+	packageFilePath: string;
+	databaseName: string;
+	ownerUri: string;
+	taskExecutionMode: TaskExecutionMode;
+}
+
 export namespace ExportRequest {
 	export const type = new RequestType<ExportParams, sqlops.DacFxResult, void, void>('dacfx/export');
 }
@@ -347,4 +362,11 @@ export namespace DeployRequest {
 	export const type = new RequestType<DeployParams, sqlops.DacFxResult, void, void>('dacfx/deploy');
 }
 
+export namespace GenerateDeployScriptRequest {
+	export const type = new RequestType<GenerateDeployScriptParams, sqlops.DacFxResult, void, void>('dacfx/generateDeploymentScript');
+}
+
+export namespace GenerateDeployPlanRequest {
+	export const type = new RequestType<GenerateDeployPlanParams, sqlops.GenerateDeployPlanResult, void, void>('dacfx/generateDeployPlan');
+}
 // ------------------------------- < DacFx > ------------------------------------

extensions/mssql/src/escapeException.ts

@@ -0,0 +1,3 @@
+'use strict';
+
+export default require('error-ex')('EscapeException');

extensions/mssql/src/features.ts

@@ -106,12 +106,40 @@ export class DacFxServicesFeature extends SqlOpsFeature<undefined> {
 			);
 		};
 
+		let generateDeployScript = (packageFilePath: string, targetDatabaseName: string, scriptFilePath: string, ownerUri: string, taskExecutionMode: sqlops.TaskExecutionMode): Thenable<sqlops.DacFxResult> => {
+			let params: contracts.GenerateDeployScriptParams = { packageFilePath: packageFilePath, databaseName: targetDatabaseName, scriptFilePath: scriptFilePath, ownerUri: ownerUri, taskExecutionMode: taskExecutionMode };
+			return client.sendRequest(contracts.GenerateDeployScriptRequest.type, params).then(
+				r => {
+					return r;
+				},
+				e => {
+					client.logFailedRequest(contracts.GenerateDeployScriptRequest.type, e);
+					return Promise.resolve(undefined);
+				}
+			);
+		};
+
+		let generateDeployPlan = (packageFilePath: string, targetDatabaseName: string, ownerUri: string, taskExecutionMode: sqlops.TaskExecutionMode): Thenable<sqlops.GenerateDeployPlanResult> => {
+			let params: contracts.GenerateDeployPlanParams = { packageFilePath: packageFilePath, databaseName: targetDatabaseName, ownerUri: ownerUri, taskExecutionMode: taskExecutionMode };
+			return client.sendRequest(contracts.GenerateDeployPlanRequest.type, params).then(
+				r => {
+					return r;
+				},
+				e => {
+					client.logFailedRequest(contracts.GenerateDeployPlanRequest.type, e);
+					return Promise.resolve(undefined);
+				}
+			);
+		};
+
 		return sqlops.dataprotocol.registerDacFxServicesProvider({
 			providerId: client.providerId,
 			exportBacpac,
 			importBacpac,
 			extractDacpac,
-			deployDacpac
+			deployDacpac,
+			generateDeployScript,
+			generateDeployPlan
 		});
 	}
 }

extensions/mssql/src/localizedConstants.ts

@@ -0,0 +1,27 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+// HDFS Constants //////////////////////////////////////////////////////////
+export const msgMissingNodeContext = localize('msgMissingNodeContext', 'Node Command called without any node passed');
+
+// Spark Job Submission Constants //////////////////////////////////////////
+export const sparkLocalFileDestinationHint = localize('sparkJobSubmission_LocalFileDestinationHint', 'Local file will be uploaded to HDFS. ');
+export const sparkJobSubmissionEndMessage = localize('sparkJobSubmission_SubmissionEndMessage', '.......................... Submit Spark Job End ............................');
+export function sparkJobSubmissionPrepareUploadingFile(localPath: string, clusterFolder: string): string { return localize('sparkJobSubmission_PrepareUploadingFile', 'Uploading file from local {0} to HDFS folder: {1}', localPath, clusterFolder); }
+export const sparkJobSubmissionUploadingFileSucceeded = localize('sparkJobSubmission_UploadingFileSucceeded', 'Upload file to cluster Succeeded!');
+export function sparkJobSubmissionUploadingFileFailed(err: string): string { return localize('sparkJobSubmission_UploadingFileFailed', 'Upload file to cluster Failed. {0}', err); }
+export function sparkJobSubmissionPrepareSubmitJob(jobName: string): string { return localize('sparkJobSubmission_PrepareSubmitJob', 'Submitting job {0} ... ', jobName); }
+export const sparkJobSubmissionSparkJobHasBeenSubmitted = localize('sparkJobSubmission_SubmitJobFinished', 'The Spark Job has been submitted.');
+export function sparkJobSubmissionSubmitJobFailed(err: string): string { return localize('sparkJobSubmission_SubmitJobFailed', 'Spark Job Submission Failed. {0} ', err); }
+export function sparkJobSubmissionYarnUIMessage(yarnUIURL: string): string { return localize('sparkJobSubmission_YarnUIMessage', 'YarnUI Url: {0} ', yarnUIURL); }
+export function sparkJobSubmissionSparkHistoryLinkMessage(sparkHistoryLink: string): string { return localize('sparkJobSubmission_SparkHistoryLinkMessage', 'Spark History Url: {0} ', sparkHistoryLink); }
+export function sparkJobSubmissionGetApplicationIdFailed(err: string): string { return localize('sparkJobSubmission_GetApplicationIdFailed', 'Get Application Id Failed. {0}', err); }
+export function sparkJobSubmissionLocalFileNotExisted(path: string): string { return localize('sparkJobSubmission_LocalFileNotExisted', 'Local file {0} does not existed. ', path); }
+export const sparkJobSubmissionNoSqlBigDataClusterFound = localize('sparkJobSubmission_NoSqlBigDataClusterFound','No Sql Server Big Data Cluster found.');

extensions/mssql/src/main.ts

@@ -5,7 +5,12 @@
 'use strict';
 
 import * as vscode from 'vscode';
+import * as sqlops from 'sqlops';
 import * as path from 'path';
+import * as os from 'os';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
 import { SqlOpsDataClient, ClientOptions } from 'dataprotocol-client';
 import { IConfig, ServerProvider, Events } from 'service-downloader';
 import { ServerOptions, TransportKind } from 'vscode-languageclient';
@@ -17,12 +22,25 @@ import { AzureResourceProvider } from './resourceProvider/resourceProvider';
 import * as Utils from './utils';
 import { Telemetry, LanguageClientErrorHandler } from './telemetry';
 import { TelemetryFeature, AgentServicesFeature, DacFxServicesFeature } from './features';
+import { AppContext } from './appContext';
+import { ApiWrapper } from './apiWrapper';
+import { UploadFilesCommand, MkDirCommand, SaveFileCommand, PreviewFileCommand, CopyPathCommand, DeleteFilesCommand } from './objectExplorerNodeProvider/hdfsCommands';
+import { IPrompter } from './prompts/question';
+import CodeAdapter from './prompts/adapter';
+import { MssqlExtensionApi, MssqlObjectExplorerBrowser } from './api/mssqlapis';
+import { OpenSparkJobSubmissionDialogCommand, OpenSparkJobSubmissionDialogFromFileCommand, OpenSparkJobSubmissionDialogTask } from './sparkFeature/dialog/dialogCommands';
+import { OpenSparkYarnHistoryTask } from './sparkFeature/historyTask';
+import { MssqlObjectExplorerNodeProvider, mssqlOutputChannel } from './objectExplorerNodeProvider/objectExplorerNodeProvider';
 
 const baseConfig = require('./config.json');
 const outputChannel = vscode.window.createOutputChannel(Constants.serviceName);
 const statusView = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left);
+const jupyterNotebookProviderId = 'jupyter';
+const msgSampleCodeDataFrame = localize('msgSampleCodeDataFrame', 'This sample code loads the file into a data frame and shows the first 10 results.');
 
-export async function activate(context: vscode.ExtensionContext) {
+let untitledCounter = 0;
+
+export async function activate(context: vscode.ExtensionContext): Promise<MssqlExtensionApi> {
 	// lets make sure we support this platform first
 	let supported = await Utils.verifyPlatform();
 
@@ -61,6 +79,9 @@ export async function activate(context: vscode.ExtensionContext) {
 		outputChannel: new CustomOutputChannel()
 	};
 
+	let prompter: IPrompter = new CodeAdapter();
+	let appContext = new AppContext(context, new ApiWrapper());
+
 	const installationStart = Date.now();
 	serverdownloader.getOrDownloadServer().then(e => {
 		const installationComplete = Date.now();
@@ -85,6 +106,11 @@ export async function activate(context: vscode.ExtensionContext) {
 		languageClient.start();
 		credentialsStore.start();
 		resourceProvider.start();
+
+		let nodeProvider = new MssqlObjectExplorerNodeProvider(appContext);
+		sqlops.dataprotocol.registerObjectExplorerNodeProvider(nodeProvider);
+		activateSparkFeatures(appContext);
+		activateNotebookTask(appContext);
 	}, e => {
 		Telemetry.sendTelemetryEvent('ServiceInitializingFailed');
 		vscode.window.showErrorMessage('Failed to start Sql tools service');
@@ -94,7 +120,112 @@ export async function activate(context: vscode.ExtensionContext) {
 	context.subscriptions.push(contextProvider);
 	context.subscriptions.push(credentialsStore);
 	context.subscriptions.push(resourceProvider);
+	context.subscriptions.push(new UploadFilesCommand(prompter, appContext));
+	context.subscriptions.push(new MkDirCommand(prompter, appContext));
+	context.subscriptions.push(new SaveFileCommand(prompter, appContext));
+	context.subscriptions.push(new PreviewFileCommand(prompter, appContext));
+	context.subscriptions.push(new CopyPathCommand(appContext));
+	context.subscriptions.push(new DeleteFilesCommand(prompter, appContext));
 	context.subscriptions.push({ dispose: () => languageClient.stop() });
+
+	let api: MssqlExtensionApi = {
+		getMssqlObjectExplorerBrowser(): MssqlObjectExplorerBrowser {
+			return {
+				getNode: (context: sqlops.ObjectExplorerContext) => {
+					let oeProvider = appContext.getService<MssqlObjectExplorerNodeProvider>(Constants.ObjectExplorerService);
+					return <any>oeProvider.findSqlClusterNodeByContext(context);
+				}
+			};
+		}
+	};
+	return api;
+}
+
+function activateSparkFeatures(appContext: AppContext): void {
+	let extensionContext = appContext.extensionContext;
+	let apiWrapper = appContext.apiWrapper;
+	let outputChannel: vscode.OutputChannel = mssqlOutputChannel;
+	extensionContext.subscriptions.push(new OpenSparkJobSubmissionDialogCommand(appContext, outputChannel));
+	extensionContext.subscriptions.push(new OpenSparkJobSubmissionDialogFromFileCommand(appContext, outputChannel));
+	apiWrapper.registerTaskHandler(Constants.mssqlClusterLivySubmitSparkJobTask, (profile: sqlops.IConnectionProfile) => {
+		new OpenSparkJobSubmissionDialogTask(appContext, outputChannel).execute(profile);
+	});
+	apiWrapper.registerTaskHandler(Constants.mssqlClusterLivyOpenSparkHistory, (profile: sqlops.IConnectionProfile) => {
+		new OpenSparkYarnHistoryTask(appContext).execute(profile, true);
+	});
+	apiWrapper.registerTaskHandler(Constants.mssqlClusterLivyOpenYarnHistory, (profile: sqlops.IConnectionProfile) => {
+		new OpenSparkYarnHistoryTask(appContext).execute(profile, false);
+	});
+}
+
+function activateNotebookTask(appContext: AppContext): void {
+	let apiWrapper = appContext.apiWrapper;
+	apiWrapper.registerTaskHandler(Constants.mssqlClusterNewNotebookTask, (profile: sqlops.IConnectionProfile) => {
+		return saveProfileAndCreateNotebook(profile);
+	});
+	apiWrapper.registerTaskHandler(Constants.mssqlClusterOpenNotebookTask, (profile: sqlops.IConnectionProfile) => {
+		return handleOpenNotebookTask(profile);
+	});
+}
+
+function saveProfileAndCreateNotebook(profile: sqlops.IConnectionProfile): Promise<void> {
+	return handleNewNotebookTask(undefined, profile);
+}
+
+async function handleNewNotebookTask(oeContext?: sqlops.ObjectExplorerContext, profile?: sqlops.IConnectionProfile): Promise<void> {
+	// Ensure we get a unique ID for the notebook. For now we're using a different prefix to the built-in untitled files
+	// to handle this. We should look into improving this in the future
+	let untitledUri = vscode.Uri.parse(`untitled:Notebook-${untitledCounter++}`);
+	let editor = await sqlops.nb.showNotebookDocument(untitledUri, {
+		connectionId: profile.id,
+		providerId: jupyterNotebookProviderId,
+		preview: false,
+		defaultKernel: {
+			name: 'pyspark3kernel',
+			display_name: 'PySpark3',
+			language: 'python'
+		}
+	});
+	if (oeContext && oeContext.nodeInfo && oeContext.nodeInfo.nodePath) {
+		// Get the file path after '/HDFS'
+		let hdfsPath: string = oeContext.nodeInfo.nodePath.substring(oeContext.nodeInfo.nodePath.indexOf('/HDFS') + '/HDFS'.length);
+		if (hdfsPath.length > 0) {
+			let analyzeCommand = "#" + msgSampleCodeDataFrame + os.EOL + "df = (spark.read.option(\"inferSchema\", \"true\")"
+				+ os.EOL + ".option(\"header\", \"true\")" + os.EOL + ".csv('{0}'))" + os.EOL + "df.show(10)";
+			editor.edit(editBuilder => {
+				editBuilder.replace(0, {
+					cell_type: 'code',
+					source: analyzeCommand.replace('{0}', hdfsPath)
+				});
+			});
+
+		}
+	}
+}
+
+async function handleOpenNotebookTask(profile: sqlops.IConnectionProfile): Promise<void> {
+	let notebookFileTypeName = localize('notebookFileType', 'Notebooks');
+	let filter = {};
+	filter[notebookFileTypeName] = 'ipynb';
+	let uris = await vscode.window.showOpenDialog({
+		filters: filter,
+		canSelectFiles: true,
+		canSelectMany: false
+	});
+	if (uris && uris.length > 0) {
+		let fileUri = uris[0];
+		// Verify this is a .ipynb file since this isn't actually filtered on Mac/Linux
+		if (path.extname(fileUri.fsPath) !== '.ipynb') {
+			// in the future might want additional supported types
+			vscode.window.showErrorMessage(localize('unsupportedFileType', 'Only .ipynb Notebooks are supported'));
+		} else {
+			await sqlops.nb.showNotebookDocument(fileUri, {
+				connectionId: profile.id,
+				providerId: jupyterNotebookProviderId,
+				preview: false
+			});
+		}
+	}
 }
 
 function generateServerOptions(executablePath: string): ServerOptions {

extensions/mssql/src/objectExplorerNodeProvider/cancelableStream.ts

@@ -0,0 +1,27 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { Transform } from 'stream';
+import * as vscode from 'vscode';
+import * as nls from 'vscode-nls';
+
+const localize = nls.loadMessageBundle();
+
+export class CancelableStream extends Transform {
+	constructor(private cancelationToken: vscode.CancellationTokenSource) {
+		super();
+	}
+
+	public _transform(chunk: any, encoding: string, callback: Function): void {
+		if (this.cancelationToken && this.cancelationToken.token.isCancellationRequested) {
+			callback(new Error(localize('streamCanceled', 'Stream operation canceled by the user')));
+		} else {
+			this.push(chunk);
+			callback();
+		}
+	}
+}

extensions/mssql/src/objectExplorerNodeProvider/command.ts

@@ -0,0 +1,176 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+import * as vscode from 'vscode';
+import * as sqlops from 'sqlops';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+import { ApiWrapper } from '../apiWrapper';
+import { TreeNode } from './treeNodes';
+import { QuestionTypes, IPrompter, IQuestion } from '../prompts/question';
+import * as utils from '../utils';
+import * as constants from '../constants';
+import { AppContext } from '../appContext';
+
+export interface ICommandContextParsingOptions {
+	editor: boolean;
+	uri: boolean;
+}
+
+export interface ICommandBaseContext {
+	command: string;
+	editor?: vscode.TextEditor;
+	uri?: vscode.Uri;
+}
+
+export interface ICommandUnknownContext extends ICommandBaseContext {
+	type: 'unknown';
+}
+
+export interface ICommandUriContext extends ICommandBaseContext {
+	type: 'uri';
+}
+
+export interface ICommandViewContext extends ICommandBaseContext {
+	type: 'view';
+	node: TreeNode;
+}
+
+export interface ICommandObjectExplorerContext extends ICommandBaseContext {
+	type: 'objectexplorer';
+	explorerContext: sqlops.ObjectExplorerContext;
+}
+
+export type CommandContext = ICommandObjectExplorerContext | ICommandViewContext | ICommandUriContext | ICommandUnknownContext;
+
+function isTextEditor(editor: any): editor is vscode.TextEditor {
+	if (editor === undefined) { return false; }
+
+	return editor.id !== undefined && ((editor as vscode.TextEditor).edit !== undefined || (editor as vscode.TextEditor).document !== undefined);
+}
+
+export abstract class Command extends vscode.Disposable {
+
+
+	protected readonly contextParsingOptions: ICommandContextParsingOptions = { editor: false, uri: false };
+
+	private disposable: vscode.Disposable;
+
+	constructor(command: string | string[], protected appContext: AppContext) {
+		super(() => this.dispose());
+
+		if (typeof command === 'string') {
+			this.disposable = this.apiWrapper.registerCommand(command, (...args: any[]) => this._execute(command, ...args), this);
+
+			return;
+		}
+
+		const subscriptions = command.map(cmd => this.apiWrapper.registerCommand(cmd, (...args: any[]) => this._execute(cmd, ...args), this));
+		this.disposable = vscode.Disposable.from(...subscriptions);
+	}
+
+	dispose(): void {
+		this.disposable && this.disposable.dispose();
+	}
+
+	protected get apiWrapper(): ApiWrapper {
+		return this.appContext.apiWrapper;
+	}
+
+	protected async preExecute(...args: any[]): Promise<any> {
+		return this.execute(...args);
+	}
+
+	abstract execute(...args: any[]): any;
+
+	protected _execute(command: string, ...args: any[]): any {
+		// TODO consider using Telemetry.trackEvent(command);
+
+		const [context, rest] = Command.parseContext(command, this.contextParsingOptions, ...args);
+		return this.preExecute(context, ...rest);
+	}
+
+	private static parseContext(command: string, options: ICommandContextParsingOptions, ...args: any[]): [CommandContext, any[]] {
+		let editor: vscode.TextEditor | undefined = undefined;
+
+		let firstArg = args[0];
+		if (options.editor && (firstArg === undefined || isTextEditor(firstArg))) {
+			editor = firstArg;
+			args = args.slice(1);
+			firstArg = args[0];
+		}
+
+		if (options.uri && (firstArg === undefined || firstArg instanceof vscode.Uri)) {
+			const [uri, ...rest] = args as [vscode.Uri, any];
+			return [{ command: command, type: 'uri', editor: editor, uri: uri }, rest];
+		}
+
+		if (firstArg instanceof TreeNode) {
+			const [node, ...rest] = args as [TreeNode, any];
+			return [{ command: command, type: constants.ViewType, node: node }, rest];
+		}
+
+		if (firstArg && utils.isObjectExplorerContext(firstArg)) {
+			const [explorerContext, ...rest] = args as [sqlops.ObjectExplorerContext, any];
+			return [{ command: command, type: constants.ObjectExplorerService, explorerContext: explorerContext }, rest];
+		}
+
+		return [{ command: command, type: 'unknown', editor: editor }, args];
+	}
+}
+
+export abstract class ProgressCommand extends Command {
+	static progressId = 0;
+	constructor(private command: string, protected prompter: IPrompter, appContext: AppContext) {
+		super(command, appContext);
+	}
+
+	protected async executeWithProgress(
+		execution: (cancelToken: vscode.CancellationTokenSource) => Promise<void>,
+		label: string,
+		isCancelable: boolean = false,
+		onCanceled?: () => void
+	): Promise<void> {
+		let disposables: vscode.Disposable[] = [];
+		const tokenSource = new vscode.CancellationTokenSource();
+		const statusBarItem = this.apiWrapper.createStatusBarItem(vscode.StatusBarAlignment.Left);
+		disposables.push(vscode.Disposable.from(statusBarItem));
+		statusBarItem.text = localize('progress', '$(sync~spin) {0}...', label);
+		if (isCancelable) {
+			const cancelCommandId = `cancelProgress${ProgressCommand.progressId++}`;
+			disposables.push(this.apiWrapper.registerCommand(cancelCommandId, async () => {
+				if (await this.confirmCancel()) {
+					tokenSource.cancel();
+				}
+			}));
+			statusBarItem.tooltip = localize('cancelTooltip', 'Cancel');
+			statusBarItem.command = cancelCommandId;
+		}
+		statusBarItem.show();
+
+		try {
+			await execution(tokenSource);
+		} catch (error) {
+			if (isCancelable && onCanceled && tokenSource.token.isCancellationRequested) {
+				// The error can be assumed to be due to cancelation occurring. Do the callback
+				onCanceled();
+			} else {
+				throw error;
+			}
+		} finally {
+			disposables.forEach(d => d.dispose());
+		}
+	}
+
+	private async confirmCancel(): Promise<boolean> {
+		return await this.prompter.promptSingle<boolean>(<IQuestion>{
+			type: QuestionTypes.confirm,
+			message: localize('cancel', 'Cancel operation?'),
+			default: true
+		});
+	}
+}

extensions/mssql/src/objectExplorerNodeProvider/connection.ts

@@ -0,0 +1,114 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+import * as constants from '../constants';
+import { IFileSource, IHdfsOptions, IRequestParams, FileSourceFactory } from './fileSources';
+
+export class SqlClusterConnection {
+	private _connection: sqlops.connection.Connection;
+	private _profile: sqlops.IConnectionProfile;
+	private _host: string;
+	private _port: string;
+	private _user: string;
+	private _password: string;
+
+	constructor(connectionInfo: sqlops.connection.Connection | sqlops.IConnectionProfile) {
+		this.validate(connectionInfo);
+
+		if ('id' in connectionInfo) {
+			this._profile = connectionInfo;
+			this._connection = this.toConnection(this._profile);
+		} else {
+			this._connection = connectionInfo;
+			this._profile = this.toConnectionProfile(this._connection);
+		}
+		this._host = this._connection.options[constants.hostPropName];
+		this._port = this._connection.options[constants.knoxPortPropName];
+		this._user = this._connection.options[constants.userPropName];
+		this._password = this._connection.options[constants.passwordPropName];
+	}
+
+	public get connection(): sqlops.connection.Connection { return this._connection; }
+	public get profile(): sqlops.IConnectionProfile { return this._profile; }
+	public get host(): string { return this._host; }
+	public get port(): string { return this._port || constants.defaultKnoxPort; }
+	public get user(): string { return this._user; }
+	public get password(): string { return this._password; }
+
+	public isMatch(connection: SqlClusterConnection | sqlops.ConnectionInfo): boolean {
+		if (!connection) { return false; }
+		let options1 = connection instanceof SqlClusterConnection ?
+			connection._connection.options : connection.options;
+		let options2 = this._connection.options;
+		return [constants.hostPropName, constants.knoxPortPropName, constants.userPropName]
+			.every(e => options1[e] === options2[e]);
+	}
+
+	public createHdfsFileSource(): IFileSource {
+		let options: IHdfsOptions = {
+			protocol: 'https',
+			host: this.host,
+			port: this.port,
+			user: this.user,
+			path: 'gateway/default/webhdfs/v1',
+			requestParams: {
+				auth: {
+					user: this.user,
+					pass: this.password
+				}
+			}
+		};
+		return FileSourceFactory.instance.createHdfsFileSource(options);
+	}
+
+	private validate(connectionInfo: sqlops.ConnectionInfo): void {
+		if (!connectionInfo) {
+			throw new Error(localize('connectionInfoUndefined', 'ConnectionInfo is undefined.'));
+		}
+		if (!connectionInfo.options) {
+			throw new Error(localize('connectionInfoOptionsUndefined', 'ConnectionInfo.options is undefined.'));
+		}
+		let missingProperties: string[] = this.getMissingProperties(connectionInfo);
+		if (missingProperties && missingProperties.length > 0) {
+			throw new Error(localize('connectionInfoOptionsMissingProperties',
+				'Some missing properties in connectionInfo.options: {0}',
+				missingProperties.join(', ')));
+		}
+	}
+
+	private getMissingProperties(connectionInfo: sqlops.ConnectionInfo): string[] {
+		if (!connectionInfo || !connectionInfo.options) { return undefined; }
+		return [
+			constants.hostPropName, constants.knoxPortPropName,
+			constants.userPropName, constants.passwordPropName
+		].filter(e => connectionInfo.options[e] === undefined);
+	}
+
+	private toConnection(connProfile: sqlops.IConnectionProfile): sqlops.connection.Connection {
+		let connection: sqlops.connection.Connection = Object.assign(connProfile,
+			{ connectionId: this._profile.id });
+		return connection;
+	}
+
+	private toConnectionProfile(connectionInfo: sqlops.connection.Connection): sqlops.IConnectionProfile {
+		let options = connectionInfo.options;
+		let connProfile: sqlops.IConnectionProfile = Object.assign(<sqlops.IConnectionProfile>{},
+			connectionInfo,
+			{
+				serverName: `${options[constants.hostPropName]},${options[constants.knoxPortPropName]}`,
+				userName: options[constants.userPropName],
+				password: options[constants.passwordPropName],
+				id: connectionInfo.connectionId,
+			}
+		);
+		return connProfile;
+	}
+}

extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts

@@ -0,0 +1,371 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+import * as fspath from 'path';
+import * as webhdfs from 'webhdfs';
+import * as fs from 'fs';
+import * as meter from 'stream-meter';
+import * as bytes from 'bytes';
+import * as https from 'https';
+import * as readline from 'readline';
+import * as os from 'os';
+
+import * as constants from '../constants';
+import * as utils from '../utils';
+
+export function joinHdfsPath(parent: string, child: string): string {
+	if (parent === constants.hdfsRootPath) {
+		return `/${child}`;
+	}
+	return `${parent}/${child}`;
+}
+
+export interface IFile {
+	path: string;
+	isDirectory: boolean;
+}
+
+export class File implements IFile {
+	constructor(public path: string, public isDirectory: boolean) {
+
+	}
+
+	public static createPath(path: string, fileName: string): string {
+		return joinHdfsPath(path, fileName);
+	}
+
+	public static createChild(parent: IFile, fileName: string, isDirectory: boolean): IFile {
+		return new File(File.createPath(parent.path, fileName), isDirectory);
+	}
+
+	public static createFile(parent: IFile, fileName: string): File {
+		return File.createChild(parent, fileName, false);
+	}
+
+	public static createDirectory(parent: IFile, fileName: string): IFile {
+		return File.createChild(parent, fileName, true);
+	}
+
+	public static getBasename(file: IFile): string {
+		return fspath.basename(file.path);
+	}
+}
+
+export interface IFileSource {
+
+	enumerateFiles(path: string): Promise<IFile[]>;
+	mkdir(dirName: string, remoteBasePath: string): Promise<void>;
+	createReadStream(path: string): fs.ReadStream;
+	readFile(path: string, maxBytes?: number): Promise<Buffer>;
+	readFileLines(path: string, maxLines: number): Promise<Buffer>;
+	writeFile(localFile: IFile, remoteDir: string): Promise<string>;
+	delete(path: string, recursive?: boolean): Promise<void>;
+	exists(path: string): Promise<boolean>;
+}
+
+export interface IHttpAuthentication {
+	user: string;
+	pass: string;
+}
+export interface IHdfsOptions {
+	host?: string;
+	port?: string;
+	protocol?: string;
+	user?: string;
+	path?: string;
+	requestParams?: IRequestParams;
+}
+
+export interface IRequestParams {
+	auth?: IHttpAuthentication;
+	/**
+	 * Timeout in milliseconds to wait for response
+	 */
+	timeout?: number;
+	agent?: https.Agent;
+}
+
+export interface IHdfsFileStatus {
+	type: 'FILE' | 'DIRECTORY';
+	pathSuffix: string;
+}
+
+export interface IHdfsClient {
+	readdir(path: string, callback: (err: Error, files: any[]) => void): void;
+
+	/**
+	 * Create readable stream for given path
+	 *
+	 * @method createReadStream
+	 * @fires Request#data
+	 * @fires WebHDFS#finish
+	 *
+	 * @param {String} path
+	 * @param {Object} [opts]
+	 *
+	 * @returns {Object}
+	 */
+	createReadStream (path: string, opts?: object): fs.ReadStream;
+
+	/**
+	 * Create writable stream for given path
+	 *
+	 * @example
+	 *
+	 * var WebHDFS = require('webhdfs');
+	 * var hdfs = WebHDFS.createClient();
+	 *
+	 * var localFileStream = fs.createReadStream('/path/to/local/file');
+	 * var remoteFileStream = hdfs.createWriteStream('/path/to/remote/file');
+	 *
+	 * localFileStream.pipe(remoteFileStream);
+	 *
+	 * remoteFileStream.on('error', function onError (err) {
+	 *   // Do something with the error
+	 * });
+	 *
+	 * remoteFileStream.on('finish', function onFinish () {
+	 *  // Upload is done
+	 * });
+	 *
+	 * @method createWriteStream
+	 * @fires WebHDFS#finish
+	 *
+	 * @param {String} path
+	 * @param {Boolean} [append] If set to true then append data to the file
+	 * @param {Object} [opts]
+	 *
+	 * @returns {Object}
+	 */
+	createWriteStream(path: string, append?: boolean, opts?: object): fs.WriteStream;
+
+	/**
+	 * Make new directory
+	 *
+	 * @method mkdir
+	 *
+	 * @param {String} path
+	 * @param {String} [mode=0777]
+	 * @param {Function} callback
+	 *
+	 * @returns {Object}
+	 */
+	mkdir (path: string, callback: Function): void;
+	mkdir (path: string, mode: string, callback: Function): void;
+
+	/**
+	 * Delete directory or file path
+	 *
+	 * @method unlink
+	 *
+	 * @param {String} path
+	 * @param {Boolean} [recursive=false]
+	 * @param {Function} callback
+	 *
+	 * @returns {Object}
+	 */
+	rmdir (path: string, recursive: boolean, callback: Function): void;
+
+	/**
+	 * Check file existence
+	 * Wraps stat method
+	 *
+	 * @method stat
+	 * @see WebHDFS.stat
+	 *
+	 * @param {String} path
+	 * @param {Function} callback
+	 *
+	 * @returns {Object}
+	 */
+	exists (path: string, callback: Function): boolean;
+}
+
+export class FileSourceFactory {
+	private static _instance: FileSourceFactory;
+
+	public static get instance(): FileSourceFactory {
+		if (!FileSourceFactory._instance) {
+			FileSourceFactory._instance = new FileSourceFactory();
+		}
+		return FileSourceFactory._instance;
+	}
+
+	public createHdfsFileSource(options: IHdfsOptions): IFileSource {
+		options = options && options.host ? FileSourceFactory.removePortFromHost(options) : options;
+		let requestParams: IRequestParams = options.requestParams ? options.requestParams : {};
+		if (requestParams.auth) {
+			// TODO Remove handling of unsigned cert once we have real certs in our Knox service
+			let agentOptions = {
+				host: options.host,
+				port: options.port,
+				path: constants.hdfsRootPath,
+				rejectUnauthorized: false
+			  };
+			let agent = new https.Agent(agentOptions);
+			requestParams['agent'] = agent;
+		}
+		return new HdfsFileSource(webhdfs.createClient(options, requestParams));
+	}
+
+	// remove port from host when port is specified after a comma or colon
+	private static removePortFromHost(options: IHdfsOptions): IHdfsOptions {
+		// determine whether the host has either a ',' or ':' in it
+		options = this.setHostAndPort(options, ',');
+		options = this.setHostAndPort(options, ':');
+		return options;
+	}
+
+	// set port and host correctly after we've identified that a delimiter exists in the host name
+	private static setHostAndPort(options: IHdfsOptions, delimeter: string): IHdfsOptions {
+		let optionsHost: string = options.host;
+		if (options.host.indexOf(delimeter) > -1) {
+			options.host = options.host.slice(0, options.host.indexOf(delimeter));
+			options.port = optionsHost.replace(options.host + delimeter, '');
+		}
+		return options;
+	}
+}
+
+export class HdfsFileSource implements IFileSource {
+	constructor(private client: IHdfsClient) {
+	}
+
+	public enumerateFiles(path: string): Promise<IFile[]> {
+		return new Promise((resolve, reject) => {
+			this.client.readdir(path, (error, files) => {
+				if (error) {
+					reject(error.message);
+				} else {
+					let hdfsFiles: IFile[] = files.map(file => {
+						let hdfsFile = <IHdfsFileStatus> file;
+						return new File(File.createPath(path, hdfsFile.pathSuffix), hdfsFile.type === 'DIRECTORY');
+					});
+					resolve(hdfsFiles);
+				}
+			});
+		});
+	}
+
+	public mkdir(dirName: string, remoteBasePath: string): Promise<void> {
+		return new Promise((resolve, reject) => {
+			let remotePath = joinHdfsPath(remoteBasePath, dirName);
+			this.client.mkdir(remotePath, (err) => {
+				if (err) {
+					reject(err);
+				} else {
+					resolve(undefined);
+				}
+			});
+		});
+	}
+
+	public createReadStream(path: string): fs.ReadStream {
+		return this.client.createReadStream(path);
+	}
+
+	public readFile(path: string, maxBytes?: number): Promise<Buffer> {
+		return new Promise((resolve, reject) => {
+			let remoteFileStream = this.client.createReadStream(path);
+			if (maxBytes) {
+				remoteFileStream = remoteFileStream.pipe(meter(maxBytes));
+			}
+			let data = [];
+			let error = undefined;
+			remoteFileStream.on('error', (err) => {
+				error = err.toString();
+				if (error.includes('Stream exceeded specified max')) {
+					error = `File exceeds max size of ${bytes(maxBytes)}`;
+				}
+				reject(error);
+			});
+
+			remoteFileStream.on('data', (chunk) => {
+				data.push(chunk);
+			});
+
+			remoteFileStream.once('finish', () => {
+				if (!error) {
+					resolve(Buffer.concat(data));
+				}
+			});
+		});
+	}
+
+	public readFileLines(path: string, maxLines: number): Promise<Buffer> {
+		return new Promise((resolve, reject) => {
+			let lineReader = readline.createInterface({
+				input: this.client.createReadStream(path)
+			});
+
+			let lineCount = 0;
+			let lineData: string[] = [];
+			let errorMsg = undefined;
+			lineReader.on('line', (line: string) => {
+				lineCount++;
+				lineData.push(line);
+				if (lineCount >= maxLines) {
+					resolve(Buffer.from(lineData.join(os.EOL)));
+					lineReader.close();
+				}
+			})
+			.on('error', (err) => {
+				errorMsg = utils.getErrorMessage(err);
+				reject(errorMsg);
+			})
+			.on('close', () => {
+				if (!errorMsg) {
+					resolve(Buffer.from(lineData.join(os.EOL)));
+				}
+			});
+		});
+	}
+
+	public writeFile(localFile: IFile, remoteDirPath: string): Promise<string> {
+		return new Promise((resolve, reject) => {
+			let fileName = fspath.basename(localFile.path);
+			let remotePath = joinHdfsPath(remoteDirPath, fileName);
+
+			let writeStream = this.client.createWriteStream(remotePath);
+
+			let readStream = fs.createReadStream(localFile.path);
+			readStream.pipe(writeStream);
+
+			let error: string | Error = undefined;
+
+			// API always calls finish, so catch error then handle exit in the finish event
+			writeStream.on('error', (err => {
+				error = err;
+				reject(error);
+			}));
+			writeStream.on('finish', (location) => {
+				if (!error) {
+					resolve(location);
+				}
+			});
+		});
+	}
+
+	public delete(path: string, recursive: boolean = false): Promise<void> {
+		return new Promise((resolve, reject) => {
+			this.client.rmdir(path, recursive, (error) => {
+				if (error) {
+					reject(error);
+				} else {
+					resolve(undefined);
+				}
+			});
+		});
+	}
+
+	public exists(path: string): Promise<boolean> {
+		return new Promise((resolve, reject) => {
+			this.client.exists(path, (result) => {
+				resolve(result);
+			});
+		});
+	}
+}

extensions/mssql/src/objectExplorerNodeProvider/hdfsCommands.ts

@@ -0,0 +1,363 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as vscode from 'vscode';
+import * as sqlops from 'sqlops';
+import * as fs from 'fs';
+import * as fspath from 'path';
+import * as clipboardy from 'clipboardy';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+import { ApiWrapper } from '../apiWrapper';
+import { Command, ICommandViewContext, ProgressCommand, ICommandObjectExplorerContext } from './command';
+import { IHdfsOptions, HdfsFileSource, File, IFile, joinHdfsPath, FileSourceFactory } from './fileSources';
+import { HdfsProvider, FolderNode, FileNode, HdfsFileSourceNode } from './hdfsProvider';
+import { IPrompter, IQuestion, QuestionTypes } from '../prompts/question';
+import * as constants from '../constants';
+import * as LocalizedConstants from '../localizedConstants';
+import * as utils from '../utils';
+import { SqlClusterConnection } from './connection';
+import { AppContext } from '../appContext';
+import { TreeNode } from './treeNodes';
+import { MssqlObjectExplorerNodeProvider } from './objectExplorerNodeProvider';
+
+function getSaveableUri(apiWrapper: ApiWrapper, fileName: string, isPreview?: boolean): vscode.Uri {
+	let root = utils.getUserHome();
+	let workspaceFolders = apiWrapper.workspaceFolders;
+	if (workspaceFolders && workspaceFolders.length > 0) {
+		root = workspaceFolders[0].uri.fsPath;
+	}
+	// Cannot preview with a file path that already exists, so keep looking for a valid path that does not exist
+	if (isPreview) {
+		let fileNum = 1;
+		let fileNameWithoutExtension = fspath.parse(fileName).name;
+		let fileExtension = fspath.parse(fileName).ext;
+		while (fs.existsSync(fspath.join(root, fileName))) {
+			fileName = `${fileNameWithoutExtension}-${fileNum}${fileExtension}`;
+			fileNum++;
+		}
+	}
+	return vscode.Uri.file(fspath.join(root, fileName));
+}
+
+export async function getNode<T extends TreeNode>(context: ICommandViewContext | ICommandObjectExplorerContext, appContext: AppContext): Promise<T> {
+	let node: T = undefined;
+	if (context && context.type === constants.ViewType && context.node) {
+		node = context.node as T;
+	} else if (context && context.type === constants.ObjectExplorerService) {
+		let oeNodeProvider = appContext.getService<MssqlObjectExplorerNodeProvider>(constants.ObjectExplorerService);
+		if (oeNodeProvider) {
+			node = await oeNodeProvider.findSqlClusterNodeByContext<T>(context);
+		}
+	} else {
+		throw new Error(LocalizedConstants.msgMissingNodeContext);
+	}
+	return node;
+}
+
+export class UploadFilesCommand extends ProgressCommand {
+
+	constructor(prompter: IPrompter, appContext: AppContext) {
+		super('mssqlCluster.uploadFiles', prompter, appContext);
+	}
+
+	protected async preExecute(context: ICommandViewContext | ICommandObjectExplorerContext, args: object = {}): Promise<any> {
+		return this.execute(context, args);
+	}
+
+	async execute(context: ICommandViewContext | ICommandObjectExplorerContext, ...args: any[]): Promise<void> {
+		try {
+			let folderNode = await getNode<FolderNode>(context, this.appContext);
+			const allFilesFilter = localize('allFiles', 'All Files');
+			let filter = {};
+			filter[allFilesFilter] = '*';
+			if (folderNode) {
+				let options: vscode.OpenDialogOptions = {
+					canSelectFiles: true,
+					canSelectFolders: false,
+					canSelectMany: true,
+					openLabel: localize('lblUploadFiles', 'Upload'),
+					filters: filter
+				};
+				let fileUris: vscode.Uri[] = await this.apiWrapper.showOpenDialog(options);
+				if (fileUris) {
+					let files: IFile[] = fileUris.map(uri => uri.fsPath).map(this.mapPathsToFiles());
+					await this.executeWithProgress(
+						async (cancelToken: vscode.CancellationTokenSource) => this.writeFiles(files, folderNode, cancelToken),
+						localize('uploading', 'Uploading files to HDFS'), true,
+						() => this.apiWrapper.showInformationMessage(localize('uploadCanceled', 'Upload operation was canceled')));
+					if (context.type === constants.ObjectExplorerService) {
+						let objectExplorerNode = await sqlops.objectexplorer.getNode(context.explorerContext.connectionProfile.id, folderNode.getNodeInfo().nodePath);
+						await objectExplorerNode.refresh();
+					}
+				}
+			}
+		} catch (err) {
+			this.apiWrapper.showErrorMessage(localize('uploadError', 'Error uploading files: {0}', utils.getErrorMessage(err)));
+		}
+	}
+
+	private mapPathsToFiles(): (value: string, index: number, array: string[]) => File {
+		return (path: string) => {
+			let isDir = fs.lstatSync(path).isDirectory();
+			return new File(path, isDir);
+		};
+	}
+
+	private async writeFiles(files: IFile[], folderNode: FolderNode, cancelToken: vscode.CancellationTokenSource): Promise<void> {
+		for (let file of files) {
+			if (cancelToken.token.isCancellationRequested) {
+				// Throw here so that all recursion is ended
+				throw new Error('Upload canceled');
+			}
+			if (file.isDirectory) {
+				let dirName = fspath.basename(file.path);
+				let subFolder = await folderNode.mkdir(dirName);
+				let children: IFile[] = fs.readdirSync(file.path)
+					.map(childFileName => joinHdfsPath(file.path, childFileName))
+					.map(this.mapPathsToFiles());
+				this.writeFiles(children, subFolder, cancelToken);
+			} else {
+				await folderNode.writeFile(file);
+			}
+		}
+	}
+}
+export class MkDirCommand extends ProgressCommand {
+
+	constructor(prompter: IPrompter, appContext: AppContext) {
+		super('mssqlCluster.mkdir', prompter, appContext);
+	}
+
+	protected async preExecute(context: ICommandViewContext | ICommandObjectExplorerContext, args: object = {}): Promise<any> {
+		return this.execute(context, args);
+	}
+
+	async execute(context: ICommandViewContext | ICommandObjectExplorerContext, ...args: any[]): Promise<void> {
+		try {
+			let folderNode = await getNode<FolderNode>(context, this.appContext);
+
+			if (folderNode) {
+				let fileName: string = await this.getDirName();
+				if (fileName && fileName.length > 0) {
+					await this.executeWithProgress(
+						async (cancelToken: vscode.CancellationTokenSource) => this.mkDir(fileName, folderNode, cancelToken),
+						localize('makingDir', 'Creating directory'), true,
+						() => this.apiWrapper.showInformationMessage(localize('mkdirCanceled', 'Operation was canceled')));
+					if (context.type === constants.ObjectExplorerService) {
+						let objectExplorerNode = await sqlops.objectexplorer.getNode(context.explorerContext.connectionProfile.id, folderNode.getNodeInfo().nodePath);
+						await objectExplorerNode.refresh();
+					}
+				}
+			}
+		} catch (err) {
+			this.apiWrapper.showErrorMessage(localize('uploadError', 'Error uploading files: {0}', utils.getErrorMessage(err)));
+		}
+	}
+
+	private async getDirName(): Promise<string> {
+		return await this.prompter.promptSingle(<IQuestion>{
+			type: QuestionTypes.input,
+			name: 'enterDirName',
+			message: localize('enterDirName', 'Enter directory name'),
+			default: ''
+		}).then(confirmed => <string>confirmed);
+	}
+
+	private async mkDir(fileName, folderNode: FolderNode, cancelToken: vscode.CancellationTokenSource): Promise<void> {
+		let subFolder = await folderNode.mkdir(fileName);
+	}
+}
+
+export class DeleteFilesCommand extends Command {
+
+	constructor(private prompter: IPrompter, appContext: AppContext) {
+		super('mssqlCluster.deleteFiles', appContext);
+	}
+
+	protected async preExecute(context: ICommandViewContext | ICommandObjectExplorerContext, args: object = {}): Promise<any> {
+		return this.execute(context, args);
+	}
+
+	async execute(context: ICommandViewContext | ICommandObjectExplorerContext, ...args: any[]): Promise<void> {
+		try {
+			let node = await getNode<TreeNode>(context, this.appContext);
+			if (node) {
+				// TODO ideally would let node define if it's deletable
+				// TODO also, would like to change this to getNodeInfo as OE is the primary use case now
+				let treeItem = await node.getTreeItem();
+				let oeNodeToRefresh: sqlops.objectexplorer.ObjectExplorerNode = undefined;
+				if (context.type === constants.ObjectExplorerService) {
+					let oeNodeToDelete = await sqlops.objectexplorer.getNode(context.explorerContext.connectionProfile.id, node.getNodeInfo().nodePath);
+					oeNodeToRefresh = await oeNodeToDelete.getParent();
+				}
+				switch (treeItem.contextValue) {
+					case constants.MssqlClusterItems.Folder:
+						await this.deleteFolder(<FolderNode>node);
+						break;
+					case constants.MssqlClusterItems.File:
+						await this.deleteFile(<FileNode>node);
+						break;
+					default:
+						return;
+				}
+				if (oeNodeToRefresh) {
+					await oeNodeToRefresh.refresh();
+				}
+			} else {
+				this.apiWrapper.showErrorMessage(LocalizedConstants.msgMissingNodeContext);
+			}
+		} catch (err) {
+			this.apiWrapper.showErrorMessage(localize('deleteError', 'Error deleting files {0}', utils.getErrorMessage(err)));
+		}
+	}
+
+	private async confirmDelete(deleteMsg: string): Promise<boolean> {
+		return await this.prompter.promptSingle(<IQuestion>{
+			type: QuestionTypes.confirm,
+			message: deleteMsg,
+			default: false
+		}).then(confirmed => <boolean>confirmed);
+	}
+
+	private async deleteFolder(node: FolderNode): Promise<void> {
+		if (node) {
+			let confirmed = await this.confirmDelete(localize('msgDeleteFolder', 'Are you sure you want to delete this folder and its contents?'));
+			if (confirmed) {
+				// TODO prompt for recursive delete if non-empty?
+				await node.delete(true);
+			}
+		}
+	}
+
+	private async deleteFile(node: FileNode): Promise<void> {
+		if (node) {
+			let confirmed = await this.confirmDelete(localize('msgDeleteFile', 'Are you sure you want to delete this file?'));
+			if (confirmed) {
+				await node.delete();
+			}
+		}
+	}
+}
+
+export class SaveFileCommand extends ProgressCommand {
+
+	constructor(prompter: IPrompter, appContext: AppContext) {
+		super('mssqlCluster.saveFile', prompter, appContext);
+	}
+
+	protected async preExecute(context: ICommandViewContext | ICommandObjectExplorerContext, args: object = {}): Promise<any> {
+		return this.execute(context, args);
+	}
+
+	async execute(context: ICommandViewContext | ICommandObjectExplorerContext, ...args: any[]): Promise<void> {
+		try {
+			let fileNode = await getNode<FileNode>(context, this.appContext);
+			if (fileNode) {
+				let defaultUri = getSaveableUri(this.apiWrapper, fspath.basename(fileNode.hdfsPath));
+				let fileUri: vscode.Uri = await this.apiWrapper.showSaveDialog({
+					defaultUri: defaultUri
+				});
+				if (fileUri) {
+					await this.executeWithProgress(
+						async (cancelToken: vscode.CancellationTokenSource) => this.doSaveAndOpen(fileUri, fileNode, cancelToken),
+						localize('saving', 'Saving HDFS Files'), true,
+						() => this.apiWrapper.showInformationMessage(localize('saveCanceled', 'Save operation was canceled')));
+				}
+			} else {
+				this.apiWrapper.showErrorMessage(LocalizedConstants.msgMissingNodeContext);
+			}
+		} catch (err) {
+			this.apiWrapper.showErrorMessage(localize('saveError', 'Error saving file: {0}', utils.getErrorMessage(err)));
+		}
+	}
+
+	private async doSaveAndOpen(fileUri: vscode.Uri, fileNode: FileNode, cancelToken: vscode.CancellationTokenSource): Promise<void> {
+		await fileNode.writeFileContentsToDisk(fileUri.fsPath, cancelToken);
+		await this.apiWrapper.executeCommand('vscode.open', fileUri);
+	}
+}
+
+export class PreviewFileCommand extends ProgressCommand {
+	public static readonly DefaultMaxSize = 30 * 1024 * 1024;
+
+	constructor(prompter: IPrompter, appContext: AppContext) {
+		super('mssqlCluster.previewFile', prompter, appContext);
+	}
+
+	protected async preExecute(context: ICommandViewContext | ICommandObjectExplorerContext, args: object = {}): Promise<any> {
+		return this.execute(context, args);
+	}
+
+	async execute(context: ICommandViewContext | ICommandObjectExplorerContext, ...args: any[]): Promise<void> {
+		try {
+			let fileNode = await getNode<FileNode>(context, this.appContext);
+			if (fileNode) {
+				await this.executeWithProgress(
+					async (cancelToken: vscode.CancellationTokenSource) => {
+						let contents = await fileNode.getFileContentsAsString(PreviewFileCommand.DefaultMaxSize);
+						let doc = await this.openTextDocument(fspath.basename(fileNode.hdfsPath));
+						let editor = await this.apiWrapper.showTextDocument(doc, vscode.ViewColumn.Active, false);
+						await editor.edit(edit => {
+							edit.insert(new vscode.Position(0, 0), contents);
+						});
+					},
+					localize('previewing', 'Generating preview'),
+					false);
+			} else {
+				this.apiWrapper.showErrorMessage(LocalizedConstants.msgMissingNodeContext);
+			}
+		} catch (err) {
+			this.apiWrapper.showErrorMessage(localize('previewError', 'Error previewing file: {0}', utils.getErrorMessage(err)));
+		}
+	}
+
+	private async openTextDocument(fileName: string): Promise<vscode.TextDocument> {
+		let docUri: vscode.Uri = getSaveableUri(this.apiWrapper, fileName, true);
+		if (docUri) {
+			docUri = docUri.with({ scheme: constants.UNTITLED_SCHEMA });
+			return await this.apiWrapper.openTextDocument(docUri);
+		} else {
+			// Can't reliably create a filename to save as so just use untitled
+			let language = fspath.extname(fileName);
+			if (language && language.length > 0) {
+				// trim the '.'
+				language = language.substring(1);
+			}
+			return await this.apiWrapper.openTextDocument({
+				language: language
+			});
+		}
+	}
+}
+
+export class CopyPathCommand extends Command {
+	public static readonly DefaultMaxSize = 30 * 1024 * 1024;
+
+	constructor(appContext: AppContext) {
+		super('mssqlCluster.copyPath', appContext);
+	}
+
+	protected async preExecute(context: ICommandViewContext | ICommandObjectExplorerContext, args: object = {}): Promise<any> {
+		return this.execute(context, args);
+	}
+
+	async execute(context: ICommandViewContext | ICommandObjectExplorerContext, ...args: any[]): Promise<void> {
+		try {
+			let node = await getNode<HdfsFileSourceNode>(context, this.appContext);
+			if (node) {
+				let path = node.hdfsPath;
+				clipboardy.writeSync(path);
+			} else {
+				this.apiWrapper.showErrorMessage(LocalizedConstants.msgMissingNodeContext);
+			}
+		} catch (err) {
+			this.apiWrapper.showErrorMessage(localize('copyPathError', 'Error copying path: {0}', utils.getErrorMessage(err)));
+		}
+	}
+}

extensions/mssql/src/objectExplorerNodeProvider/hdfsProvider.ts

@@ -0,0 +1,366 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+import * as fspath from 'path';
+import * as fs from 'fs';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+import { ApiWrapper } from '../apiWrapper';
+import * as Constants from '../constants';
+import { IFileSource, IHdfsOptions, HdfsFileSource, IFile, File, FileSourceFactory } from './fileSources';
+import { CancelableStream } from './cancelableStream';
+import { TreeNode } from './treeNodes';
+import * as utils from '../utils';
+import { IFileNode } from './types';
+
+export interface ITreeChangeHandler {
+	notifyNodeChanged(node: TreeNode): void;
+}
+export class TreeDataContext {
+
+	constructor(public extensionContext: vscode.ExtensionContext, public changeHandler: ITreeChangeHandler) {
+
+	}
+}
+
+export class HdfsProvider implements vscode.TreeDataProvider<TreeNode>, ITreeChangeHandler {
+	static readonly NoConnectionsMessage = 'No connections added';
+	static readonly ConnectionsLabel = 'Connections';
+
+	private connections: ConnectionNode[];
+	private _onDidChangeTreeData = new vscode.EventEmitter<TreeNode>();
+	private context: TreeDataContext;
+
+	constructor(extensionContext: vscode.ExtensionContext, private vscodeApi: ApiWrapper) {
+		this.connections = [];
+		this.context = new TreeDataContext(extensionContext, this);
+	}
+
+	public get onDidChangeTreeData(): vscode.Event<TreeNode> {
+		return this._onDidChangeTreeData.event;
+	}
+
+	getTreeItem(element: TreeNode): vscode.TreeItem | Thenable<vscode.TreeItem> {
+		return element.getTreeItem();
+	}
+
+	getChildren(element?: TreeNode): vscode.ProviderResult<TreeNode[]> {
+		if (element) {
+			return element.getChildren(false);
+		} else {
+			return this.connections.length > 0 ? this.connections : [ErrorNode.create(HdfsProvider.NoConnectionsMessage, element)];
+		}
+	}
+
+	addConnection(displayName: string, fileSource: IFileSource): void {
+		if (!this.connections.find(c => c.getDisplayName() === displayName)) {
+			this.connections.push(new ConnectionNode(this.context, displayName, fileSource));
+			this._onDidChangeTreeData.fire();
+		}
+	}
+
+	addHdfsConnection(options: IHdfsOptions): void {
+		let displayName = `${options.user}@${options.host}:${options.port}`;
+		let fileSource = FileSourceFactory.instance.createHdfsFileSource(options);
+		this.addConnection(displayName, fileSource);
+	}
+
+	notifyNodeChanged(node: TreeNode): void {
+		this._onDidChangeTreeData.fire(node);
+	}
+}
+
+export abstract class HdfsFileSourceNode extends TreeNode {
+	constructor(protected context: TreeDataContext, protected _path: string, protected fileSource: IFileSource) {
+		super();
+	}
+
+	public get hdfsPath(): string {
+		return this._path;
+	}
+
+	public get nodePathValue(): string {
+		return this.getDisplayName();
+	}
+
+	getDisplayName(): string {
+		return fspath.basename(this._path);
+	}
+
+	public async delete(recursive: boolean = false): Promise<void> {
+		await this.fileSource.delete(this.hdfsPath, recursive);
+		// Notify parent should be updated. If at top, will return undefined which will refresh whole tree
+		(<HdfsFileSourceNode>this.parent).onChildRemoved();
+		this.context.changeHandler.notifyNodeChanged(this.parent);
+	}
+	public abstract onChildRemoved(): void;
+}
+
+export class FolderNode extends HdfsFileSourceNode {
+	private children: TreeNode[];
+	protected _nodeType: string;
+	constructor(context: TreeDataContext, path: string, fileSource: IFileSource, nodeType?: string) {
+		super(context, path, fileSource);
+		this._nodeType = nodeType ? nodeType : Constants.MssqlClusterItems.Folder;
+	}
+
+	private ensureChildrenExist(): void {
+		if (!this.children) {
+			this.children = [];
+		}
+	}
+
+	public onChildRemoved(): void {
+		this.children = undefined;
+	}
+
+	async getChildren(refreshChildren: boolean): Promise<TreeNode[]> {
+		if (refreshChildren || !this.children) {
+			this.ensureChildrenExist();
+			try {
+				let files: IFile[] = await this.fileSource.enumerateFiles(this._path);
+				if (files) {
+					// Note: for now, assuming HDFS-provided sorting is sufficient
+					this.children = files.map((file) => {
+						let node: TreeNode = file.isDirectory ? new FolderNode(this.context, file.path, this.fileSource)
+							: new FileNode(this.context, file.path, this.fileSource);
+						node.parent = this;
+						return node;
+					});
+				}
+			} catch (error) {
+				this.children = [ErrorNode.create(localize('errorExpanding', 'Error: {0}', utils.getErrorMessage(error)), this)];
+			}
+		}
+		return this.children;
+	}
+
+	getTreeItem(): vscode.TreeItem | Promise<vscode.TreeItem> {
+		let item = new vscode.TreeItem(this.getDisplayName(), vscode.TreeItemCollapsibleState.Collapsed);
+		// For now, folder always looks the same. We're using SQL icons to differentiate remote vs local files
+		item.iconPath = {
+			dark: this.context.extensionContext.asAbsolutePath('resources/light/Folder.svg'),
+			light: this.context.extensionContext.asAbsolutePath('resources/light/Folder.svg')
+		};
+		item.contextValue = this._nodeType;
+		return item;
+	}
+
+	getNodeInfo(): sqlops.NodeInfo {
+		// TODO handle error message case by returning it in the OE API
+		// TODO support better mapping of node type
+		let nodeInfo: sqlops.NodeInfo = {
+			label: this.getDisplayName(),
+			isLeaf: false,
+			errorMessage: undefined,
+			metadata: undefined,
+			nodePath: this.generateNodePath(),
+			nodeStatus: undefined,
+			nodeType: this._nodeType,
+			nodeSubType: undefined,
+			iconType: 'Folder'
+		};
+		return nodeInfo;
+	}
+
+	public async writeFile(localFile: IFile): Promise<FileNode> {
+		return this.runChildAddAction<FileNode>(() => this.writeFileAsync(localFile));
+	}
+
+	private async writeFileAsync(localFile: IFile): Promise<FileNode> {
+		await this.fileSource.writeFile(localFile, this._path);
+		let fileNode = new FileNode(this.context, File.createPath(this._path, File.getBasename(localFile)), this.fileSource);
+		return fileNode;
+	}
+
+	public async mkdir(name: string): Promise<FolderNode> {
+		return this.runChildAddAction<FolderNode>(() => this.mkdirAsync(name));
+	}
+
+	private async mkdirAsync(name: string): Promise<FolderNode> {
+		await this.fileSource.mkdir(name, this._path);
+		let subDir = new FolderNode(this.context, File.createPath(this._path, name), this.fileSource);
+		return subDir;
+	}
+
+	private async runChildAddAction<T extends TreeNode>(action: () => Promise<T>): Promise<T> {
+		let node = await action();
+		await this.getChildren(true);
+		if (this.children) {
+			// Find the child matching the node. This is necessary
+			// since writing can add duplicates.
+			node = this.children.find(n => n.nodePathValue === node.nodePathValue) as T;
+			this.context.changeHandler.notifyNodeChanged(this);
+		} else {
+			// Failed to retrieve children from server so something went wrong
+			node = undefined;
+		}
+		return node;
+	}
+}
+
+export class ConnectionNode extends FolderNode {
+
+	constructor(context: TreeDataContext, private displayName: string, fileSource: IFileSource) {
+		super(context, '/', fileSource, Constants.MssqlClusterItems.Connection);
+	}
+
+	getDisplayName(): string {
+		return this.displayName;
+	}
+
+	public async delete(): Promise<void> {
+		throw new Error(localize('errDeleteConnectionNode', 'Cannot delete a connection. Only subfolders and files can be deleted.'));
+	}
+
+	async getTreeItem(): Promise<vscode.TreeItem> {
+		let item = await super.getTreeItem();
+		item.contextValue = this._nodeType;
+		return item;
+	}
+}
+
+export class FileNode extends HdfsFileSourceNode implements IFileNode {
+
+	constructor(context: TreeDataContext, path: string, fileSource: IFileSource) {
+		super(context, path, fileSource);
+	}
+
+	public onChildRemoved(): void {
+		// do nothing
+	}
+
+	getChildren(refreshChildren: boolean): TreeNode[] | Promise<TreeNode[]> {
+		return [];
+	}
+
+	getTreeItem(): vscode.TreeItem | Promise<vscode.TreeItem> {
+		let item = new vscode.TreeItem(this.getDisplayName(), vscode.TreeItemCollapsibleState.None);
+		item.iconPath = {
+			dark: this.context.extensionContext.asAbsolutePath('resources/dark/file_inverse.svg'),
+			light: this.context.extensionContext.asAbsolutePath('resources/light/file.svg')
+		};
+		item.contextValue = Constants.MssqlClusterItems.File;
+		return item;
+	}
+
+
+	getNodeInfo(): sqlops.NodeInfo {
+		// TODO improve node type handling so it's not tied to SQL Server types
+		let nodeInfo: sqlops.NodeInfo = {
+			label: this.getDisplayName(),
+			isLeaf: true,
+			errorMessage: undefined,
+			metadata: undefined,
+			nodePath: this.generateNodePath(),
+			nodeStatus: undefined,
+			nodeType: Constants.MssqlClusterItems.File,
+			nodeSubType: this.getSubType(),
+			iconType: 'FileGroupFile'
+		};
+		return nodeInfo;
+	}
+
+	public async getFileContentsAsString(maxBytes?: number): Promise<string> {
+		let contents: Buffer = await this.fileSource.readFile(this.hdfsPath, maxBytes);
+		return contents ? contents.toString('utf8') : '';
+	}
+
+	public async getFileLinesAsString(maxLines: number): Promise<string> {
+		let contents: Buffer = await this.fileSource.readFileLines(this.hdfsPath, maxLines);
+		return contents ? contents.toString('utf8') : '';
+	}
+
+	public writeFileContentsToDisk(localPath: string, cancelToken?: vscode.CancellationTokenSource): Promise<vscode.Uri> {
+		return new Promise((resolve, reject) => {
+			let readStream: fs.ReadStream = this.fileSource.createReadStream(this.hdfsPath);
+			let writeStream = fs.createWriteStream(localPath, {
+				encoding: 'utf8'
+			});
+			let cancelable = new CancelableStream(cancelToken);
+			cancelable.on('error', (err) => {
+				reject(err);
+			});
+			readStream.pipe(cancelable).pipe(writeStream);
+
+			let error: string | Error = undefined;
+
+			writeStream.on('error', (err) => {
+				error = err;
+				reject(error);
+			});
+			writeStream.on('finish', (location) => {
+				if (!error) {
+					resolve(vscode.Uri.file(localPath));
+				}
+			});
+		});
+	}
+
+	private getSubType(): string {
+		if (this.getDisplayName().toLowerCase().endsWith('.jar') || this.getDisplayName().toLowerCase().endsWith('.py')) {
+			return Constants.MssqlClusterItemsSubType.Spark;
+		}
+
+		return undefined;
+	}
+}
+
+export class ErrorNode extends TreeNode {
+	static messageNum: number = 0;
+
+	private _nodePathValue: string;
+	constructor(private message: string) {
+		super();
+	}
+
+	public static create(message: string, parent: TreeNode): ErrorNode {
+		let node = new ErrorNode(message);
+		node.parent = parent;
+		return node;
+	}
+
+	private ensureNodePathValue(): void {
+		if (!this._nodePathValue) {
+			this._nodePathValue = `message_${ErrorNode.messageNum++}`;
+		}
+	}
+
+	public get nodePathValue(): string {
+		this.ensureNodePathValue();
+		return this._nodePathValue;
+	}
+
+	public getChildren(refreshChildren: boolean): TreeNode[] | Promise<TreeNode[]> {
+		return [];
+	}
+
+	public getTreeItem(): vscode.TreeItem | Promise<vscode.TreeItem> {
+		let item = new vscode.TreeItem(this.message, vscode.TreeItemCollapsibleState.None);
+		item.contextValue = Constants.MssqlClusterItems.Error;
+		return item;
+	}
+
+
+	getNodeInfo(): sqlops.NodeInfo {
+		let nodeInfo: sqlops.NodeInfo = {
+			label: this.message,
+			isLeaf: false,
+			errorMessage: undefined,
+			metadata: undefined,
+			nodePath: this.generateNodePath(),
+			nodeStatus: undefined,
+			nodeType: Constants.MssqlClusterItems.Error,
+			nodeSubType: undefined,
+			iconType: 'MessageType'
+		};
+		return nodeInfo;
+	}
+}

extensions/mssql/src/objectExplorerNodeProvider/objectExplorerNodeProvider.ts

@@ -0,0 +1,328 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+import { ProviderBase } from './providerBase';
+import { SqlClusterConnection } from './connection';
+import * as utils from '../utils';
+import { TreeNode } from './treeNodes';
+import { ConnectionNode, TreeDataContext, ITreeChangeHandler } from './hdfsProvider';
+import { IFileSource } from './fileSources';
+import { AppContext } from '../appContext';
+import * as constants from '../constants';
+import * as SqlClusterLookUp from '../sqlClusterLookUp';
+import { ICommandObjectExplorerContext } from './command';
+
+export const mssqlOutputChannel = vscode.window.createOutputChannel(constants.providerId);
+
+export class MssqlObjectExplorerNodeProvider extends ProviderBase implements sqlops.ObjectExplorerNodeProvider, ITreeChangeHandler {
+	public readonly supportedProviderId: string = constants.providerId;
+	private sessionMap: Map<string, SqlClusterSession>;
+	private expandCompleteEmitter = new vscode.EventEmitter<sqlops.ObjectExplorerExpandInfo>();
+
+	constructor(private appContext: AppContext) {
+		super();
+		this.sessionMap = new Map<string, SqlClusterSession>();
+		this.appContext.registerService<MssqlObjectExplorerNodeProvider>(constants.ObjectExplorerService, this);
+	}
+
+	handleSessionOpen(session: sqlops.ObjectExplorerSession): Thenable<boolean> {
+		return new Promise((resolve, reject) => {
+			if (!session) {
+				reject('handleSessionOpen requires a session object to be passed');
+			} else {
+				resolve(this.doSessionOpen(session));
+			}
+		});
+	}
+
+	private async doSessionOpen(session: sqlops.ObjectExplorerSession): Promise<boolean> {
+		if (!session || !session.sessionId) { return false; }
+
+		let sqlConnProfile = await sqlops.objectexplorer.getSessionConnectionProfile(session.sessionId);
+		if (!sqlConnProfile) { return false; }
+
+		let clusterConnInfo = await SqlClusterLookUp.getSqlClusterConnection(sqlConnProfile);
+		if (!clusterConnInfo) { return false; }
+
+		let clusterConnection = new SqlClusterConnection(clusterConnInfo);
+		let clusterSession = new SqlClusterSession(clusterConnection, session, sqlConnProfile, this.appContext, this);
+		this.sessionMap.set(session.sessionId, clusterSession);
+		return true;
+	}
+
+	expandNode(nodeInfo: sqlops.ExpandNodeInfo, isRefresh: boolean = false): Thenable<boolean> {
+		return new Promise((resolve, reject) => {
+			if (!nodeInfo) {
+				reject('expandNode requires a nodeInfo object to be passed');
+			} else {
+				resolve(this.doExpandNode(nodeInfo, isRefresh));
+			}
+		});
+	}
+
+	private async doExpandNode(nodeInfo: sqlops.ExpandNodeInfo, isRefresh: boolean = false): Promise<boolean> {
+		let session = this.sessionMap.get(nodeInfo.sessionId);
+		let response = {
+			sessionId: nodeInfo.sessionId,
+			nodePath: nodeInfo.nodePath,
+			errorMessage: undefined,
+			nodes: []
+		};
+
+		if (!session) {
+			// This is not an error case. Just fire reponse with empty nodes for example: request from standalone SQL instance
+			this.expandCompleteEmitter.fire(response);
+			return false;
+		} else {
+			setTimeout(() => {
+
+				// Running after promise resolution as we need the Ops Studio-side map to have been updated
+				// Intentionally not awaiting or catching errors.
+				// Any failure in startExpansion should be emitted in the expand complete result
+				// We want this to be async and ideally return true before it completes
+				this.startExpansion(session, nodeInfo, isRefresh);
+			}, 10);
+		}
+		return true;
+	}
+
+	private async startExpansion(session: SqlClusterSession, nodeInfo: sqlops.ExpandNodeInfo, isRefresh: boolean = false): Promise<void> {
+		let expandResult: sqlops.ObjectExplorerExpandInfo = {
+			sessionId: session.sessionId,
+			nodePath: nodeInfo.nodePath,
+			errorMessage: undefined,
+			nodes: []
+		};
+		try {
+			let node = await session.rootNode.findNodeByPath(nodeInfo.nodePath, true);
+			if (node) {
+				expandResult.errorMessage = node.getNodeInfo().errorMessage;
+				let children = await node.getChildren(true);
+				if (children) {
+					expandResult.nodes = children.map(c => c.getNodeInfo());
+					// There is only child returned when failure happens
+					if (children.length === 1) {
+						let child = children[0].getNodeInfo();
+						if (child && child.nodeType === constants.MssqlClusterItems.Error) {
+							expandResult.errorMessage = child.label;
+							expandResult.nodes = [];
+						}
+					}
+				}
+			}
+		} catch (error) {
+			expandResult.errorMessage = utils.getErrorMessage(error);
+		}
+		this.expandCompleteEmitter.fire(expandResult);
+	}
+
+	refreshNode(nodeInfo: sqlops.ExpandNodeInfo): Thenable<boolean> {
+		// TODO #3815 implement properly
+		return this.expandNode(nodeInfo, true);
+	}
+
+	handleSessionClose(closeSessionInfo: sqlops.ObjectExplorerCloseSessionInfo): void {
+		this.sessionMap.delete(closeSessionInfo.sessionId);
+	}
+
+	findNodes(findNodesInfo: sqlops.FindNodesInfo): Thenable<sqlops.ObjectExplorerFindNodesResponse> {
+		// TODO #3814 implement
+		let response: sqlops.ObjectExplorerFindNodesResponse = {
+			nodes: []
+		};
+		return Promise.resolve(response);
+	}
+
+	registerOnExpandCompleted(handler: (response: sqlops.ObjectExplorerExpandInfo) => any): void {
+		this.expandCompleteEmitter.event(handler);
+	}
+
+	notifyNodeChanged(node: TreeNode): void {
+		this.notifyNodeChangesAsync(node);
+	}
+
+	private async notifyNodeChangesAsync(node: TreeNode): Promise<void> {
+		try {
+			let session = this.getSqlClusterSessionForNode(node);
+			if (!session) {
+				this.appContext.apiWrapper.showErrorMessage(localize('sessionNotFound', 'Session for node {0} does not exist', node.nodePathValue));
+			} else {
+				let nodeInfo = node.getNodeInfo();
+				let expandInfo: sqlops.ExpandNodeInfo = {
+					nodePath: nodeInfo.nodePath,
+					sessionId: session.sessionId
+				};
+				await this.refreshNode(expandInfo);
+			}
+		} catch (err) {
+			mssqlOutputChannel.appendLine(localize('notifyError', 'Error notifying of node change: {0}', err));
+		}
+	}
+
+	private getSqlClusterSessionForNode(node: TreeNode): SqlClusterSession {
+		let sqlClusterSession: SqlClusterSession = undefined;
+		while (node !== undefined) {
+			if (node instanceof DataServicesNode) {
+				sqlClusterSession = node.session;
+				break;
+			} else {
+				node = node.parent;
+			}
+		}
+		return sqlClusterSession;
+	}
+
+	async findSqlClusterNodeByContext<T extends TreeNode>(context: ICommandObjectExplorerContext | sqlops.ObjectExplorerContext): Promise<T> {
+		let node: T = undefined;
+		let explorerContext = 'explorerContext' in context ? context.explorerContext : context;
+		let sqlConnProfile = explorerContext.connectionProfile;
+		let session = this.findSqlClusterSessionBySqlConnProfile(sqlConnProfile);
+		if (session) {
+			if (explorerContext.isConnectionNode) {
+				// Note: ideally fix so we verify T matches RootNode and go from there
+				node = <T><any>session.rootNode;
+			} else {
+				// Find the node under the session
+				node = <T><any>await session.rootNode.findNodeByPath(explorerContext.nodeInfo.nodePath, true);
+			}
+		}
+		return node;
+	}
+
+	public findSqlClusterSessionBySqlConnProfile(connectionProfile: sqlops.IConnectionProfile): SqlClusterSession {
+		for (let session of this.sessionMap.values()) {
+			if (session.isMatchedSqlConnection(connectionProfile)) {
+				return session;
+			}
+		}
+		return undefined;
+	}
+}
+
+export class SqlClusterSession {
+	private _rootNode: SqlClusterRootNode;
+
+	constructor(
+		private _sqlClusterConnection: SqlClusterConnection,
+		private _sqlSession: sqlops.ObjectExplorerSession,
+		private _sqlConnectionProfile: sqlops.IConnectionProfile,
+		private _appContext: AppContext,
+		private _changeHandler: ITreeChangeHandler
+	) {
+		this._rootNode = new SqlClusterRootNode(this,
+			new TreeDataContext(this._appContext.extensionContext, this._changeHandler),
+			this._sqlSession.rootNode.nodePath);
+	}
+
+	public get sqlClusterConnection(): SqlClusterConnection { return this._sqlClusterConnection; }
+	public get sqlSession(): sqlops.ObjectExplorerSession { return this._sqlSession; }
+	public get sqlConnectionProfile(): sqlops.IConnectionProfile { return this._sqlConnectionProfile; }
+	public get sessionId(): string { return this._sqlSession.sessionId; }
+	public get rootNode(): SqlClusterRootNode { return this._rootNode; }
+
+	public isMatchedSqlConnection(sqlConnProfile: sqlops.IConnectionProfile): boolean {
+		return this._sqlConnectionProfile.id === sqlConnProfile.id;
+	}
+}
+
+class SqlClusterRootNode extends TreeNode {
+	private _children: TreeNode[];
+	constructor(
+		private _session: SqlClusterSession,
+		private _treeDataContext: TreeDataContext,
+		private _nodePathValue: string
+	) {
+		super();
+	}
+
+	public get session(): SqlClusterSession {
+		return this._session;
+	}
+
+	public get nodePathValue(): string {
+		return this._nodePathValue;
+	}
+
+	public getChildren(refreshChildren: boolean): TreeNode[] | Promise<TreeNode[]> {
+		if (refreshChildren || !this._children) {
+			this._children = [];
+			let dataServicesNode = new DataServicesNode(this._session, this._treeDataContext, this._nodePathValue);
+			dataServicesNode.parent = this;
+			this._children.push(dataServicesNode);
+		}
+		return this._children;
+	}
+
+	getTreeItem(): vscode.TreeItem | Promise<vscode.TreeItem> {
+		throw new Error('Not intended for use in a file explorer view.');
+	}
+
+	getNodeInfo(): sqlops.NodeInfo {
+		let nodeInfo: sqlops.NodeInfo = {
+			label: localize('rootLabel', 'Root'),
+			isLeaf: false,
+			errorMessage: undefined,
+			metadata: undefined,
+			nodePath: this.generateNodePath(),
+			nodeStatus: undefined,
+			nodeType: 'sqlCluster:root',
+			nodeSubType: undefined,
+			iconType: 'folder'
+		};
+		return nodeInfo;
+	}
+}
+
+class DataServicesNode extends TreeNode {
+	private _children: TreeNode[];
+	constructor(private _session: SqlClusterSession, private _context: TreeDataContext, private _nodePath: string) {
+		super();
+	}
+
+	public get session(): SqlClusterSession {
+		return this._session;
+	}
+
+	public get nodePathValue(): string {
+		return this._nodePath;
+	}
+
+	public getChildren(refreshChildren: boolean): TreeNode[] | Promise<TreeNode[]> {
+		if (refreshChildren || !this._children) {
+			this._children = [];
+			let fileSource: IFileSource = this.session.sqlClusterConnection.createHdfsFileSource();
+			let hdfsNode = new ConnectionNode(this._context, localize('hdfsFolder', 'HDFS'), fileSource);
+			hdfsNode.parent = this;
+			this._children.push(hdfsNode);
+		}
+		return this._children;
+	}
+
+	getTreeItem(): vscode.TreeItem | Promise<vscode.TreeItem> {
+		throw new Error('Not intended for use in a file explorer view.');
+	}
+
+	getNodeInfo(): sqlops.NodeInfo {
+		let nodeInfo: sqlops.NodeInfo = {
+			label: localize('dataServicesLabel', 'Data Services'),
+			isLeaf: false,
+			errorMessage: undefined,
+			metadata: undefined,
+			nodePath: this.generateNodePath(),
+			nodeStatus: undefined,
+			nodeType: 'dataservices',
+			nodeSubType: undefined,
+			iconType: 'folder'
+		};
+		return nodeInfo;
+	}
+}

extensions/mssql/src/objectExplorerNodeProvider/providerBase.ts

@@ -1,4 +1,5 @@
 
+
 /*---------------------------------------------------------------------------------------------
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
@@ -6,11 +7,9 @@
 
 'use strict';
 
-// TODO: The content of this file should be refactored to an extension
-export function getKnoxUrl(host: string, port: string): string {
-	return `https://${host}:${port}/gateway`;
-}
+import * as constants from '../constants';
 
-export function getLivyUrl(serverName: string, port: string): string {
-	return getKnoxUrl(serverName, port) + '/default/livy/v1/';
-}
+export abstract class ProviderBase {
+	public readonly providerId: string = constants.mssqlClusterProviderName;
+	public handle: number;
+}

extensions/mssql/src/objectExplorerNodeProvider/treeNodes.ts

@@ -0,0 +1,78 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+import { ITreeNode } from './types';
+
+type TreeNodePredicate = (node: TreeNode) => boolean;
+
+export abstract class TreeNode implements ITreeNode {
+	private _parent: TreeNode = undefined;
+
+	public get parent(): TreeNode {
+		return this._parent;
+	}
+
+	public set parent(node: TreeNode) {
+		this._parent = node;
+	}
+
+	public generateNodePath(): string {
+		let path = undefined;
+		if (this.parent) {
+			path = this.parent.generateNodePath();
+		}
+		path = path ? `${path}/${this.nodePathValue}` : this.nodePathValue;
+		return path;
+	}
+
+	public findNodeByPath(path: string, expandIfNeeded: boolean = false): Promise<TreeNode> {
+		let condition: TreeNodePredicate = (node: TreeNode) => node.getNodeInfo().nodePath === path || node.getNodeInfo().nodePath.startsWith(path);
+		let filter: TreeNodePredicate = (node: TreeNode) => path.startsWith(node.getNodeInfo().nodePath);
+		return TreeNode.findNode(this, condition, filter, true);
+	}
+
+	public static async findNode(node: TreeNode, condition: TreeNodePredicate, filter: TreeNodePredicate, expandIfNeeded: boolean): Promise<TreeNode> {
+		if (!node) {
+			return undefined;
+		}
+
+		if (condition(node)) {
+			return node;
+		}
+
+		let nodeInfo = node.getNodeInfo();
+		if (nodeInfo.isLeaf) {
+			return undefined;
+		}
+
+		// TODO #3813 support filtering by already expanded / not yet expanded
+		let children = await node.getChildren(false);
+		if (children) {
+			for (let child of children) {
+				if (filter && filter(child)) {
+					let childNode =  await this.findNode(child, condition, filter, expandIfNeeded);
+					if (childNode) {
+						return childNode;
+					}
+				}
+			}
+		}
+		return undefined;
+	}
+
+	/**
+	 * The value to use for this node in the node path
+	 */
+	public abstract get nodePathValue(): string;
+
+	abstract getChildren(refreshChildren: boolean): TreeNode[] | Promise<TreeNode[]>;
+	abstract getTreeItem(): vscode.TreeItem | Promise<vscode.TreeItem>;
+
+	abstract getNodeInfo(): sqlops.NodeInfo;
+}

extensions/mssql/src/objectExplorerNodeProvider/types.d.ts

@@ -0,0 +1,30 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as sqlops from 'sqlops';
+
+/**
+ * A tree node in the object explorer tree
+ *
+ * @export
+ * @interface ITreeNode
+ */
+export interface ITreeNode {
+	getNodeInfo(): sqlops.NodeInfo;
+	getChildren(refreshChildren: boolean): ITreeNode[] | Promise<ITreeNode[]>;
+}
+
+/**
+ * A HDFS file node. This is a leaf node in the object explorer tree, and its contents
+ * can be queried
+ *
+ * @export
+ * @interface IFileNode
+ * @extends {ITreeNode}
+ */
+export interface IFileNode extends ITreeNode {
+	getFileContentsAsString(maxBytes?: number): Promise<string>;
+}

extensions/mssql/src/prompts/adapter.ts

@@ -0,0 +1,111 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import {window, OutputChannel } from 'vscode';
+import * as nodeUtil from 'util';
+import PromptFactory from './factory';
+import EscapeException from '../escapeException';
+import { IQuestion, IPrompter, IPromptCallback } from './question';
+
+// Supports simple pattern for prompting for user input and acting on this
+export default class CodeAdapter implements IPrompter {
+
+	private outChannel: OutputChannel;
+	private outBuffer: string = '';
+	private messageLevelFormatters = {};
+	constructor() {
+		// TODO Decide whether output channel logging should be saved here?
+		this.outChannel = window.createOutputChannel('test');
+		// this.outChannel.clear();
+	}
+
+	public logError(message: any): void {
+		let line = `error: ${message.message}\n    Code - ${message.code}`;
+
+		this.outBuffer += `${line}\n`;
+		this.outChannel.appendLine(line);
+	}
+
+	private formatMessage(message: any): string {
+		const prefix = `${message.level}: (${message.id}) `;
+		return `${prefix}${message.message}`;
+	}
+
+	public clearLog(): void {
+		this.outChannel.clear();
+	}
+
+	public showLog(): void {
+		this.outChannel.show();
+	}
+
+	// TODO define question interface
+	private fixQuestion(question: any): any {
+		if (question.type === 'checkbox' && Array.isArray(question.choices)) {
+			// For some reason when there's a choice of checkboxes, they aren't formatted properly
+			// Not sure where the issue is
+			question.choices = question.choices.map(item => {
+				if (typeof (item) === 'string') {
+					return { checked: false, name: item, value: item };
+				} else {
+					return item;
+				}
+			});
+		}
+	}
+
+	public promptSingle<T>(question: IQuestion, ignoreFocusOut?: boolean): Promise<T> {
+		let questions: IQuestion[] = [question];
+		return this.prompt(questions, ignoreFocusOut).then( (answers: {[key: string]: T})  => {
+			if (answers) {
+				let response: T = answers[question.name];
+				return response || undefined;
+			}
+		});
+	}
+
+	public prompt<T>(questions: IQuestion[], ignoreFocusOut?: boolean): Promise<{[key: string]: T}> {
+		let answers: {[key: string]: T} = {};
+
+		// Collapse multiple questions into a set of prompt steps
+		let promptResult: Promise<{[key: string]: T}> = questions.reduce((promise: Promise<{[key: string]: T}>, question: IQuestion) => {
+			this.fixQuestion(question);
+
+			return promise.then(() => {
+				return PromptFactory.createPrompt(question, ignoreFocusOut);
+			}).then(prompt => {
+				if (!question.shouldPrompt || question.shouldPrompt(answers) === true) {
+					return prompt.render().then(result => {
+						answers[question.name] = result;
+
+						if (question.onAnswered) {
+							question.onAnswered(result);
+						}
+						return answers;
+					});
+				}
+				return answers;
+			});
+		}, Promise.resolve());
+
+		return promptResult.catch(err => {
+			if (err instanceof EscapeException || err instanceof TypeError) {
+				return undefined;
+			}
+
+			window.showErrorMessage(err.message);
+		});
+	}
+
+	// Helper to make it possible to prompt using callback pattern. Generally Promise is a preferred flow
+	public promptCallback(questions: IQuestion[], callback: IPromptCallback): void {
+		// Collapse multiple questions into a set of prompt steps
+		this.prompt(questions).then(answers => {
+			if (callback) {
+				callback(answers);
+			}
+		});
+	}
+}

extensions/mssql/src/prompts/checkbox.ts

@@ -0,0 +1,52 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import { window } from 'vscode';
+import Prompt from './prompt';
+import EscapeException from '../escapeException';
+
+const figures = require('figures');
+
+export default class CheckboxPrompt extends Prompt {
+
+	constructor(question: any, ignoreFocusOut?: boolean) {
+		super(question, ignoreFocusOut);
+	}
+
+	public render(): any {
+		let choices = this._question.choices.reduce((result, choice) => {
+			let choiceName = choice.name || choice;
+			result[`${choice.checked === true ? figures.radioOn : figures.radioOff} ${choiceName}`] = choice;
+			return result;
+		}, {});
+
+		let options = this.defaultQuickPickOptions;
+		options.placeHolder = this._question.message;
+
+		let quickPickOptions = Object.keys(choices);
+		quickPickOptions.push(figures.tick);
+
+		return window.showQuickPick(quickPickOptions, options)
+			.then(result => {
+				if (result === undefined) {
+					throw new EscapeException();
+				}
+
+				if (result !== figures.tick) {
+					choices[result].checked = !choices[result].checked;
+
+					return this.render();
+				}
+
+				return this._question.choices.reduce((result2, choice) => {
+					if (choice.checked === true) {
+						result2.push(choice.value);
+					}
+
+					return result2;
+				}, []);
+			});
+	}
+}

extensions/mssql/src/prompts/confirm.ts

@@ -0,0 +1,36 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+import { window } from 'vscode';
+import Prompt from './prompt';
+import EscapeException from '../escapeException';
+
+export default class ConfirmPrompt extends Prompt {
+
+	constructor(question: any, ignoreFocusOut?: boolean) {
+		super(question, ignoreFocusOut);
+	}
+
+	public render(): any {
+		let choices: { [id: string]: boolean } = {};
+		choices[localize('msgYes', 'Yes')] = true;
+		choices[localize('msgNo', 'No')] = false;
+
+		let options = this.defaultQuickPickOptions;
+		options.placeHolder = this._question.message;
+
+		return window.showQuickPick(Object.keys(choices), options)
+			.then(result => {
+				if (result === undefined) {
+					throw new EscapeException();
+				}
+
+				return choices[result] || false;
+			});
+	}
+}

extensions/mssql/src/prompts/expand.ts

@@ -0,0 +1,78 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import vscode = require('vscode');
+import Prompt from './prompt';
+import EscapeException from '../escapeException';
+import { INameValueChoice } from './question';
+
+const figures = require('figures');
+
+export default class ExpandPrompt extends Prompt {
+
+	constructor(question: any, ignoreFocusOut?: boolean) {
+		super(question, ignoreFocusOut);
+	}
+
+	public render(): any {
+		// label indicates this is a quickpick item. Otherwise it's a name-value pair
+		if (this._question.choices[0].label) {
+			return this.renderQuickPick(this._question.choices);
+		} else {
+			return this.renderNameValueChoice(this._question.choices);
+		}
+	}
+
+	private renderQuickPick(choices: vscode.QuickPickItem[]): any {
+		let options = this.defaultQuickPickOptions;
+		options.placeHolder = this._question.message;
+
+		return vscode.window.showQuickPick(choices, options)
+			.then(result => {
+				if (result === undefined) {
+					throw new EscapeException();
+				}
+
+				return this.validateAndReturn(result || false);
+			});
+	}
+	private renderNameValueChoice(choices: INameValueChoice[]): any {
+		const choiceMap = this._question.choices.reduce((result, choice) => {
+			result[choice.name] = choice.value;
+			return result;
+		}, {});
+
+		let options = this.defaultQuickPickOptions;
+		options.placeHolder = this._question.message;
+
+		return vscode.window.showQuickPick(Object.keys(choiceMap), options)
+			.then(result => {
+				if (result === undefined) {
+					throw new EscapeException();
+				}
+
+				// Note: cannot be used with 0 or false responses
+				let returnVal = choiceMap[result] || false;
+				return this.validateAndReturn(returnVal);
+			});
+	}
+
+	private validateAndReturn(value: any): any {
+		if (!this.validate(value)) {
+			return this.render();
+		}
+		return value;
+	}
+
+	private validate(value: any): boolean {
+		const validationError = this._question.validate ? this._question.validate(value || '') : undefined;
+
+		if (validationError) {
+			this._question.message = `${figures.warning} ${validationError}`;
+			return false;
+		}
+		return true;
+	}
+}

extensions/mssql/src/prompts/factory.ts

@@ -0,0 +1,35 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import Prompt from './prompt';
+import InputPrompt from './input';
+import PasswordPrompt from './password';
+import ListPrompt from './list';
+import ConfirmPrompt from './confirm';
+import CheckboxPrompt from './checkbox';
+import ExpandPrompt from './expand';
+
+export default class PromptFactory {
+
+	public static createPrompt(question: any, ignoreFocusOut?: boolean): Prompt {
+		switch (question.type || 'input') {
+			case 'string':
+			case 'input':
+				return new InputPrompt(question, ignoreFocusOut);
+			case 'password':
+				return new PasswordPrompt(question, ignoreFocusOut);
+			case 'list':
+				return new ListPrompt(question, ignoreFocusOut);
+			case 'confirm':
+				return new ConfirmPrompt(question, ignoreFocusOut);
+			case 'checkbox':
+				return new CheckboxPrompt(question, ignoreFocusOut);
+			case 'expand':
+				return new ExpandPrompt(question, ignoreFocusOut);
+			default:
+				throw new Error(`Could not find a prompt for question type ${question.type}`);
+		}
+	}
+}

extensions/mssql/src/prompts/input.ts

@@ -0,0 +1,59 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import { window, InputBoxOptions } from 'vscode';
+import Prompt from './prompt';
+import EscapeException from '../escapeException';
+
+const figures = require('figures');
+
+export default class InputPrompt extends Prompt {
+
+	protected _options: InputBoxOptions;
+
+	constructor(question: any, ignoreFocusOut?: boolean) {
+		super(question, ignoreFocusOut);
+
+		this._options = this.defaultInputBoxOptions;
+		this._options.prompt = this._question.message;
+	}
+
+	// Helper for callers to know the right type to get from the type factory
+	public static get promptType(): string { return 'input'; }
+
+	public render(): any {
+		// Prefer default over the placeHolder, if specified
+		let placeHolder = this._question.default ? this._question.default : this._question.placeHolder;
+
+		if (this._question.default instanceof Error) {
+			placeHolder = this._question.default.message;
+			this._question.default = undefined;
+		}
+
+		this._options.placeHolder = placeHolder;
+
+		return window.showInputBox(this._options)
+			.then(result => {
+				if (result === undefined) {
+					throw new EscapeException();
+				}
+
+				if (result === '') {
+					// Use the default value, if defined
+					result = this._question.default || '';
+				}
+
+				const validationError = this._question.validate ? this._question.validate(result || '') : undefined;
+
+				if (validationError) {
+					this._question.default = new Error(`${figures.warning} ${validationError}`);
+
+					return this.render();
+				}
+
+				return result;
+			});
+	}
+}

extensions/mssql/src/prompts/list.ts

@@ -0,0 +1,33 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import { window } from 'vscode';
+import Prompt from './prompt';
+import EscapeException from '../escapeException';
+
+export default class ListPrompt extends Prompt {
+	constructor(question: any, ignoreFocusOut?: boolean) {
+		super(question, ignoreFocusOut);
+	}
+
+	public render(): any {
+		const choices = this._question.choices.reduce((result, choice) => {
+			result[choice.name] = choice.value;
+			return result;
+		}, {});
+
+		let options = this.defaultQuickPickOptions;
+		options.placeHolder = this._question.message;
+
+		return window.showQuickPick(Object.keys(choices), options)
+			.then(result => {
+				if (result === undefined) {
+					throw new EscapeException();
+				}
+
+				return choices[result];
+			});
+	}
+}

extensions/mssql/src/prompts/password.ts

@@ -0,0 +1,15 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import InputPrompt from './input';
+
+export default class PasswordPrompt extends InputPrompt {
+
+	constructor(question: any, ignoreFocusOut?: boolean) {
+		super(question, ignoreFocusOut);
+
+		this._options.password = true;
+	}
+}

extensions/mssql/src/prompts/progressIndicator.ts

@@ -0,0 +1,70 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import {window, StatusBarItem, StatusBarAlignment} from 'vscode';
+
+export default class ProgressIndicator {
+
+	private _statusBarItem: StatusBarItem;
+
+	constructor() {
+		this._statusBarItem = window.createStatusBarItem(StatusBarAlignment.Left);
+	   }
+
+	private _tasks: string[] = [];
+	public beginTask(task: string): void {
+		this._tasks.push(task);
+		this.displayProgressIndicator();
+	}
+
+	public endTask(task: string): void {
+		if (this._tasks.length > 0) {
+			this._tasks.pop();
+		}
+
+		this.setMessage();
+	}
+
+	private setMessage(): void {
+		if (this._tasks.length === 0) {
+			this._statusBarItem.text = '';
+			this.hideProgressIndicator();
+			return;
+		}
+
+		this._statusBarItem.text = this._tasks[this._tasks.length - 1];
+		this._statusBarItem.show();
+	}
+
+	private _interval: any;
+	private displayProgressIndicator(): void {
+		this.setMessage();
+		this.hideProgressIndicator();
+		this._interval = setInterval(() => this.onDisplayProgressIndicator(), 100);
+	}
+	private hideProgressIndicator(): void {
+		if (this._interval) {
+			clearInterval(this._interval);
+			this._interval = undefined;
+		}
+		this.ProgressCounter = 0;
+	}
+
+	private ProgressText = ['|', '/', '-', '\\', '|', '/', '-', '\\'];
+	private ProgressCounter = 0;
+	private onDisplayProgressIndicator(): void {
+		if (this._tasks.length === 0) {
+			return;
+		}
+
+		let txt = this.ProgressText[this.ProgressCounter];
+		this._statusBarItem.text = this._tasks[this._tasks.length - 1] + ' ' + txt;
+		this.ProgressCounter++;
+
+		if (this.ProgressCounter >= this.ProgressText.length - 1) {
+			this.ProgressCounter = 0;
+		}
+	}
+}

extensions/mssql/src/prompts/prompt.ts

@@ -0,0 +1,33 @@
+'use strict';
+
+// This code is originally from https://github.com/DonJayamanne/bowerVSCode
+// License: https://github.com/DonJayamanne/bowerVSCode/blob/master/LICENSE
+
+import { InputBoxOptions, QuickPickOptions } from 'vscode';
+
+abstract class Prompt {
+
+	protected _question: any;
+	protected _ignoreFocusOut?: boolean;
+
+	constructor(question: any, ignoreFocusOut?: boolean) {
+		this._question = question;
+		this._ignoreFocusOut = ignoreFocusOut ? ignoreFocusOut : false;
+	}
+
+	public abstract render(): any;
+
+	protected get defaultQuickPickOptions(): QuickPickOptions {
+		return {
+			ignoreFocusOut: this._ignoreFocusOut
+		};
+	}
+
+	protected get defaultInputBoxOptions(): InputBoxOptions {
+		return {
+			ignoreFocusOut: this._ignoreFocusOut
+		};
+	}
+}
+
+export default Prompt;

extensions/mssql/src/prompts/question.ts

@@ -0,0 +1,68 @@
+
+'use strict';
+
+import vscode = require('vscode');
+
+export class QuestionTypes {
+	public static get input(): string { return 'input'; }
+	public static get password(): string { return 'password'; }
+	public static get list(): string { return 'list'; }
+	public static get confirm(): string { return 'confirm'; }
+	public static get checkbox(): string { return 'checkbox'; }
+	public static get expand(): string { return 'expand'; }
+}
+
+// Question interface to clarify how to use the prompt feature
+// based on Bower Question format: https://github.com/bower/bower/blob/89069784bb46bfd6639b4a75e98a0d7399a8c2cb/packages/bower-logger/README.md
+export interface IQuestion {
+	// Type of question (see QuestionTypes)
+	type: string;
+	// Name of the question for disambiguation
+	name: string;
+	// Message to display to the user
+	message: string;
+	// Optional placeHolder to give more detailed information to the user
+	placeHolder?: any;
+	// Optional default value - this will be used instead of placeHolder
+	default?: any;
+	// optional set of choices to be used. Can be QuickPickItems or a simple name-value pair
+	choices?: Array<vscode.QuickPickItem | INameValueChoice>;
+	// Optional validation function that returns an error string if validation fails
+	validate?: (value: any) => string;
+	// Optional pre-prompt function. Takes in set of answers so far, and returns true if prompt should occur
+	shouldPrompt?: (answers: { [id: string]: any }) => boolean;
+	// Optional action to take on the question being answered
+	onAnswered?: (value: any) => void;
+	// Optional set of options to support matching choices.
+	matchOptions?: vscode.QuickPickOptions;
+}
+
+// Pair used to display simple choices to the user
+export interface INameValueChoice {
+	name: string;
+	value: any;
+}
+
+// Generic object that can be used to define a set of questions and handle the result
+export interface IQuestionHandler {
+	// Set of questions to be answered
+	questions: IQuestion[];
+	// Optional callback, since questions may handle themselves
+	callback?: IPromptCallback;
+}
+
+export interface IPrompter {
+	promptSingle<T>(question: IQuestion, ignoreFocusOut?: boolean): Promise<T>;
+	/**
+	 * Prompts for multiple questions
+	 *
+	 * @returns {[questionId: string]: T} Map of question IDs to results, or undefined if
+	 * the user canceled the question session
+	 */
+	prompt<T>(questions: IQuestion[], ignoreFocusOut?: boolean): Promise<{ [questionId: string]: any }>;
+	promptCallback(questions: IQuestion[], callback: IPromptCallback): void;
+}
+
+export interface IPromptCallback {
+	(answers: { [id: string]: any }): void;
+}

extensions/mssql/src/resourceProvider/resourceProvider.ts

@@ -4,15 +4,12 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import * as path from 'path';
+import * as sqlops from 'sqlops';
 import { IConfig, ServerProvider } from 'service-downloader';
 import { SqlOpsDataClient, SqlOpsFeature, ClientOptions } from 'dataprotocol-client';
 import { ServerCapabilities, ClientCapabilities, RPCMessageType, ServerOptions, TransportKind } from 'vscode-languageclient';
 import * as UUID from 'vscode-languageclient/lib/utils/uuid';
-
-import * as sqlops from 'sqlops';
 import { Disposable } from 'vscode';
-
 import { CreateFirewallRuleRequest, HandleFirewallRuleRequest, CreateFirewallRuleParams, HandleFirewallRuleParams } from './contracts';
 import * as Constants from './constants';
 import * as Utils from '../utils';

extensions/mssql/src/sparkFeature/dialog/dialogCommands.ts

@@ -0,0 +1,140 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as nls from 'vscode-nls';
+import * as vscode from 'vscode';
+const localize = nls.loadMessageBundle();
+
+import { ICommandViewContext, Command, ICommandObjectExplorerContext, ICommandUnknownContext } from '../../objectExplorerNodeProvider/command';
+import { SparkJobSubmissionDialog } from './sparkJobSubmission/sparkJobSubmissionDialog';
+import { AppContext } from '../../appContext';
+import { getErrorMessage } from '../../utils';
+import * as constants from '../../constants';
+import { HdfsFileSourceNode } from '../../objectExplorerNodeProvider/hdfsProvider';
+import { getNode } from '../../objectExplorerNodeProvider/hdfsCommands';
+import * as LocalizedConstants from '../../localizedConstants';
+import * as SqlClusterLookUp from '../../sqlClusterLookUp';
+import { SqlClusterConnection } from '../../objectExplorerNodeProvider/connection';
+
+export class OpenSparkJobSubmissionDialogCommand extends Command {
+	constructor(appContext: AppContext, private outputChannel: vscode.OutputChannel) {
+		super(constants.mssqlClusterLivySubmitSparkJobCommand, appContext);
+	}
+
+	protected async preExecute(context: ICommandUnknownContext | ICommandObjectExplorerContext, args: object = {}): Promise<any> {
+		return this.execute(context, args);
+	}
+
+	async execute(context: ICommandUnknownContext | ICommandObjectExplorerContext, ...args: any[]): Promise<void> {
+		try {
+			let sqlClusterConnection: SqlClusterConnection = undefined;
+			if (context.type === constants.ObjectExplorerService) {
+				sqlClusterConnection = SqlClusterLookUp.findSqlClusterConnection(context, this.appContext);
+			}
+			if (!sqlClusterConnection) {
+				sqlClusterConnection = await this.selectConnection();
+			}
+
+			let dialog = new SparkJobSubmissionDialog(sqlClusterConnection, this.appContext, this.outputChannel);
+			await dialog.openDialog();
+		} catch (error) {
+			this.appContext.apiWrapper.showErrorMessage(getErrorMessage(error));
+		}
+	}
+
+	private async selectConnection(): Promise<SqlClusterConnection> {
+		let connectionList: sqlops.connection.Connection[] = await this.apiWrapper.getActiveConnections();
+		let displayList: string[] = new Array();
+		let connectionMap: Map<string, sqlops.connection.Connection> = new Map();
+		if (connectionList && connectionList.length > 0) {
+			connectionList.forEach(conn => {
+				if (conn.providerName === constants.sqlProviderName) {
+					displayList.push(conn.options.host);
+					connectionMap.set(conn.options.host, conn);
+				}
+			});
+		}
+
+		let selectedHost: string = await vscode.window.showQuickPick(displayList, {
+			placeHolder:
+				localize('sparkJobSubmission_PleaseSelectSqlWithCluster',
+					'Please select SQL Server with Big Data Cluster. ')
+		});
+		let errorMsg = localize('sparkJobSubmission_NoSqlSelected', 'No Sql Server is selected.');
+		if (!selectedHost) { throw new Error(errorMsg); }
+
+		let sqlConnection = connectionMap.get(selectedHost);
+		if (!sqlConnection) { throw new Error(errorMsg); }
+
+		let sqlClusterConnection = await SqlClusterLookUp.getSqlClusterConnection(sqlConnection);
+		if (!sqlClusterConnection) {
+			throw new Error(LocalizedConstants.sparkJobSubmissionNoSqlBigDataClusterFound);
+		}
+
+		return new SqlClusterConnection(sqlClusterConnection);
+	}
+}
+
+// Open the submission dialog for a specific file path.
+export class OpenSparkJobSubmissionDialogFromFileCommand extends Command {
+	constructor(appContext: AppContext, private outputChannel: vscode.OutputChannel) {
+		super(constants.mssqlClusterLivySubmitSparkJobFromFileCommand, appContext);
+	}
+
+	protected async preExecute(context: ICommandViewContext | ICommandObjectExplorerContext, args: object = {}): Promise<any> {
+		return this.execute(context, args);
+	}
+
+	async execute(context: ICommandViewContext | ICommandObjectExplorerContext, ...args: any[]): Promise<void> {
+		let path: string = undefined;
+		try {
+			let node = await getNode<HdfsFileSourceNode>(context, this.appContext);
+			if (node && node.hdfsPath) {
+				path = node.hdfsPath;
+			} else {
+				this.apiWrapper.showErrorMessage(LocalizedConstants.msgMissingNodeContext);
+				return;
+			}
+		} catch (err) {
+			this.apiWrapper.showErrorMessage(localize('sparkJobSubmission_GetFilePathFromSelectedNodeFailed', 'Error Get File Path: {0}', err));
+			return;
+		}
+
+		try {
+			let sqlClusterConnection: SqlClusterConnection = undefined;
+			if (context.type === constants.ObjectExplorerService) {
+				sqlClusterConnection = await SqlClusterLookUp.findSqlClusterConnection(context, this.appContext);
+			}
+			if (!sqlClusterConnection) {
+				throw new Error(LocalizedConstants.sparkJobSubmissionNoSqlBigDataClusterFound);
+			}
+			let dialog = new SparkJobSubmissionDialog(sqlClusterConnection, this.appContext, this.outputChannel);
+			await dialog.openDialog(path);
+		} catch (error) {
+			this.appContext.apiWrapper.showErrorMessage(getErrorMessage(error));
+		}
+	}
+}
+
+export class OpenSparkJobSubmissionDialogTask {
+	constructor(private appContext: AppContext, private outputChannel: vscode.OutputChannel) {
+	}
+
+	async execute(profile: sqlops.IConnectionProfile, ...args: any[]): Promise<void> {
+		try {
+			let sqlClusterConnection = SqlClusterLookUp.findSqlClusterConnection(profile, this.appContext);
+			if (!sqlClusterConnection) {
+				throw new Error(LocalizedConstants.sparkJobSubmissionNoSqlBigDataClusterFound);
+			}
+			let dialog = new SparkJobSubmissionDialog(sqlClusterConnection, this.appContext, this.outputChannel);
+			await dialog.openDialog();
+		} catch (error) {
+			this.appContext.apiWrapper.showErrorMessage(getErrorMessage(error));
+		}
+	}
+}

extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkAdvancedTab.ts

@@ -0,0 +1,81 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+
+import { SparkJobSubmissionModel } from './sparkJobSubmissionModel';
+import { AppContext } from '../../../appContext';
+import { ApiWrapper } from '../../../apiWrapper';
+
+export class SparkAdvancedTab {
+	private _tab: sqlops.window.modelviewdialog.DialogTab;
+	public get tab(): sqlops.window.modelviewdialog.DialogTab { return this._tab; }
+
+	private _referenceFilesInputBox: sqlops.InputBoxComponent;
+	private _referenceJARFilesInputBox: sqlops.InputBoxComponent;
+	private _referencePyFilesInputBox: sqlops.InputBoxComponent;
+
+	private get apiWrapper(): ApiWrapper {
+		return this.appContext.apiWrapper;
+	}
+
+	constructor(private appContext: AppContext) {
+		this._tab = this.apiWrapper.createTab(localize('sparkJobSubmission_AdvancedTabName', 'ADVANCED'));
+
+		this._tab.registerContent(async (modelView) => {
+			let builder = modelView.modelBuilder;
+			let parentLayout: sqlops.FormItemLayout = {
+				horizontal: false,
+				componentWidth: '400px'
+			};
+
+			let formContainer = builder.formContainer();
+
+			this._referenceJARFilesInputBox = builder.inputBox().component();
+			formContainer.addFormItem({
+				component: this._referenceJARFilesInputBox,
+				title: localize('sparkJobSubmission_ReferenceJarList', 'Reference Jars')
+			},
+				Object.assign(
+					{
+						info: localize('sparkJobSubmission_ReferenceJarListToolTip',
+							'Jars to be placed in executor working directory. The Jar path needs to be an HDFS Path. Multiple paths should be split by semicolon (;)')
+					},
+					parentLayout));
+
+			this._referencePyFilesInputBox = builder.inputBox().component();
+			formContainer.addFormItem({
+				component: this._referencePyFilesInputBox,
+				title: localize('sparkJobSubmission_ReferencePyList', 'Reference py Files')
+			},
+				Object.assign(
+					{
+						info: localize('sparkJobSubmission_ReferencePyListTooltip',
+							'Py Files to be placed in executor working directory. The file path needs to be an HDFS Path. Multiple paths should be split by semicolon(;)')
+					},
+					parentLayout));
+
+			this._referenceFilesInputBox = builder.inputBox().component();
+			formContainer.addFormItem({
+				component: this._referenceFilesInputBox,
+				title: localize('sparkJobSubmission_ReferenceFilesList', 'Reference Files')
+			},
+				Object.assign({
+					info: localize('sparkJobSubmission_ReferenceFilesListTooltip',
+						'Files to be placed in executor working directory. The file path needs to be an HDFS Path. Multiple paths should be split by semicolon(;)')
+				}, parentLayout));
+
+			await modelView.initializeModel(formContainer.component());
+		});
+	}
+
+	public getInputValues(): string[] {
+		return [this._referenceJARFilesInputBox.value, this._referencePyFilesInputBox.value, this._referenceFilesInputBox.value];
+	}
+}

extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkConfigurationTab.ts

@@ -0,0 +1,280 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as nls from 'vscode-nls';
+import * as fspath from 'path';
+import * as fs from 'fs';
+import * as vscode from 'vscode';
+import * as utils from '../../../utils';
+import * as LocalizedConstants from '../../../localizedConstants';
+import * as constants from '../../../constants';
+
+import { AppContext } from '../../../appContext';
+import { ApiWrapper } from '../../../apiWrapper';
+import { SparkJobSubmissionModel } from './sparkJobSubmissionModel';
+import { SparkFileSource } from './sparkJobSubmissionService';
+
+const localize = nls.loadMessageBundle();
+
+export class SparkConfigurationTab {
+	private _tab: sqlops.window.modelviewdialog.DialogTab;
+	public get tab(): sqlops.window.modelviewdialog.DialogTab { return this._tab; }
+
+	private _jobNameInputBox: sqlops.InputBoxComponent;
+	private _sparkContextLabel: sqlops.TextComponent;
+	private _fileSourceDropDown: sqlops.DropDownComponent;
+	private _sparkSourceFileInputBox: sqlops.InputBoxComponent;
+	private _filePickerButton: sqlops.ButtonComponent;
+	private _sourceFlexContainer: sqlops.FlexContainer;
+	private _sourceFlexContainerWithHint: sqlops.FlexContainer;
+	private _localUploadDestinationLabel: sqlops.TextComponent;
+	private _mainClassInputBox: sqlops.InputBoxComponent;
+	private _argumentsInputBox: sqlops.InputBoxComponent;
+
+	private get apiWrapper(): ApiWrapper {
+		return this.appContext.apiWrapper;
+	}
+
+	// If path is specified, means the default source setting for this tab is HDFS file, otherwise, it would be local file.
+	constructor(private _dataModel: SparkJobSubmissionModel, private appContext: AppContext, private _path?: string) {
+		this._tab = this.apiWrapper.createTab(localize('sparkJobSubmission_GeneralTabName', 'GENERAL'));
+
+		this._tab.registerContent(async (modelView) => {
+			let builder = modelView.modelBuilder;
+			let parentLayout: sqlops.FormItemLayout = {
+				horizontal: false,
+				componentWidth: '400px'
+			};
+
+			let formContainer = builder.formContainer();
+
+			this._jobNameInputBox = builder.inputBox().withProperties({
+				placeHolder: localize('sparkJobSubmission_JobNamePlaceHolder', 'Enter a name ...'),
+				value: (this._path) ? fspath.basename(this._path) : ''
+			}).component();
+
+			formContainer.addFormItem({
+				component: this._jobNameInputBox,
+				title: localize('sparkJobSubmission_JobName', 'Job Name'),
+				required: true
+			}, parentLayout);
+
+			this._sparkContextLabel = builder.text().withProperties({
+				value: this._dataModel.getSparkClusterUrl()
+			}).component();
+			formContainer.addFormItem({
+				component: this._sparkContextLabel,
+				title: localize('sparkJobSubmission_SparkCluster', 'Spark Cluster')
+			}, parentLayout);
+
+			this._fileSourceDropDown = builder.dropDown().withProperties<sqlops.DropDownProperties>({
+				values: [SparkFileSource.Local.toString(), SparkFileSource.HDFS.toString()],
+				value: (this._path) ? SparkFileSource.HDFS.toString() : SparkFileSource.Local.toString()
+			}).component();
+
+			this._fileSourceDropDown.onValueChanged(selection => {
+				let isLocal = selection.selected === SparkFileSource.Local.toString();
+				// Disable browser button for cloud source.
+				if (this._filePickerButton) {
+					this._filePickerButton.updateProperties({
+						enabled: isLocal,
+						required: isLocal
+					});
+				}
+
+				// Clear the path When switching source.
+				if (this._sparkSourceFileInputBox) {
+					this._sparkSourceFileInputBox.value = '';
+				}
+
+				if (this._localUploadDestinationLabel) {
+					if (isLocal) {
+						this._localUploadDestinationLabel.value = LocalizedConstants.sparkLocalFileDestinationHint;
+					} else {
+						this._localUploadDestinationLabel.value = '';
+					}
+				}
+			});
+
+			this._sparkSourceFileInputBox = builder.inputBox().withProperties({
+				required: true,
+				placeHolder: localize('sparkJobSubmission_FilePathPlaceHolder', 'Path to a .jar or .py file'),
+				value: (this._path) ? this._path : ''
+			}).component();
+			this._sparkSourceFileInputBox.onTextChanged(text => {
+				if (this._fileSourceDropDown.value === SparkFileSource.Local.toString()) {
+					this._dataModel.updateModelByLocalPath(text);
+					if (this._localUploadDestinationLabel) {
+						if (text) {
+							this._localUploadDestinationLabel.value = localize('sparkJobSubmission_LocalFileDestinationHintWithPath',
+								'The selected local file will be uploaded to HDFS: {0}', this._dataModel.hdfsSubmitFilePath);
+						} else {
+							this._localUploadDestinationLabel.value = LocalizedConstants.sparkLocalFileDestinationHint;
+						}
+					}
+				} else {
+					this._dataModel.hdfsSubmitFilePath = text;
+				}
+
+				// main class disable/enable is according to whether it's jar file.
+				let isJarFile = this._dataModel.isJarFile();
+				this._mainClassInputBox.updateProperties({ enabled: isJarFile, required: isJarFile });
+				if (!isJarFile) {
+					// Clear main class for py file.
+					this._mainClassInputBox.value = '';
+				}
+			});
+
+			this._filePickerButton = builder.button().withProperties({
+				required: (this._path) ? false : true,
+				enabled: (this._path) ? false : true,
+				label: '•••',
+				width: constants.mssqlClusterSparkJobFileSelectorButtonWidth,
+				height: constants.mssqlClusterSparkJobFileSelectorButtonHeight
+			}).component();
+			this._filePickerButton.onDidClick(() => this.onSelectFile());
+
+			this._sourceFlexContainer = builder.flexContainer().component();
+			this._sourceFlexContainer.addItem(this._fileSourceDropDown, { flex: '0 0 auto', CSSStyles: { 'minWidth': '75px', 'marginBottom': '5px', 'paddingRight': '3px' } });
+			this._sourceFlexContainer.addItem(this._sparkSourceFileInputBox, { flex: '1 1 auto', CSSStyles: { 'marginBottom': '5px', 'paddingRight': '3px' } });
+			// Do not add margin for file picker button as the label forces it to have 5px margin
+			this._sourceFlexContainer.addItem(this._filePickerButton, { flex: '0 0 auto' });
+			this._sourceFlexContainer.setLayout({
+				flexFlow: 'row',
+				height: '100%',
+				justifyContent: 'center',
+				alignItems: 'center',
+				alignContent: 'stretch'
+			});
+
+			this._localUploadDestinationLabel = builder.text().withProperties({
+				value: (this._path) ? '' : LocalizedConstants.sparkLocalFileDestinationHint
+			}).component();
+			this._sourceFlexContainerWithHint = builder.flexContainer().component();
+			this._sourceFlexContainerWithHint.addItem(this._sourceFlexContainer, { flex: '0 0 auto' });
+			this._sourceFlexContainerWithHint.addItem(this._localUploadDestinationLabel, { flex: '1 1 auto' });
+			this._sourceFlexContainerWithHint.setLayout({
+				flexFlow: 'column',
+				width: '100%',
+				justifyContent: 'center',
+				alignItems: 'stretch',
+				alignContent: 'stretch'
+			});
+
+			formContainer.addFormItem({
+				component: this._sourceFlexContainerWithHint,
+				title: localize('sparkJobSubmission_MainFilePath', 'JAR/py File'),
+				required: true
+			}, parentLayout);
+
+			this._mainClassInputBox = builder.inputBox().component();
+			formContainer.addFormItem({
+				component: this._mainClassInputBox,
+				title: localize('sparkJobSubmission_MainClass', 'Main Class'),
+				required: true
+			}, parentLayout);
+
+			this._argumentsInputBox = builder.inputBox().component();
+			formContainer.addFormItem({
+				component: this._argumentsInputBox,
+				title: localize('sparkJobSubmission_Arguments', 'Arguments')
+			},
+				Object.assign(
+					{ info: localize('sparkJobSubmission_ArgumentsTooltip', 'Command line arguments used in your main class, multiple arguments should be split by space.') },
+					parentLayout));
+
+			await modelView.initializeModel(formContainer.component());
+		});
+	}
+
+	public async validate(): Promise<boolean> {
+		if (!this._jobNameInputBox.value) {
+			this._dataModel.showDialogError(localize('sparkJobSubmission_NotSpecifyJobName', 'Property Job Name is not specified.'));
+			return false;
+		}
+
+		if (this._fileSourceDropDown.value === SparkFileSource.Local.toString()) {
+			if (this._sparkSourceFileInputBox.value) {
+				this._dataModel.isMainSourceFromLocal = true;
+				this._dataModel.updateModelByLocalPath(this._sparkSourceFileInputBox.value);
+			} else {
+				this._dataModel.showDialogError(localize('sparkJobSubmission_NotSpecifyJARPYPath', 'Property JAR/py File is not specified.'));
+				return false;
+			}
+		} else {
+			if (this._sparkSourceFileInputBox.value) {
+				this._dataModel.isMainSourceFromLocal = false;
+				this._dataModel.hdfsSubmitFilePath = this._sparkSourceFileInputBox.value;
+			} else {
+				this._dataModel.showDialogError(localize('sparkJobSubmission_NotSpecifyJARPYPath', 'Property JAR/py File is not specified.'));
+				return false;
+			}
+		}
+
+		if (this._dataModel.isJarFile() && !this._mainClassInputBox.value) {
+			this._dataModel.showDialogError(localize('sparkJobSubmission_NotSpecifyMainClass', 'Property Main Class is not specified.'));
+			return false;
+		}
+
+		// 1. For local file Source check whether they existed.
+		if (this._dataModel.isMainSourceFromLocal) {
+			if (!fs.existsSync(this._dataModel.localFileSourcePath)) {
+				this._dataModel.showDialogError(LocalizedConstants.sparkJobSubmissionLocalFileNotExisted(this._dataModel.localFileSourcePath));
+				return false;
+			}
+		} else {
+			// 2. Check HDFS file existed for HDFS source.
+			try {
+				let isFileExisted = await this._dataModel.isClusterFileExisted(this._dataModel.hdfsSubmitFilePath);
+				if (!isFileExisted) {
+					this._dataModel.showDialogError(localize('sparkJobSubmission_HDFSFileNotExistedWithPath', '{0} does not exist in Cluster or exception thrown. ', this._dataModel.hdfsSubmitFilePath));
+					return false;
+				}
+			} catch (error) {
+				this._dataModel.showDialogError(localize('sparkJobSubmission_HDFSFileNotExisted', 'The specified HDFS file does not exist. '));
+				return false;
+			}
+		}
+
+		return true;
+	}
+
+	private async onSelectFile(): Promise<void> {
+		let filePath = await this.pickFile();
+		if (filePath) {
+			this._sparkSourceFileInputBox.value = filePath;
+		}
+	}
+
+	public getInputValues(): string[] {
+		return [this._jobNameInputBox.value, this._mainClassInputBox.value, this._argumentsInputBox.value];
+	}
+
+	public async pickFile(): Promise<string> {
+		try {
+			let filter = { 'JAR/py files': ['jar', 'py'] };
+			let options: vscode.OpenDialogOptions = {
+				canSelectFiles: true,
+				canSelectFolders: false,
+				canSelectMany: false,
+				openLabel: localize('sparkSelectLocalFile', 'Select'),
+				filters: filter
+			};
+
+			let fileUris: vscode.Uri[] = await this.apiWrapper.showOpenDialog(options);
+			if (fileUris && fileUris[0]) {
+				return fileUris[0].fsPath;
+			}
+
+			return undefined;
+		} catch (err) {
+			this.apiWrapper.showErrorMessage(localize('sparkJobSubmission_SelectFileError', 'Error in locating the file due to Error: {0}', utils.getErrorMessage(err)));
+			return undefined;
+		}
+	}
+}

extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkJobSubmissionDialog.ts

@@ -0,0 +1,168 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+import * as nls from 'vscode-nls';
+import * as utils from '../../../utils';
+import * as LocalizedConstants from '../../../localizedConstants';
+
+import { AppContext } from '../../../appContext';
+import { ApiWrapper } from '../../../apiWrapper';
+import { SparkJobSubmissionModel } from './sparkJobSubmissionModel';
+import { SparkConfigurationTab } from './sparkConfigurationTab';
+import { SparkJobSubmissionInput } from './sparkJobSubmissionService';
+import { SparkAdvancedTab } from './sparkAdvancedTab';
+import { SqlClusterConnection } from '../../../objectExplorerNodeProvider/connection';
+
+const localize = nls.loadMessageBundle();
+
+export class SparkJobSubmissionDialog {
+	private _dialog: sqlops.window.modelviewdialog.Dialog;
+	private _dataModel: SparkJobSubmissionModel;
+	private _sparkConfigTab: SparkConfigurationTab;
+	private _sparkAdvancedTab: SparkAdvancedTab;
+	private get apiWrapper(): ApiWrapper {
+		return this.appContext.apiWrapper;
+	}
+
+	constructor(
+		private sqlClusterConnection: SqlClusterConnection,
+		private appContext: AppContext,
+		private outputChannel: vscode.OutputChannel) {
+		if (!this.sqlClusterConnection || !this.appContext || !this.outputChannel) {
+			throw new Error(localize('sparkJobSubmission_SparkJobSubmissionDialogInitializeError',
+				'Parameters for SparkJobSubmissionDialog is illegal'));
+		}
+	}
+
+	public async openDialog(path?: string): Promise<void> {
+		this._dialog = this.apiWrapper.createDialog(localize('sparkJobSubmission_DialogTitleNewJob', 'New Job'));
+
+		this._dataModel = new SparkJobSubmissionModel(this.sqlClusterConnection, this._dialog, this.appContext);
+
+		this._sparkConfigTab = new SparkConfigurationTab(this._dataModel, this.appContext, path);
+		this._sparkAdvancedTab = new SparkAdvancedTab(this.appContext);
+
+		this._dialog.content = [this._sparkConfigTab.tab, this._sparkAdvancedTab.tab];
+
+		this._dialog.cancelButton.label = localize('sparkJobSubmission_DialogCancelButton', 'Cancel');
+
+		this._dialog.okButton.label = localize('sparkJobSubmission_DialogSubmitButton', 'Submit');
+		this._dialog.okButton.onClick(() => this.onClickOk());
+
+		this._dialog.registerCloseValidator(() => this.handleValidate());
+
+		await this.apiWrapper.openDialog(this._dialog);
+	}
+
+	private onClickOk(): void {
+		let jobName = localize('sparkJobSubmission_SubmitSparkJob', '{0} Spark Job Submission:',
+			this._sparkConfigTab.getInputValues()[0]);
+		this.apiWrapper.startBackgroundOperation(
+			{
+				connection: this.sqlClusterConnection.connection,
+				displayName: jobName,
+				description: jobName,
+				isCancelable: false,
+				operation: op => {
+					this.onSubmit(op);
+				}
+			}
+		);
+	}
+
+	private async onSubmit(op: sqlops.BackgroundOperation): Promise<void> {
+		try {
+			this.outputChannel.show();
+			let msg = localize('sparkJobSubmission_SubmissionStartMessage',
+				'.......................... Submit Spark Job Start ..........................');
+			this.outputChannel.appendLine(msg);
+			// 1. Upload local file to HDFS for local source.
+			if (this._dataModel.isMainSourceFromLocal) {
+				try {
+					this.outputChannel.appendLine(this.addInfoTag(LocalizedConstants.sparkJobSubmissionPrepareUploadingFile(this._dataModel.localFileSourcePath, this._dataModel.hdfsFolderDestinationPath)));
+					op.updateStatus(sqlops.TaskStatus.InProgress, LocalizedConstants.sparkJobSubmissionPrepareUploadingFile(this._dataModel.localFileSourcePath, this._dataModel.hdfsFolderDestinationPath));
+					await this._dataModel.uploadFile(this._dataModel.localFileSourcePath, this._dataModel.hdfsFolderDestinationPath);
+					vscode.window.showInformationMessage(LocalizedConstants.sparkJobSubmissionUploadingFileSucceeded);
+					this.outputChannel.appendLine(this.addInfoTag(LocalizedConstants.sparkJobSubmissionUploadingFileSucceeded));
+					op.updateStatus(sqlops.TaskStatus.InProgress, LocalizedConstants.sparkJobSubmissionUploadingFileSucceeded);
+				} catch (error) {
+					vscode.window.showErrorMessage(LocalizedConstants.sparkJobSubmissionUploadingFileFailed(utils.getErrorMessage(error)));
+					this.outputChannel.appendLine(this.addErrorTag(LocalizedConstants.sparkJobSubmissionUploadingFileFailed(utils.getErrorMessage(error))));
+					op.updateStatus(sqlops.TaskStatus.Failed, LocalizedConstants.sparkJobSubmissionUploadingFileFailed(utils.getErrorMessage(error)));
+					this.outputChannel.appendLine(LocalizedConstants.sparkJobSubmissionEndMessage);
+					return;
+				}
+			}
+
+			// 2. Submit job to cluster.
+			let submissionSettings: SparkJobSubmissionInput = this.getSubmissionInput();
+			this.outputChannel.appendLine(this.addInfoTag(LocalizedConstants.sparkJobSubmissionPrepareSubmitJob(submissionSettings.jobName)));
+			op.updateStatus(sqlops.TaskStatus.InProgress, LocalizedConstants.sparkJobSubmissionPrepareSubmitJob(submissionSettings.jobName));
+			let livyBatchId = await this._dataModel.submitBatchJobByLivy(submissionSettings);
+			vscode.window.showInformationMessage(LocalizedConstants.sparkJobSubmissionSparkJobHasBeenSubmitted);
+			this.outputChannel.appendLine(this.addInfoTag(LocalizedConstants.sparkJobSubmissionSparkJobHasBeenSubmitted));
+			op.updateStatus(sqlops.TaskStatus.InProgress, LocalizedConstants.sparkJobSubmissionSparkJobHasBeenSubmitted);
+
+			// 3. Get SparkHistory/YarnUI Url.
+			try {
+				let appId = await this._dataModel.getApplicationID(submissionSettings, livyBatchId);
+
+				let sparkHistoryUrl = this._dataModel.generateSparkHistoryUIUrl(submissionSettings, appId);
+				vscode.window.showInformationMessage(LocalizedConstants.sparkJobSubmissionSparkHistoryLinkMessage(sparkHistoryUrl));
+				this.outputChannel.appendLine(this.addInfoTag(LocalizedConstants.sparkJobSubmissionSparkHistoryLinkMessage(sparkHistoryUrl)));
+				op.updateStatus(sqlops.TaskStatus.Succeeded, LocalizedConstants.sparkJobSubmissionSparkHistoryLinkMessage(sparkHistoryUrl));
+
+				/*
+				// Spark Tracking URl is not working now.
+				let sparkTrackingUrl = this._dataModel.generateSparkTrackingUIUrl(submissionSettings, appId);
+				vscode.window.showInformationMessage(LocalizedConstants.sparkJobSubmissionTrackingLinkMessage(sparkTrackingUrl));
+				this.outputChannel.appendLine(this.addInfoTag(LocalizedConstants.sparkJobSubmissionTrackingLinkMessage(sparkTrackingUrl)));
+				op.updateStatus(sqlops.TaskStatus.Succeeded, LocalizedConstants.sparkJobSubmissionTrackingLinkMessage(sparkTrackingUrl));
+				*/
+
+				let yarnUIUrl = this._dataModel.generateYarnUIUrl(submissionSettings, appId);
+				vscode.window.showInformationMessage(LocalizedConstants.sparkJobSubmissionYarnUIMessage(yarnUIUrl));
+				this.outputChannel.appendLine(this.addInfoTag(LocalizedConstants.sparkJobSubmissionYarnUIMessage(yarnUIUrl)));
+				op.updateStatus(sqlops.TaskStatus.Succeeded, LocalizedConstants.sparkJobSubmissionYarnUIMessage(yarnUIUrl));
+			} catch (error) {
+				vscode.window.showErrorMessage(LocalizedConstants.sparkJobSubmissionGetApplicationIdFailed(utils.getErrorMessage(error)));
+				this.outputChannel.appendLine(this.addErrorTag(LocalizedConstants.sparkJobSubmissionGetApplicationIdFailed(utils.getErrorMessage(error))));
+				op.updateStatus(sqlops.TaskStatus.Failed, LocalizedConstants.sparkJobSubmissionGetApplicationIdFailed(utils.getErrorMessage(error)));
+				this.outputChannel.appendLine(LocalizedConstants.sparkJobSubmissionEndMessage);
+				return;
+			}
+
+			this.outputChannel.appendLine(LocalizedConstants.sparkJobSubmissionEndMessage);
+		} catch (error) {
+			vscode.window.showErrorMessage(LocalizedConstants.sparkJobSubmissionSubmitJobFailed(utils.getErrorMessage(error)));
+			this.outputChannel.appendLine(this.addErrorTag(LocalizedConstants.sparkJobSubmissionSubmitJobFailed(utils.getErrorMessage(error))));
+			op.updateStatus(sqlops.TaskStatus.Failed, LocalizedConstants.sparkJobSubmissionSubmitJobFailed(utils.getErrorMessage(error)));
+			this.outputChannel.appendLine(LocalizedConstants.sparkJobSubmissionEndMessage);
+		}
+	}
+
+	private async handleValidate(): Promise<boolean> {
+		return this._sparkConfigTab.validate();
+	}
+
+	private getSubmissionInput(): SparkJobSubmissionInput {
+		let generalConfig = this._sparkConfigTab.getInputValues();
+		let advancedConfig = this._sparkAdvancedTab.getInputValues();
+		return new SparkJobSubmissionInput(generalConfig[0], this._dataModel.hdfsSubmitFilePath, generalConfig[1], generalConfig[2],
+			advancedConfig[0], advancedConfig[1], advancedConfig[2]);
+	}
+
+	private addInfoTag(info: string): string {
+		return `[Info]  ${info}`;
+	}
+
+	private addErrorTag(error: string): string {
+		return `[Error] ${error}`;
+	}
+}

extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkJobSubmissionModel.ts

@@ -0,0 +1,206 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+import * as fs from 'fs';
+import * as fspath from 'path';
+import * as os from 'os';
+
+import * as constants from '../../../constants';
+import { SqlClusterConnection } from '../../../objectExplorerNodeProvider/connection';
+import * as LocalizedConstants from '../../../localizedConstants';
+import * as utils from '../../../utils';
+import { SparkJobSubmissionService, SparkJobSubmissionInput, LivyLogResponse } from './sparkJobSubmissionService';
+import { AppContext } from '../../../appContext';
+import { IFileSource, File, joinHdfsPath } from '../../../objectExplorerNodeProvider/fileSources';
+
+
+// Stores important state and service methods used by the Spark Job Submission Dialog.
+export class SparkJobSubmissionModel {
+	private _dialogService: SparkJobSubmissionService;
+	private _guidForClusterFolder: string;
+	public get guidForClusterFolder(): string { return this._guidForClusterFolder; }
+
+	// Whether the file is from local or HDFS
+	public isMainSourceFromLocal: boolean;
+
+	// indicate the final path to be submitted within HDFS
+	public hdfsSubmitFilePath: string;
+
+	// local file uploading related path: source; destinationFolder
+	public localFileSourcePath: string;
+	public hdfsFolderDestinationPath: string;
+
+	constructor(
+		private readonly _sqlClusterConnection: SqlClusterConnection,
+		private readonly _dialog: sqlops.window.modelviewdialog.Dialog,
+		private readonly _appContext: AppContext,
+		requestService?: (args: any) => any) {
+
+		if (!this._sqlClusterConnection || !this._dialog || !this._appContext) {
+			throw new Error(localize('sparkJobSubmission_SparkJobSubmissionModelInitializeError',
+				'Parameters for SparkJobSubmissionModel is illegal'));
+		}
+
+		this._dialogService = new SparkJobSubmissionService(requestService);
+		this._guidForClusterFolder = utils.generateGuid();
+	}
+
+	public get connection(): SqlClusterConnection { return this._sqlClusterConnection; }
+	public get dialogService(): SparkJobSubmissionService { return this._dialogService; }
+	public get dialog(): sqlops.window.modelviewdialog.Dialog { return this._dialog; }
+
+	public isJarFile(): boolean {
+		if (this.hdfsSubmitFilePath) {
+			return this.hdfsSubmitFilePath.toLowerCase().endsWith('jar');
+		}
+
+		return false;
+	}
+
+	public showDialogError(message: string): void {
+		let errorLevel = sqlops.window.modelviewdialog.MessageLevel ? sqlops.window.modelviewdialog.MessageLevel : 0;
+		this._dialog.message = {
+			text: message,
+			level: <sqlops.window.modelviewdialog.MessageLevel>errorLevel
+		};
+	}
+
+	public showDialogInfo(message: string): void {
+		let infoLevel = sqlops.window.modelviewdialog.MessageLevel ? sqlops.window.modelviewdialog.MessageLevel.Information : 2;
+		this._dialog.message = {
+			text: message,
+			level: infoLevel
+		};
+	}
+
+	public getSparkClusterUrl(): string {
+		if (this._sqlClusterConnection && this._sqlClusterConnection.host && this._sqlClusterConnection.port) {
+			return `https://${this._sqlClusterConnection.host}:${this._sqlClusterConnection.port}`;
+		}
+
+		// Only for safety check, Won't happen with correct Model initialize.
+		return '';
+	}
+
+	public async submitBatchJobByLivy(submissionArgs: SparkJobSubmissionInput): Promise<string> {
+		try {
+			if (!submissionArgs) {
+				return Promise.reject(localize('sparkJobSubmission_submissionArgsIsInvalid', 'submissionArgs is invalid. '));
+			}
+
+			submissionArgs.setSparkClusterInfo(this._sqlClusterConnection);
+			let livyBatchId = await this._dialogService.submitBatchJob(submissionArgs);
+			return livyBatchId;
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	public async getApplicationID(submissionArgs: SparkJobSubmissionInput, livyBatchId: string, retryTime?: number): Promise<string> {
+		// TODO: whether set timeout as 15000ms
+		try {
+			if (!submissionArgs) {
+				return Promise.reject(localize('sparkJobSubmission_submissionArgsIsInvalid', 'submissionArgs is invalid. '));
+			}
+
+			if (!utils.isValidNumber(livyBatchId)) {
+				return Promise.reject(new Error(localize('sparkJobSubmission_LivyBatchIdIsInvalid', 'livyBatchId is invalid. ')));
+			}
+
+			if (!retryTime) {
+				retryTime = constants.mssqlClusterLivyRetryTimesForCheckYarnApp;
+			}
+
+			submissionArgs.setSparkClusterInfo(this._sqlClusterConnection);
+			let response: LivyLogResponse = undefined;
+			let timeOutCount: number = 0;
+			do {
+				timeOutCount++;
+				await this.sleep(constants.mssqlClusterLivyTimeInMSForCheckYarnApp);
+				response = await this._dialogService.getYarnAppId(submissionArgs, livyBatchId);
+			} while (response.appId === '' && timeOutCount < retryTime);
+
+			if (response.appId === '') {
+				return Promise.reject(localize('sparkJobSubmission_GetApplicationIdTimeOut', 'Get Application Id time out. {0}[Log]   {1}', os.EOL, response.log));
+			} else {
+				return response.appId;
+			}
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	public async uploadFile(localFilePath: string, hdfsFolderPath: string): Promise<void> {
+		try {
+			if (!localFilePath || !hdfsFolderPath) {
+				return Promise.reject(localize('sparkJobSubmission_localFileOrFolderNotSpecified.', 'Property localFilePath or hdfsFolderPath is not specified. '));
+			}
+
+			if (!fs.existsSync(localFilePath)) {
+				return Promise.reject(LocalizedConstants.sparkJobSubmissionLocalFileNotExisted(localFilePath));
+			}
+
+			let fileSource: IFileSource = this._sqlClusterConnection.createHdfsFileSource();
+			await fileSource.writeFile(new File(localFilePath, false), hdfsFolderPath);
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	public async isClusterFileExisted(path: string): Promise<boolean> {
+		try {
+			if (!path) {
+				return Promise.reject(localize('sparkJobSubmission_PathNotSpecified.', 'Property Path is not specified. '));
+			}
+
+			let fileSource: IFileSource = this._sqlClusterConnection.createHdfsFileSource();
+			return await fileSource.exists(path);
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	public updateModelByLocalPath(localPath: string): void {
+		if (localPath) {
+			this.localFileSourcePath = localPath;
+			this.hdfsFolderDestinationPath = this.generateDestinationFolder();
+			let fileName = fspath.basename(localPath);
+			this.hdfsSubmitFilePath = joinHdfsPath(this.hdfsFolderDestinationPath, fileName);
+		} else {
+			this.hdfsSubmitFilePath = '';
+		}
+	}
+
+	// Example path: /SparkSubmission/2018/08/21/b682a6c4-1954-401e-8542-9c573d69d9c0/default_artifact.jar
+	private generateDestinationFolder(): string {
+		let day = new Date();
+		return `/SparkSubmission/${day.getUTCFullYear()}/${day.getUTCMonth() + 1}/${day.getUTCDate()}/${this._guidForClusterFolder}`;
+	}
+
+	// Example: https://host:30443/gateway/default/yarn/cluster/app/application_1532646201938_0057
+	public generateYarnUIUrl(submissionArgs: SparkJobSubmissionInput, appId: string): string {
+		return `https://${submissionArgs.host}:${submissionArgs.port}/gateway/default/yarn/cluster/app/${appId}`;
+	}
+
+	// Example: https://host:30443/gateway/default/yarn/proxy/application_1532646201938_0411
+	public generateSparkTrackingUIUrl(submissionArgs: SparkJobSubmissionInput, appId: string): string {
+		return `https://${submissionArgs.host}:${submissionArgs.port}/gateway/default/yarn/proxy/${appId}`;
+	}
+
+	// Example: https://host:30443/gateway/default/sparkhistory/history/application_1532646201938_0057/1
+	public generateSparkHistoryUIUrl(submissionArgs: SparkJobSubmissionInput, appId: string): string {
+		return `https://${submissionArgs.host}:${submissionArgs.port}/gateway/default/sparkhistory/history/${appId}/1`;
+	}
+
+	private async sleep(ms: number): Promise<{}> {
+		// tslint:disable-next-line no-string-based-set-timeout
+		return new Promise(resolve => setTimeout(resolve, ms));
+	}
+}

extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkJobSubmissionService.ts

@@ -0,0 +1,187 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as os from 'os';
+import * as nls from 'vscode-nls';
+const localize = nls.loadMessageBundle();
+import * as constants from '../../../constants';
+import { SqlClusterConnection } from '../../../objectExplorerNodeProvider/connection';
+import * as utils from '../../../utils';
+
+export class SparkJobSubmissionService {
+	private _requestPromise: (args: any) => any;
+
+	constructor(
+		requestService?: (args: any) => any) {
+		if (requestService) {
+			// this is to fake the request service for test.
+			this._requestPromise = requestService;
+		} else {
+			this._requestPromise = require('request-promise');
+		}
+	}
+
+	public async submitBatchJob(submissionArgs: SparkJobSubmissionInput): Promise<string> {
+		try {
+			let livyUrl: string = `https://${submissionArgs.host}:${submissionArgs.port}${submissionArgs.livyPath}/`;
+			let options = {
+				uri: livyUrl,
+				method: 'POST',
+				json: true,
+				// TODO, change it back after service's authentication changed.
+				rejectUnauthorized: false,
+				body: {
+					file: submissionArgs.sparkFile,
+					proxyUser: submissionArgs.user,
+					className: submissionArgs.mainClass,
+					name: submissionArgs.jobName
+				},
+				// authentication headers
+				headers: {
+					'Authorization': 'Basic ' + new Buffer(submissionArgs.user + ':' + submissionArgs.password).toString('base64')
+				}
+			};
+
+			// Set arguments
+			if (submissionArgs.jobArguments && submissionArgs.jobArguments.trim()) {
+				let argsList = submissionArgs.jobArguments.split(' ');
+				if (argsList.length > 0) {
+					options.body['args'] = argsList;
+				}
+			}
+
+			// Set jars files
+			if (submissionArgs.jarFileList && submissionArgs.jarFileList.trim()) {
+				let jarList = submissionArgs.jarFileList.split(';');
+				if (jarList.length > 0) {
+					options.body['jars'] = jarList;
+				}
+			}
+
+			// Set py files
+			if (submissionArgs.pyFileList && submissionArgs.pyFileList.trim()) {
+				let pyList = submissionArgs.pyFileList.split(';');
+				if (pyList.length > 0) {
+					options.body['pyFiles'] = pyList;
+				}
+			}
+
+			// Set other files
+			if (submissionArgs.otherFileList && submissionArgs.otherFileList.trim()) {
+				let otherList = submissionArgs.otherFileList.split(';');
+				if (otherList.length > 0) {
+					options.body['files'] = otherList;
+				}
+			}
+
+			const response = await this._requestPromise(options);
+			if (response && utils.isValidNumber(response.id)) {
+				return response.id;
+			}
+
+			return Promise.reject(new Error(localize('sparkJobSubmission_LivyNoBatchIdReturned',
+				'No Spark job batch id is returned from response.{0}[Error] {1}', os.EOL, JSON.stringify(response))));
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	public async getYarnAppId(submissionArgs: SparkJobSubmissionInput, livyBatchId: string): Promise<LivyLogResponse> {
+		try {
+			let livyUrl = `https://${submissionArgs.host}:${submissionArgs.port}${submissionArgs.livyPath}/${livyBatchId}/log`;
+			let options = {
+				uri: livyUrl,
+				method: 'GET',
+				json: true,
+				rejectUnauthorized: false,
+				// authentication headers
+				headers: {
+					'Authorization': 'Basic ' + new Buffer(submissionArgs.user + ':' + submissionArgs.password).toString('base64')
+				}
+			};
+
+			const response = await this._requestPromise(options);
+			if (response && response.log) {
+				return this.extractYarnAppIdFromLog(response.log);
+			}
+
+			return Promise.reject(localize('sparkJobSubmission_LivyNoLogReturned',
+				'No log is returned within response.{0}[Error] {1}', os.EOL, JSON.stringify(response)));
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+
+	private extractYarnAppIdFromLog(log: any): LivyLogResponse {
+		let logForPrint = log;
+		if (Array.isArray(log)) {
+			logForPrint = log.join(os.EOL);
+		}
+
+		// eg: '18/08/23 11:02:50 INFO yarn.Client: Application report for application_1532646201938_0182 (state: ACCEPTED)'
+		for (let entry of log) {
+			if (entry.indexOf('Application report for') >= 0 && entry.indexOf('(state: ACCEPTED)') >= 0) {
+				let tokens = entry.split(' ');
+				for (let token of tokens) {
+					if (token.startsWith('application_')) {
+						return new LivyLogResponse(logForPrint, token);
+					}
+				}
+			}
+		}
+
+		return new LivyLogResponse(logForPrint, '');
+	}
+}
+
+export class SparkJobSubmissionInput {
+	public setSparkClusterInfo(sqlClusterConnection: SqlClusterConnection): void {
+		this._host = sqlClusterConnection.host;
+		this._port = sqlClusterConnection.port;
+		this._livyPath = constants.mssqlClusterLivySubmitPath;
+		this._user = sqlClusterConnection.user;
+		this._passWord = sqlClusterConnection.password;
+	}
+
+	constructor(
+		private readonly _jobName: string,
+		private readonly _sparkFile: string,
+		private readonly _mainClass: string,
+		private readonly _arguments: string,
+		private readonly _jarFileList: string,
+		private readonly _pyFileList: string,
+		private readonly _otherFileList: string,
+		private _host?: string,
+		private _port?: string,
+		private _livyPath?: string,
+		private _user?: string,
+		private _passWord?: string) {
+	}
+
+	public get jobName(): string { return this._jobName; }
+	public get sparkFile(): string { return this._sparkFile; }
+	public get mainClass(): string { return this._mainClass; }
+	public get jobArguments(): string { return this._arguments; }
+	public get jarFileList(): string { return this._jarFileList; }
+	public get otherFileList(): string { return this._otherFileList; }
+	public get pyFileList(): string { return this._pyFileList; }
+	public get host(): string { return this._host; }
+	public get port(): string { return this._port; }
+	public get livyPath(): string { return this._livyPath; }
+	public get user(): string { return this._user; }
+	public get password(): string { return this._passWord; }
+}
+
+export enum SparkFileSource {
+	HDFS = <any>'HDFS',
+	Local = <any>'Local'
+}
+
+export class LivyLogResponse {
+	constructor(public log: string, public appId: string) { }
+}

extensions/mssql/src/sparkFeature/historyTask.ts

@@ -0,0 +1,45 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+import { AppContext } from '../appContext';
+import { getErrorMessage } from '../utils';
+import * as SqlClusterLookUp from '../sqlClusterLookUp';
+
+export class OpenSparkYarnHistoryTask {
+	constructor(private appContext: AppContext) {
+	}
+
+	async execute(sqlConnProfile: sqlops.IConnectionProfile, isSpark: boolean): Promise<void> {
+		try {
+			let sqlClusterConnection = SqlClusterLookUp.findSqlClusterConnection(sqlConnProfile, this.appContext);
+			if (!sqlClusterConnection)
+			{
+				let name = isSpark? 'Spark' : 'Yarn';
+				this.appContext.apiWrapper.showErrorMessage(`Please connect to the Spark cluster before View ${name} History.`);
+				return;
+			}
+			if (isSpark) {
+				vscode.commands.executeCommand('vscode.open', vscode.Uri.parse(this.generateSparkHistoryUrl(sqlClusterConnection.host, sqlClusterConnection.port)));
+			}
+			else {
+				vscode.commands.executeCommand('vscode.open', vscode.Uri.parse(this.generateYarnHistoryUrl(sqlClusterConnection.host, sqlClusterConnection.port)));
+			}
+		} catch (error) {
+			this.appContext.apiWrapper.showErrorMessage(getErrorMessage(error));
+		}
+	}
+
+	private generateSparkHistoryUrl(host: string, port: string): string {
+		return `https://${host}:${port}/gateway/default/sparkhistory/`;
+	}
+
+	private generateYarnHistoryUrl(host: string, port: string): string {
+		return `https://${host}:${port}/gateway/default/yarn/cluster/apps`;
+	}
+}

extensions/mssql/src/sparkFeature/sparkUtils.ts

@@ -0,0 +1,220 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as childProcess from 'child_process';
+import * as fs from 'fs-extra';
+import * as nls from 'vscode-nls';
+import * as path from 'path';
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+import * as which from 'which';
+import * as Constants from '../constants';
+
+const localize = nls.loadMessageBundle();
+
+export function getDropdownValue(dropdownValue: string | sqlops.CategoryValue): string {
+	if (typeof(dropdownValue) === 'string') {
+		return <string>dropdownValue;
+	} else {
+		return dropdownValue ? (<sqlops.CategoryValue>dropdownValue).name : undefined;
+	}
+}
+
+export function getServerAddressFromName(connection: sqlops.ConnectionInfo | string): string {
+	// Strip TDS port number from the server URI
+	if ((<sqlops.ConnectionInfo>connection).options && (<sqlops.ConnectionInfo>connection).options['host']) {
+		return (<sqlops.ConnectionInfo>connection).options['host'].split(',')[0].split(':')[0];
+	} else if ((<sqlops.ConnectionInfo>connection).options && (<sqlops.ConnectionInfo>connection).options['server']) {
+		return (<sqlops.ConnectionInfo>connection).options['server'].split(',')[0].split(':')[0];
+	} else {
+		return (<string>connection).split(',')[0].split(':')[0];
+	}
+}
+
+export function getKnoxUrl(host: string, port: string): string {
+	return `https://${host}:${port}/gateway`;
+}
+
+export function getLivyUrl(serverName: string, port: string): string {
+	return this.getKnoxUrl(serverName, port) + '/default/livy/v1/';
+}
+
+export function getTemplatePath(extensionPath: string, templateName: string): string {
+	return path.join(extensionPath, 'resources', templateName);
+}
+export function shellWhichResolving(cmd: string): Promise<string> {
+	return new Promise<string>(resolve => {
+		which(cmd, (err, foundPath) => {
+			if (err) {
+				resolve(undefined);
+			} else {
+				// NOTE: Using realpath b/c some system installs are symlinked from */bin
+				resolve(fs.realpathSync(foundPath));
+			}
+		});
+	});
+}
+
+export async function mkDir(dirPath: string, outputChannel?: vscode.OutputChannel): Promise<void> {
+	if (!await fs.exists(dirPath)) {
+		if (outputChannel) {
+			outputChannel.appendLine(localize('mkdirOutputMsg', '... Creating {0}', dirPath));
+		}
+		await fs.ensureDir(dirPath);
+	}
+}
+
+export function getErrorMessage(error: Error | string): string {
+	return (error instanceof Error) ? error.message : error;
+}
+
+// COMMAND EXECUTION HELPERS ///////////////////////////////////////////////
+export function executeBufferedCommand(cmd: string, options: childProcess.ExecOptions, outputChannel?: vscode.OutputChannel): Thenable<string> {
+	return new Promise<string>((resolve, reject) => {
+		if (outputChannel) {
+			outputChannel.appendLine(`	> ${cmd}`);
+		}
+
+		let child = childProcess.exec(cmd, options, (err, stdout) => {
+			if (err) {
+				reject(err);
+			} else {
+				resolve(stdout);
+			}
+		});
+
+		// Add listeners to print stdout and stderr if an output channel was provided
+		if (outputChannel) {
+			child.stdout.on('data', data => { outputDataChunk(data, outputChannel, '	stdout: '); });
+			child.stderr.on('data', data => { outputDataChunk(data, outputChannel, '	stderr: '); });
+		}
+	});
+}
+
+export function executeExitCodeCommand(cmd: string, outputChannel?: vscode.OutputChannel): Thenable<number> {
+	return new Promise<number>((resolve, reject) => {
+		if (outputChannel) {
+			outputChannel.appendLine(`	> ${cmd}`);
+		}
+
+		let child = childProcess.spawn(cmd, [], { shell: true, detached: false });
+
+		// Add listeners for the process to exit
+		child.on('error', reject);
+		child.on('exit', (code: number) => { resolve(code); });
+
+		// Add listeners to print stdout and stderr if an output channel was provided
+		if (outputChannel) {
+			child.stdout.on('data', data => { outputDataChunk(data, outputChannel, '	stdout: '); });
+			child.stderr.on('data', data => { outputDataChunk(data, outputChannel, '	stderr: '); });
+		}
+	});
+}
+
+export function executeStreamedCommand(cmd: string, outputChannel?: vscode.OutputChannel): Thenable<void> {
+	return new Promise<void>((resolve, reject) => {
+		// Start the command
+		if (outputChannel) {
+			outputChannel.appendLine(`	> ${cmd}`);
+		}
+		let child = childProcess.spawn(cmd, [], { shell: true, detached: false });
+
+		// Add listeners to resolve/reject the promise on exit
+		child.on('error', reject);
+		child.on('exit', (code: number) => {
+			if (code === 0) {
+				resolve();
+			} else {
+				reject(localize('executeCommandProcessExited', 'Process exited with code {0}', code));
+			}
+		});
+
+		// Add listeners to print stdout and stderr if an output channel was provided
+		if (outputChannel) {
+			child.stdout.on('data', data => { outputDataChunk(data, outputChannel, '	stdout: '); });
+			child.stderr.on('data', data => { outputDataChunk(data, outputChannel, '	stderr: '); });
+		}
+	});
+}
+
+export function isObjectExplorerContext(object: any): object is sqlops.ObjectExplorerContext {
+	return 'connectionProfile' in object && 'isConnectionNode' in object;
+}
+
+export function getUserHome(): string {
+	return process.env.HOME || process.env.USERPROFILE;
+}
+
+export enum Platform {
+	Mac,
+	Linux,
+	Windows,
+	Others
+}
+
+export function getOSPlatform(): Platform {
+	switch (process.platform) {
+		case 'win32':
+			return Platform.Windows;
+		case 'darwin':
+			return Platform.Mac;
+		case 'linux':
+			return Platform.Linux;
+		default:
+			return Platform.Others;
+	}
+}
+
+export function getOSPlatformId(): string {
+	var platformId = undefined;
+	switch (process.platform) {
+		case 'win32':
+			platformId = 'win-x64';
+			break;
+		case 'darwin':
+			platformId = 'osx';
+			break;
+		default:
+			platformId = 'linux-x64';
+			break;
+	}
+	return platformId;
+}
+
+// PRIVATE HELPERS /////////////////////////////////////////////////////////
+function outputDataChunk(data: string | Buffer, outputChannel: vscode.OutputChannel, header: string): void {
+	data.toString().split(/\r?\n/)
+		.forEach(line => {
+			outputChannel.appendLine(header + line);
+		});
+}
+
+export function clone<T>(obj: T): T {
+	if (!obj || typeof obj !== 'object') {
+		return obj;
+	}
+	if (obj instanceof RegExp) {
+		// See https://github.com/Microsoft/TypeScript/issues/10990
+		return obj as any;
+	}
+	const result = (Array.isArray(obj)) ? <any>[] : <any>{};
+	Object.keys(obj).forEach(key => {
+		if (obj[key] && typeof obj[key] === 'object') {
+			result[key] = clone(obj[key]);
+		} else {
+			result[key] = obj[key];
+		}
+	});
+	return result;
+}
+
+export function isValidNumber(maybeNumber: any) {
+	return maybeNumber !== undefined
+		&& maybeNumber !== null
+		&& maybeNumber !== ''
+		&& !isNaN(Number(maybeNumber.toString()));
+}

extensions/mssql/src/sqlClusterLookUp.ts

@@ -0,0 +1,146 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as sqlops from 'sqlops';
+import * as constants from './constants';
+import * as UUID from 'vscode-languageclient/lib/utils/uuid';
+import { AppContext } from './appContext';
+import { SqlClusterConnection } from './objectExplorerNodeProvider/connection';
+import { ICommandObjectExplorerContext } from './objectExplorerNodeProvider/command';
+import { MssqlObjectExplorerNodeProvider } from './objectExplorerNodeProvider/objectExplorerNodeProvider';
+
+
+export function findSqlClusterConnection(
+	obj: ICommandObjectExplorerContext | sqlops.IConnectionProfile,
+	appContext: AppContext) : SqlClusterConnection  {
+
+	if (!obj || !appContext) { return undefined; }
+
+	let sqlConnProfile: sqlops.IConnectionProfile;
+	if ('type' in obj && obj.type === constants.ObjectExplorerService
+		&& 'explorerContext' in obj && obj.explorerContext && obj.explorerContext.connectionProfile) {
+		sqlConnProfile = obj.explorerContext.connectionProfile;
+	} else if ('options' in obj) {
+		sqlConnProfile = obj;
+	}
+
+	let sqlClusterConnection: SqlClusterConnection = undefined;
+	if (sqlConnProfile) {
+		sqlClusterConnection = findSqlClusterConnectionBySqlConnProfile(sqlConnProfile, appContext);
+	}
+	return sqlClusterConnection;
+}
+
+function findSqlClusterConnectionBySqlConnProfile(sqlConnProfile: sqlops.IConnectionProfile, appContext: AppContext): SqlClusterConnection {
+	if (!sqlConnProfile || !appContext) { return undefined; }
+
+	let sqlOeNodeProvider = appContext.getService<MssqlObjectExplorerNodeProvider>(constants.ObjectExplorerService);
+	if (!sqlOeNodeProvider) { return undefined; }
+
+	let sqlClusterSession = sqlOeNodeProvider.findSqlClusterSessionBySqlConnProfile(sqlConnProfile);
+	if (!sqlClusterSession) { return undefined; }
+
+	return  sqlClusterSession.sqlClusterConnection;
+}
+
+export async function getSqlClusterConnection(
+	obj: sqlops.IConnectionProfile | sqlops.connection.Connection | ICommandObjectExplorerContext): Promise<ConnectionParam> {
+
+	if (!obj) { return undefined; }
+
+	let sqlClusterConnInfo: ConnectionParam = undefined;
+	if ('providerName' in obj) {
+		if (obj.providerName === constants.mssqlClusterProviderName) {
+			sqlClusterConnInfo = 'id' in obj ? connProfileToConnectionParam(obj) : connToConnectionParam(obj);
+		} else {
+			sqlClusterConnInfo = await createSqlClusterConnInfo(obj);
+		}
+	} else {
+		sqlClusterConnInfo = await createSqlClusterConnInfo(obj.explorerContext.connectionProfile);
+	}
+
+	return sqlClusterConnInfo;
+}
+
+async function createSqlClusterConnInfo(sqlConnInfo: sqlops.IConnectionProfile | sqlops.connection.Connection): Promise<ConnectionParam> {
+	if (!sqlConnInfo) { return undefined; }
+
+	let connectionId: string = 'id' in sqlConnInfo ? sqlConnInfo.id : sqlConnInfo.connectionId;
+	if (!connectionId) { return undefined; }
+
+	let serverInfo = await sqlops.connection.getServerInfo(connectionId);
+	if (!serverInfo || !serverInfo.options) { return undefined; }
+
+	let endpoints: IEndpoint[] = serverInfo.options[constants.clusterEndpointsProperty];
+	if (!endpoints || endpoints.length === 0) { return undefined; }
+
+	let index = endpoints.findIndex(ep => ep.serviceName === constants.hadoopKnoxEndpointName);
+	if (index < 0) { return undefined; }
+
+	let credentials = await sqlops.connection.getCredentials(connectionId);
+	if (!credentials) { return undefined; }
+
+	let clusterConnInfo = <ConnectionParam>{
+		providerName: constants.mssqlClusterProviderName,
+		connectionId: UUID.generateUuid(),
+		options: {}
+	};
+
+	clusterConnInfo.options[constants.hostPropName] = endpoints[index].ipAddress;
+	clusterConnInfo.options[constants.knoxPortPropName] = endpoints[index].port;
+	clusterConnInfo.options[constants.userPropName] = 'root'; //should be the same user as sql master
+	clusterConnInfo.options[constants.passwordPropName] = credentials.password;
+	clusterConnInfo = connToConnectionParam(clusterConnInfo);
+
+	return clusterConnInfo;
+}
+
+function connProfileToConnectionParam(connectionProfile: sqlops.IConnectionProfile): ConnectionParam {
+	let result = Object.assign(connectionProfile, { connectionId: connectionProfile.id });
+	return <ConnectionParam>result;
+}
+
+function connToConnectionParam(connection: sqlops.connection.Connection): ConnectionParam {
+	let connectionId = connection.connectionId;
+	let options = connection.options;
+	let result = Object.assign(connection,
+		{
+			serverName: `${options[constants.hostPropName]},${options[constants.knoxPortPropName]}`,
+			userName: options[constants.userPropName],
+			password: options[constants.passwordPropName],
+			id: connectionId,
+		}
+	);
+	return <ConnectionParam>result;
+}
+
+interface IEndpoint {
+	serviceName: string;
+	ipAddress: string;
+	port: number;
+}
+
+class ConnectionParam implements sqlops.connection.Connection, sqlops.IConnectionProfile, sqlops.ConnectionInfo
+{
+	public connectionName: string;
+	public serverName: string;
+	public databaseName: string;
+	public userName: string;
+	public password: string;
+	public authenticationType: string;
+	public savePassword: boolean;
+	public groupFullName: string;
+	public groupId: string;
+	public saveProfile: boolean;
+	public id: string;
+	public azureTenantId?: string;
+
+	public providerName: string;
+	public connectionId: string;
+
+	public options: { [name: string]: any; };
+}

extensions/mssql/src/typings/refs.d.ts

@@ -4,4 +4,5 @@
  *--------------------------------------------------------------------------------------------*/
 
 /// <reference path='../../../../src/sql/sqlops.d.ts'/>
-/// <reference path='../../../../src/vs/vscode.d.ts'/>
+/// <reference path='../../../../src/sql/sqlops.proposed.d.ts'/>
+/// <reference path='../../../../src/vs/vscode.d.ts'/>

extensions/mssql/src/utils.ts

@@ -4,16 +4,19 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
 import * as path from 'path';
 import * as crypto from 'crypto';
 import * as os from 'os';
-import {workspace, WorkspaceConfiguration} from 'vscode';
 import * as findRemoveSync from 'find-remove';
+import * as constants from './constants';
 
 const configTracingLevel = 'tracingLevel';
 const configLogRetentionMinutes = 'logRetentionMinutes';
 const configLogFilesRemovalLimit = 'logFilesRemovalLimit';
 const extensionConfigSectionName = 'mssql';
+const configLogDebugInfo = 'logDebugInfo';
 
 // The function is a duplicate of \src\paths.js. IT would be better to import path.js but it doesn't
 // work for now because the extension is running in different process.
@@ -27,56 +30,53 @@ export function getAppDataPath() {
 	}
 }
 
-export function removeOldLogFiles(prefix: string) : JSON {
-	return findRemoveSync(getDefaultLogDir(), {prefix: `${prefix}_`,  age: {seconds: getConfigLogRetentionSeconds()}, limit: getConfigLogFilesRemovalLimit()});
+export function removeOldLogFiles(prefix: string): JSON {
+	return findRemoveSync(getDefaultLogDir(), { prefix: `${prefix}_`, age: { seconds: getConfigLogRetentionSeconds() }, limit: getConfigLogFilesRemovalLimit() });
 }
 
-export function getConfiguration(config: string = extensionConfigSectionName) : WorkspaceConfiguration {
-	return workspace.getConfiguration(extensionConfigSectionName);
+export function getConfiguration(config: string = extensionConfigSectionName): vscode.WorkspaceConfiguration {
+	return vscode.workspace.getConfiguration(extensionConfigSectionName);
 }
 
-export function getConfigLogFilesRemovalLimit() : number {
+export function getConfigLogFilesRemovalLimit(): number {
 	let config = getConfiguration();
 	if (config) {
 		return Number((config[configLogFilesRemovalLimit]).toFixed(0));
 	}
-	else
-	{
+	else {
 		return undefined;
 	}
 }
 
-export function getConfigLogRetentionSeconds() : number {
+export function getConfigLogRetentionSeconds(): number {
 	let config = getConfiguration();
 	if (config) {
 		return Number((config[configLogRetentionMinutes] * 60).toFixed(0));
 	}
-	else
-	{
+	else {
 		return undefined;
 	}
 }
 
-export function getConfigTracingLevel() : string {
+export function getConfigTracingLevel(): string {
 	let config = getConfiguration();
 	if (config) {
 		return config[configTracingLevel];
 	}
-	else
-	{
+	else {
 		return undefined;
 	}
 }
 
-export function getDefaultLogDir() : string {
-	return path.join(process.env['VSCODE_LOGS'], '..', '..','mssql');
+export function getDefaultLogDir(): string {
+	return path.join(process.env['VSCODE_LOGS'], '..', '..', 'mssql');
 }
 
-export function getDefaultLogFile(prefix: string, pid: number) : string {
+export function getDefaultLogFile(prefix: string, pid: number): string {
 	return path.join(getDefaultLogDir(), `${prefix}_${pid}.log`);
 }
 
-export function getCommonLaunchArgsAndCleanupOldLogFiles(prefix: string, executablePath: string) : string [] {
+export function getCommonLaunchArgsAndCleanupOldLogFiles(prefix: string, executablePath: string): string[] {
 	let launchArgs = [];
 	launchArgs.push('--log-file');
 	let logFile = getDefaultLogFile(prefix, process.pid);
@@ -169,3 +169,64 @@ export function verifyPlatform(): Thenable<boolean> {
 		return Promise.resolve(true);
 	}
 }
+
+export function getErrorMessage(error: Error | string): string {
+	return (error instanceof Error) ? error.message : error;
+}
+
+export function isObjectExplorerContext(object: any): object is sqlops.ObjectExplorerContext {
+	return 'connectionProfile' in object && 'isConnectionNode' in object;
+}
+
+export function getUserHome(): string {
+	return process.env.HOME || process.env.USERPROFILE;
+}
+
+export async function getClusterEndpoint(profileId: string, serviceName: string): Promise<IEndpoint> {
+
+	let serverInfo: sqlops.ServerInfo = await sqlops.connection.getServerInfo(profileId);
+	if (!serverInfo || !serverInfo.options) {
+		return undefined;
+	}
+	let endpoints: IEndpoint[] = serverInfo.options[constants.clusterEndpointsProperty];
+	if (!endpoints || endpoints.length === 0) {
+		return undefined;
+	}
+	let index = endpoints.findIndex(ep => ep.serviceName === serviceName);
+	if (index === -1) {
+		return undefined;
+	}
+	let clusterEndpoint: IEndpoint = {
+		serviceName: endpoints[index].serviceName,
+		ipAddress: endpoints[index].ipAddress,
+		port: endpoints[index].port
+	};
+	return clusterEndpoint;
+}
+
+interface IEndpoint {
+	serviceName: string;
+	ipAddress: string;
+	port: number;
+}
+
+export function isValidNumber(maybeNumber: any) {
+	return maybeNumber !== undefined
+		&& maybeNumber !== null
+		&& maybeNumber !== ''
+		&& !isNaN(Number(maybeNumber.toString()));
+}
+
+/**
+ * Helper to log messages to the developer console if enabled
+ * @param msg Message to log to the console
+ */
+export function logDebug(msg: any): void {
+	let config = vscode.workspace.getConfiguration(extensionConfigSectionName);
+	let logDebugInfo = config[configLogDebugInfo];
+	if (logDebugInfo === true) {
+		let currentTime = new Date().toLocaleTimeString();
+		let outputMsg = '[' + currentTime + ']: ' + msg ? msg.toString() : '';
+		console.log(outputMsg);
+	}
+}

extensions/mssql/tsconfig.json

@@ -11,7 +11,7 @@
         "emitDecoratorMetadata": true,
         "experimentalDecorators": true,
         "moduleResolution": "node",
-        "declaration": true
+        "declaration": false
     },
     "exclude": [
         "node_modules"

extensions/mssql/yarn.lock

@@ -9,6 +9,16 @@ agent-base@4, agent-base@^4.1.0:
   dependencies:
     es6-promisify "^5.0.0"
 
+ajv@^6.5.5:
+  version "6.8.1"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.8.1.tgz#0890b93742985ebf8973cd365c5b23920ce3cb20"
+  integrity sha512-eqxCp82P+JfqL683wwsL73XmFs1eG6qjw+RD3YHx+Jll1r0jNd4dh8QG9NYAeNGA/hnZjeEDgtTskgJULbxpWQ==
+  dependencies:
+    fast-deep-equal "^2.0.1"
+    fast-json-stable-stringify "^2.0.0"
+    json-schema-traverse "^0.4.1"
+    uri-js "^4.2.2"
+
 applicationinsights@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/applicationinsights/-/applicationinsights-1.0.1.tgz#53446b830fe8d5d619eee2a278b31d3d25030927"
@@ -18,10 +28,49 @@ applicationinsights@1.0.1:
     diagnostic-channel-publishers "0.2.1"
     zone.js "0.7.6"
 
-base64-js@0.0.8:
-  version "0.0.8"
-  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-0.0.8.tgz#1101e9544f4a76b1bc3b26d452ca96d7a35e7978"
-  integrity sha1-EQHpVE9KdrG8OybUUsqW16NeeXg=
+arch@^2.1.0:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/arch/-/arch-2.1.1.tgz#8f5c2731aa35a30929221bb0640eed65175ec84e"
+  integrity sha512-BLM56aPo9vLLFVa8+/+pJLnrZ7QGGTVHWsCwieAWT9o9K8UeGaQbzZbGoabWLOo2ksBCztoXdqBZBplqLDDCSg==
+
+asn1@~0.2.3:
+  version "0.2.4"
+  resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
+  integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==
+  dependencies:
+    safer-buffer "~2.1.0"
+
+assert-plus@1.0.0, assert-plus@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525"
+  integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=
+
+asynckit@^0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
+  integrity sha1-x57Zf380y48robyXkLzDZkdLS3k=
+
+aws-sign2@~0.7.0:
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
+  integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=
+
+aws4@^1.8.0:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
+  integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==
+
+base64-js@^1.0.2:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.0.tgz#cab1e6118f051095e58b5281aea8c1cd22bfc0e3"
+  integrity sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==
+
+bcrypt-pbkdf@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
+  integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=
+  dependencies:
+    tweetnacl "^0.14.3"
 
 bl@^1.0.0:
   version "1.2.2"
@@ -31,6 +80,11 @@ bl@^1.0.0:
     readable-stream "^2.3.5"
     safe-buffer "^5.1.1"
 
+bluebird@^3.5.0:
+  version "3.5.3"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.3.tgz#7d01c6f9616c9a51ab0f8c549a79dfe6ec33efa7"
+  integrity sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==
+
 buffer-alloc-unsafe@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0"
@@ -54,14 +108,43 @@ buffer-fill@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c"
   integrity sha1-+PeLdniYiO858gXNY39o5wISKyw=
 
-buffer@^3.0.1:
-  version "3.6.0"
-  resolved "https://registry.yarnpkg.com/buffer/-/buffer-3.6.0.tgz#a72c936f77b96bf52f5f7e7b467180628551defb"
-  integrity sha1-pyyTb3e5a/UvX357RnGAYoVR3vs=
+buffer-stream-reader@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/buffer-stream-reader/-/buffer-stream-reader-0.1.1.tgz#ca8bf93631deedd8b8f8c3bb44991cc30951e259"
+  integrity sha1-yov5NjHe7di4+MO7RJkcwwlR4lk=
+
+buffer@^5.2.1:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.2.1.tgz#dd57fa0f109ac59c602479044dca7b8b3d0b71d6"
+  integrity sha512-c+Ko0loDaFfuPWiL02ls9Xd3GO3cPVmUobQ6t3rXNUk304u6hGq+8N/kFi+QEIKhzK3uwolVhLzszmfLmMLnqg==
   dependencies:
-    base64-js "0.0.8"
+    base64-js "^1.0.2"
     ieee754 "^1.1.4"
-    isarray "^1.0.0"
+
+bytes@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
+  integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
+
+caseless@~0.12.0:
+  version "0.12.0"
+  resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
+  integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=
+
+clipboardy@^1.2.3:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/clipboardy/-/clipboardy-1.2.3.tgz#0526361bf78724c1f20be248d428e365433c07ef"
+  integrity sha512-2WNImOvCRe6r63Gk9pShfkwXsVtKCroMAevIbiae021mS850UkWPbevxsBz3tnvjZIEGvlwaqCPsw+4ulzNgJA==
+  dependencies:
+    arch "^2.1.0"
+    execa "^0.8.0"
+
+combined-stream@^1.0.6, combined-stream@~1.0.6:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.7.tgz#2d1d24317afb8abe95d6d2c0b07b57813539d828"
+  integrity sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==
+  dependencies:
+    delayed-stream "~1.0.0"
 
 commander@~2.8.1:
   version "2.8.1"
@@ -70,14 +153,30 @@ commander@~2.8.1:
   dependencies:
     graceful-readlink ">= 1.0.0"
 
-core-util-is@~1.0.0:
+core-util-is@1.0.2, core-util-is@~1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
-"dataprotocol-client@github:Microsoft/sqlops-dataprotocolclient#0.2.10":
-  version "0.2.10"
-  resolved "https://codeload.github.com/Microsoft/sqlops-dataprotocolclient/tar.gz/4de3f7caf0eba54159911b977ddb4f5d7c0a9ca8"
+cross-spawn@^5.0.1:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-5.1.0.tgz#e8bd0efee58fcff6f8f94510a0a554bbfa235449"
+  integrity sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=
+  dependencies:
+    lru-cache "^4.0.1"
+    shebang-command "^1.2.0"
+    which "^1.2.9"
+
+dashdash@^1.12.0:
+  version "1.14.1"
+  resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
+  integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=
+  dependencies:
+    assert-plus "^1.0.0"
+
+"dataprotocol-client@github:Microsoft/sqlops-dataprotocolclient#0.2.15":
+  version "0.2.15"
+  resolved "https://codeload.github.com/Microsoft/sqlops-dataprotocolclient/tar.gz/a2cd2db109de882f0959f7b6421c86afa585f460"
   dependencies:
     vscode-languageclient "3.5.1"
 
@@ -148,6 +247,11 @@ decompress@^4.2.0:
     pify "^2.3.0"
     strip-dirs "^2.0.0"
 
+delayed-stream@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
+  integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk=
+
 diagnostic-channel-publishers@0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/diagnostic-channel-publishers/-/diagnostic-channel-publishers-0.2.1.tgz#8e2d607a8b6d79fe880b548bc58cc6beb288c4f3"
@@ -160,6 +264,14 @@ diagnostic-channel@0.2.0:
   dependencies:
     semver "^5.3.0"
 
+ecc-jsbn@~0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
+  integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=
+  dependencies:
+    jsbn "~0.1.0"
+    safer-buffer "^2.1.0"
+
 end-of-stream@^1.0.0:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43"
@@ -167,6 +279,13 @@ end-of-stream@^1.0.0:
   dependencies:
     once "^1.4.0"
 
+error-ex@^1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf"
+  integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==
+  dependencies:
+    is-arrayish "^0.2.1"
+
 es6-promise@^4.0.3:
   version "4.2.5"
   resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.5.tgz#da6d0d5692efb461e082c14817fe2427d8f5d054"
@@ -179,11 +298,54 @@ es6-promisify@^5.0.0:
   dependencies:
     es6-promise "^4.0.3"
 
+escape-string-regexp@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
+  integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=
+
 eventemitter2@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-5.0.1.tgz#6197a095d5fb6b57e8942f6fd7eaad63a09c9452"
   integrity sha1-YZegldX7a1folC9v1+qtY6CclFI=
 
+execa@^0.8.0:
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/execa/-/execa-0.8.0.tgz#d8d76bbc1b55217ed190fd6dd49d3c774ecfc8da"
+  integrity sha1-2NdrvBtVIX7RkP1t1J08d07PyNo=
+  dependencies:
+    cross-spawn "^5.0.1"
+    get-stream "^3.0.0"
+    is-stream "^1.1.0"
+    npm-run-path "^2.0.0"
+    p-finally "^1.0.0"
+    signal-exit "^3.0.0"
+    strip-eof "^1.0.0"
+
+extend@^3.0.0, extend@~3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
+  integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==
+
+extsprintf@1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05"
+  integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=
+
+extsprintf@^1.2.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f"
+  integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8=
+
+fast-deep-equal@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49"
+  integrity sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=
+
+fast-json-stable-stringify@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2"
+  integrity sha1-1RQsDK7msRifh9OnYREGT4bIu/I=
+
 fd-slicer@~1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e"
@@ -191,6 +353,13 @@ fd-slicer@~1.1.0:
   dependencies:
     pend "~1.2.0"
 
+figures@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/figures/-/figures-2.0.0.tgz#3ab1a2d2a62c8bfb431a0c94cb797a2fce27c962"
+  integrity sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=
+  dependencies:
+    escape-string-regexp "^1.0.5"
+
 file-type@^3.8.0:
   version "3.9.0"
   resolved "https://registry.yarnpkg.com/file-type/-/file-type-3.9.0.tgz#257a078384d1db8087bc449d107d52a52672b9e9"
@@ -206,6 +375,20 @@ file-type@^6.1.0:
   resolved "https://registry.yarnpkg.com/file-type/-/file-type-6.2.0.tgz#e50cd75d356ffed4e306dc4f5bcf52a79903a919"
   integrity sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==
 
+forever-agent@~0.6.1:
+  version "0.6.1"
+  resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
+  integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=
+
+form-data@~2.3.2:
+  version "2.3.3"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"
+  integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.6"
+    mime-types "^2.1.12"
+
 fs-constants@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
@@ -219,6 +402,18 @@ get-stream@^2.2.0:
     object-assign "^4.0.1"
     pinkie-promise "^2.0.0"
 
+get-stream@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14"
+  integrity sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=
+
+getpass@^0.1.1:
+  version "0.1.7"
+  resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa"
+  integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=
+  dependencies:
+    assert-plus "^1.0.0"
+
 graceful-fs@^4.1.10:
   version "4.1.15"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00"
@@ -229,6 +424,19 @@ graceful-fs@^4.1.10:
   resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725"
   integrity sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=
 
+har-schema@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
+  integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=
+
+har-validator@~5.1.0:
+  version "5.1.3"
+  resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.3.tgz#1ef89ebd3e4996557675eed9893110dc350fa080"
+  integrity sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==
+  dependencies:
+    ajv "^6.5.5"
+    har-schema "^2.0.0"
+
 http-proxy-agent@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz#e4821beef5b2142a2026bd73926fe537631c5405"
@@ -237,6 +445,15 @@ http-proxy-agent@^2.1.0:
     agent-base "4"
     debug "3.1.0"
 
+http-signature@~1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
+  integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=
+  dependencies:
+    assert-plus "^1.0.0"
+    jsprim "^1.2.2"
+    sshpk "^1.7.0"
+
 https-proxy-agent@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0"
@@ -255,6 +472,16 @@ inherits@~2.0.3:
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
   integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
 
+ip-regex@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
+  integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=
+
+is-arrayish@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
+  integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=
+
 is-natural-number@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/is-natural-number/-/is-natural-number-4.0.1.tgz#ab9d76e1db4ced51e35de0c72ebecf09f734cde8"
@@ -265,11 +492,69 @@ is-stream@^1.1.0:
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
   integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ=
 
-isarray@^1.0.0, isarray@~1.0.0:
+is-typedarray@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
+  integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=
+
+isarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
   integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
 
+isexe@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
+  integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=
+
+isstream@~0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
+  integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=
+
+jsbn@~0.1.0:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
+  integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM=
+
+json-schema-traverse@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
+  integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
+
+json-schema@0.2.3:
+  version "0.2.3"
+  resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13"
+  integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=
+
+json-stringify-safe@~5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
+  integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=
+
+jsprim@^1.2.2:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
+  integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=
+  dependencies:
+    assert-plus "1.0.0"
+    extsprintf "1.3.0"
+    json-schema "0.2.3"
+    verror "1.10.0"
+
+lodash@^4.13.1:
+  version "4.17.11"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
+  integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
+
+lru-cache@^4.0.1:
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
+  integrity sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==
+  dependencies:
+    pseudomap "^1.0.2"
+    yallist "^2.1.2"
+
 make-dir@^1.0.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"
@@ -277,6 +562,18 @@ make-dir@^1.0.0:
   dependencies:
     pify "^3.0.0"
 
+mime-db@~1.37.0:
+  version "1.37.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8"
+  integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==
+
+mime-types@^2.1.12, mime-types@~2.1.19:
+  version "2.1.21"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96"
+  integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==
+  dependencies:
+    mime-db "~1.37.0"
+
 minimist@0.0.8:
   version "0.0.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
@@ -299,6 +596,18 @@ ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a"
   integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==
 
+npm-run-path@^2.0.0:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f"
+  integrity sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=
+  dependencies:
+    path-key "^2.0.0"
+
+oauth-sign@~0.9.0:
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
+  integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==
+
 object-assign@^4.0.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
@@ -321,11 +630,26 @@ os-tmpdir@~1.0.2:
   resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
   integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=
 
+p-finally@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
+  integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=
+
+path-key@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
+  integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=
+
 pend@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
   integrity sha1-elfrVQpng/kRUzH89GY9XI4AelA=
 
+performance-now@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
+  integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
+
 pify@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
@@ -353,7 +677,32 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa"
   integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==
 
-readable-stream@^2.3.0, readable-stream@^2.3.5:
+pseudomap@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
+  integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM=
+
+psl@^1.1.24, psl@^1.1.28:
+  version "1.1.31"
+  resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.31.tgz#e9aa86d0101b5b105cbe93ac6b784cd547276184"
+  integrity sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==
+
+punycode@^1.4.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
+  integrity sha1-wNWmOycYgArY4esPpSachN1BhF4=
+
+punycode@^2.1.0, punycode@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
+  integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
+
+qs@~6.5.2:
+  version "6.5.2"
+  resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
+  integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
+
+readable-stream@^2.1.4, readable-stream@^2.3.0, readable-stream@^2.3.5:
   version "2.3.6"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
   integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==
@@ -366,11 +715,59 @@ readable-stream@^2.3.0, readable-stream@^2.3.5:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+request-promise-core@1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6"
+  integrity sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=
+  dependencies:
+    lodash "^4.13.1"
+
+request-promise@^4.2.2:
+  version "4.2.2"
+  resolved "https://registry.yarnpkg.com/request-promise/-/request-promise-4.2.2.tgz#d1ea46d654a6ee4f8ee6a4fea1018c22911904b4"
+  integrity sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=
+  dependencies:
+    bluebird "^3.5.0"
+    request-promise-core "1.1.1"
+    stealthy-require "^1.1.0"
+    tough-cookie ">=2.3.3"
+
+request@^2.74.0:
+  version "2.88.0"
+  resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
+  integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==
+  dependencies:
+    aws-sign2 "~0.7.0"
+    aws4 "^1.8.0"
+    caseless "~0.12.0"
+    combined-stream "~1.0.6"
+    extend "~3.0.2"
+    forever-agent "~0.6.1"
+    form-data "~2.3.2"
+    har-validator "~5.1.0"
+    http-signature "~1.2.0"
+    is-typedarray "~1.0.0"
+    isstream "~0.1.2"
+    json-stringify-safe "~5.0.1"
+    mime-types "~2.1.19"
+    oauth-sign "~0.9.0"
+    performance-now "^2.1.0"
+    qs "~6.5.2"
+    safe-buffer "^5.1.2"
+    tough-cookie "~2.4.3"
+    tunnel-agent "^0.6.0"
+    uuid "^3.3.2"
+
+safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
+safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
+  integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
+
 seek-bzip@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/seek-bzip/-/seek-bzip-1.0.5.tgz#cfe917cb3d274bcffac792758af53173eb1fabdc"
@@ -394,6 +791,50 @@ semver@^5.3.0:
     mkdirp "^0.5.1"
     tmp "^0.0.33"
 
+shebang-command@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea"
+  integrity sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=
+  dependencies:
+    shebang-regex "^1.0.0"
+
+shebang-regex@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
+  integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=
+
+signal-exit@^3.0.0:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d"
+  integrity sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=
+
+sshpk@^1.7.0:
+  version "1.16.1"
+  resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
+  integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==
+  dependencies:
+    asn1 "~0.2.3"
+    assert-plus "^1.0.0"
+    bcrypt-pbkdf "^1.0.0"
+    dashdash "^1.12.0"
+    ecc-jsbn "~0.1.1"
+    getpass "^0.1.1"
+    jsbn "~0.1.0"
+    safer-buffer "^2.0.2"
+    tweetnacl "~0.14.0"
+
+stealthy-require@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
+  integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
+
+stream-meter@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/stream-meter/-/stream-meter-1.0.4.tgz#52af95aa5ea760a2491716704dbff90f73afdd1d"
+  integrity sha1-Uq+Vql6nYKJJFxZwTb/5D3Ov3R0=
+  dependencies:
+    readable-stream "^2.1.4"
+
 string_decoder@~1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
@@ -408,6 +849,11 @@ strip-dirs@^2.0.0:
   dependencies:
     is-natural-number "^4.0.1"
 
+strip-eof@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"
+  integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=
+
 tar-stream@^1.5.2:
   version "1.6.2"
   resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.6.2.tgz#8ea55dab37972253d9a9af90fdcd559ae435c555"
@@ -421,7 +867,7 @@ tar-stream@^1.5.2:
     to-buffer "^1.1.1"
     xtend "^4.0.0"
 
-through@^2.3.6:
+through@^2.3.8:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
   integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=
@@ -438,19 +884,69 @@ to-buffer@^1.1.1:
   resolved "https://registry.yarnpkg.com/to-buffer/-/to-buffer-1.1.1.tgz#493bd48f62d7c43fcded313a03dcadb2e1213a80"
   integrity sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==
 
-unbzip2-stream@^1.0.9:
-  version "1.3.1"
-  resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.3.1.tgz#7854da51622a7e63624221196357803b552966a1"
-  integrity sha512-fIZnvdjblYs7Cru/xC6tCPVhz7JkYcVQQkePwMLyQELzYTds2Xn8QefPVnvdVhhZqubxNA1cASXEH5wcK0Bucw==
+tough-cookie@>=2.3.3:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-3.0.1.tgz#9df4f57e739c26930a018184887f4adb7dca73b2"
+  integrity sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==
   dependencies:
-    buffer "^3.0.1"
-    through "^2.3.6"
+    ip-regex "^2.1.0"
+    psl "^1.1.28"
+    punycode "^2.1.1"
+
+tough-cookie@~2.4.3:
+  version "2.4.3"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781"
+  integrity sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==
+  dependencies:
+    psl "^1.1.24"
+    punycode "^1.4.1"
+
+tunnel-agent@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"
+  integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=
+  dependencies:
+    safe-buffer "^5.0.1"
+
+tweetnacl@^0.14.3, tweetnacl@~0.14.0:
+  version "0.14.5"
+  resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
+  integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=
+
+unbzip2-stream@^1.0.9:
+  version "1.3.3"
+  resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz#d156d205e670d8d8c393e1c02ebd506422873f6a"
+  integrity sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg==
+  dependencies:
+    buffer "^5.2.1"
+    through "^2.3.8"
+
+uri-js@^4.2.2:
+  version "4.2.2"
+  resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0"
+  integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==
+  dependencies:
+    punycode "^2.1.0"
 
 util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
 
+uuid@^3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
+  integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
+
+verror@1.10.0:
+  version "1.10.0"
+  resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400"
+  integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=
+  dependencies:
+    assert-plus "^1.0.0"
+    core-util-is "1.0.2"
+    extsprintf "^1.2.0"
+
 vscode-extension-telemetry@^0.0.15:
   version "0.0.15"
   resolved "https://registry.yarnpkg.com/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.15.tgz#685c32f3b67e8fb85ba689c1d7f88ff90ff87856"
@@ -483,6 +979,27 @@ vscode-languageserver-types@3.5.0:
   resolved "https://registry.yarnpkg.com/vscode-languageserver-types/-/vscode-languageserver-types-3.5.0.tgz#e48d79962f0b8e02de955e3f524908e2b19c0374"
   integrity sha1-5I15li8LjgLelV4/UkkI4rGcA3Q=
 
+vscode-nls@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/vscode-nls/-/vscode-nls-4.0.0.tgz#4001c8a6caba5cedb23a9c5ce1090395c0e44002"
+  integrity sha512-qCfdzcH+0LgQnBpZA53bA32kzp9rpq/f66Som577ObeuDlFIrtbEJ+A/+CCxjIh4G8dpJYNCKIsxpRAHIfsbNw==
+
+webhdfs@^1.1.1:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/webhdfs/-/webhdfs-1.2.0.tgz#c41b08ae33944a0220863bfd4b6719b9aaec1d37"
+  integrity sha512-h8D/NT7ruDMuGCdJNEJHJh8vDTEtZ5hBL+eRzXTq/INTd92LKOhsTCwlQI+8kTt79qPZq5O8ev7j/Y19VeYCHQ==
+  dependencies:
+    buffer-stream-reader "^0.1.1"
+    extend "^3.0.0"
+    request "^2.74.0"
+
+which@^1.2.9:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
+  integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
+  dependencies:
+    isexe "^2.0.0"
+
 wrappy@1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
@@ -493,6 +1010,11 @@ xtend@^4.0.0:
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"
   integrity sha1-pcbVMr5lbiPbgg77lDofBJmNY68=
 
+yallist@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
+  integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=
+
 yauzl@^2.4.2:
   version "2.10.0"
   resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"

extensions/notebook/package.json

@@ -17,14 +17,28 @@
 			"type": "object",
 			"title": "%notebook.configuration.title%",
 			"properties": {
-				"notebook.enabled": {
+				"notebook.pythonPath": {
+					"type": "string",
+					"default": "",
+					"description": "%notebook.pythonPath.description%"
+				},
+				"notebook.sqlKernelEnabled": {
+					"type": "boolean",
+					"default": false,
+					"description": "%notebook.sqlKernelEnabled.description%"
+				},
+				"notebook.overrideEditorTheming": {
 					"type": "boolean",
 					"default": true,
-					"description": "%notebook.enabled.description%"
+					"description": "%notebook.overrideEditorTheming.description%"
 				}
 			}
 		},
 		"commands": [
+			{
+				"command": "notebook.command.analyzeNotebook",
+				"title": "%notebook.analyzeJupyterNotebook%"
+			},
 			{
 				"command": "notebook.command.new",
 				"title": "%notebook.command.new%",
@@ -40,25 +54,76 @@
 					"dark": "resources/dark/open_notebook_inverse.svg",
 					"light": "resources/light/open_notebook.svg"
 				}
+			},
+			{
+				"command": "notebook.command.runactivecell",
+				"title": "%notebook.command.runactivecell%"
+			},
+			{
+				"command": "notebook.command.addcode",
+				"title": "%notebook.command.addcode%"
+			},
+			{
+				"command": "notebook.command.addtext",
+				"title": "%notebook.command.addtext%"
 			}
 		],
 		"menus": {
 			"commandPalette": [
 				{
-					"command": "notebook.command.new",
-					"when": "config.notebook.enabled"
+					"command": "notebook.command.analyzeNotebook"
 				},
 				{
-					"command": "notebook.command.open",
-					"when": "config.notebook.enabled"
+					"command": "notebook.command.new"
+				},
+				{
+					"command": "notebook.command.open"
+				},
+				{
+					"command": "notebook.command.runactivecell",
+					"when": "notebookEditorVisible"
+				},
+				{
+					"command": "notebook.command.addcode",
+					"when": "notebookEditorVisible"
+				},
+				{
+					"command": "notebook.command.addtext",
+					"when": "notebookEditorVisible"
+				}
+			],
+			"objectExplorer/item/context": [
+				{
+					"command": "notebook.command.new",
+					"when": "connectionProvider == MSSQL && nodeType == dataservices",
+					"group": "1root@1"
+				},
+				{
+					"command": "notebook.command.analyzeNotebook",
+					"when": "nodeType=~/^mssqlCluster/ && nodeLabel=~/[^\\s]+(\\.(csv|tsv|txt))$/ && nodeType == mssqlCluster:file",
+					"group": "1notebook@1"
 				}
 			]
 		},
 		"keybindings": [
 			{
 				"command": "notebook.command.new",
-				"key": "Ctrl+Shift+N",
-				"when": "config.notebook.enabled"
+				"key": "Ctrl+Shift+N"
+			},
+			{
+				"command": "notebook.command.runactivecell",
+				"key": "F5",
+				"when": "notebookEditorVisible"
+			},
+			{
+				"command": "notebook.command.addcode",
+				"key": "Ctrl+Shift+C",
+				"when": "notebookEditorVisible"
+			},
+			{
+				"command": "notebook.command.addtext",
+				"key": "Ctrl+Shift+T",
+				"when": "notebookEditorVisible"
 			}
 		]
 	},
@@ -68,4 +133,4 @@
 	"devDependencies": {
 		"@types/node": "8.0.33"
 	}
-}
+}