deploy BDC wizard improvement for CU1 (#7756)

* unified admin user account (#7485)

* azdata changes

* spaces

* error message

* comments

* support AD authentication for bdc deployment (#7518)

* enable ad authentication

* remove export for internal interface

* add comments

* more changes after testing

* update notebooks

* escape slash

* more comments

* Update deploy-bdc-aks.ipynb

* Update deploy-bdc-existing-aks.ipynb

* Update deploy-bdc-existing-kubeadm.ipynb

* AD changes and review feedback (#7618)

* enable ad authentication

* remove export for internal interface

* add comments

* more changes after testing

* update notebooks

* escape slash

* more comments

* Update deploy-bdc-aks.ipynb

* Update deploy-bdc-existing-aks.ipynb

* Update deploy-bdc-existing-kubeadm.ipynb

* address comments from scenario review (#7546)

* support AD authentication for bdc deployment (#7518)

* enable ad authentication

* remove export for internal interface

* add comments

* more changes after testing

* update notebooks

* escape slash

* more comments

* Update deploy-bdc-aks.ipynb

* Update deploy-bdc-existing-aks.ipynb

* Update deploy-bdc-existing-kubeadm.ipynb

* scenario review feedbacks

* more fixes

* adjust the display order of resource types

* different way to implement left side buttons

* revert unwanted changes

* rename variable

* more fixes for the scenario review feedback (#7589)

* fix more issues

* add help links

* model view readonly text with links

* fix size string

* address comments

* update notebooks

* text update

* address the feedback of 2nd round of deploy BDC wizard review (#7646)

* 2nd review meeting comments

* fix the unit test failure

* recent changes in azdata

* notebook background execution with azdata (#7741)

* notebook background execution with azdata

* prompt to open notebook in case of failure

* fix path quote issue

* better temp file handling

* expose docker settings (#7751)

* add docker settings

* new icon for container image
This commit is contained in:
Alan Ren
2019-10-16 20:41:15 -07:00
committed by GitHub
parent 5d4da455bd
commit 2ab7a47353
40 changed files with 2019 additions and 730 deletions

View File

@@ -5,23 +5,42 @@
import * as path from 'path';
import { IPlatformService } from './platformService';
import { BigDataClusterDeploymentProfile } from './bigDataClusterDeploymentProfile';
import { BdcDeploymentType } from '../interfaces';
interface BdcConfigListOutput {
stdout: string[];
result: string[];
}
export interface BdcEndpoint {
endpoint: string;
name: 'sql-server-master';
}
export interface IAzdataService {
getDeploymentProfiles(): Promise<BigDataClusterDeploymentProfile[]>;
getDeploymentProfiles(deploymentType: BdcDeploymentType): Promise<BigDataClusterDeploymentProfile[]>;
getEndpoints(clusterName: string, userName: string, password: string): Promise<BdcEndpoint[]>;
}
export class AzdataService implements IAzdataService {
constructor(private platformService: IPlatformService) {
}
public async getDeploymentProfiles(): Promise<BigDataClusterDeploymentProfile[]> {
public async getDeploymentProfiles(deploymentType: BdcDeploymentType): Promise<BigDataClusterDeploymentProfile[]> {
let profilePrefix: string;
switch (deploymentType) {
case BdcDeploymentType.NewAKS:
case BdcDeploymentType.ExistingAKS:
profilePrefix = 'aks';
break;
case BdcDeploymentType.ExistingKubeAdm:
profilePrefix = 'kubeadm';
break;
default:
throw new Error(`Unknown deployment type: ${deploymentType}`);
}
await this.ensureWorkingDirectoryExists();
const profileNames = await this.getDeploymentProfileNames();
return await Promise.all(profileNames.map(profile => this.getDeploymentProfileInfo(profile)));
return await Promise.all(profileNames.filter(profile => profile.startsWith(profilePrefix)).map(profile => this.getDeploymentProfileInfo(profile)));
}
private async getDeploymentProfileNames(): Promise<string[]> {
@@ -29,17 +48,16 @@ export class AzdataService implements IAzdataService {
// azdata requires this environment variables to be set
env['ACCEPT_EULA'] = 'yes';
const cmd = 'azdata bdc config list -o json';
// Run the command twice to workaround the issue:
// First time use of the azdata will have extra EULA related string in the output
// there is no easy and reliable way to filter out the profile names from it.
await this.platformService.runCommand(cmd, { additionalEnvironmentVariables: env });
const stdout = await this.platformService.runCommand(cmd);
const stdout = await this.platformService.runCommand(cmd, { additionalEnvironmentVariables: env });
const output = <BdcConfigListOutput>JSON.parse(stdout);
return output.stdout;
return output.result;
}
private async getDeploymentProfileInfo(profileName: string): Promise<BigDataClusterDeploymentProfile> {
await this.platformService.runCommand(`azdata bdc config init --source ${profileName} --target ${profileName} --force`, { workingDirectory: this.platformService.storagePath() });
const env: NodeJS.ProcessEnv = {};
// azdata requires this environment variables to be set
env['ACCEPT_EULA'] = 'yes';
await this.platformService.runCommand(`azdata bdc config init --source ${profileName} --target ${profileName} --force`, { workingDirectory: this.platformService.storagePath(), additionalEnvironmentVariables: env });
const configObjects = await Promise.all([
this.getJsonObjectFromFile(path.join(this.platformService.storagePath(), profileName, 'bdc.json')),
this.getJsonObjectFromFile(path.join(this.platformService.storagePath(), profileName, 'control.json'))
@@ -56,4 +74,16 @@ export class AzdataService implements IAzdataService {
private async getJsonObjectFromFile(path: string): Promise<any> {
return JSON.parse(await this.platformService.readTextFile(path));
}
public async getEndpoints(clusterName: string, userName: string, password: string): Promise<BdcEndpoint[]> {
const env: NodeJS.ProcessEnv = {};
env['AZDATA_USERNAME'] = userName;
env['AZDATA_PASSWORD'] = password;
env['ACCEPT_EULA'] = 'yes';
let cmd = 'azdata login -n ' + clusterName;
await this.platformService.runCommand(cmd, { additionalEnvironmentVariables: env });
cmd = 'azdata bdc endpoint list';
const stdout = await this.platformService.runCommand(cmd, { additionalEnvironmentVariables: env });
return <BdcEndpoint[]>JSON.parse(stdout);
}
}

View File

@@ -2,7 +2,7 @@
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { AuthenticationMode } from '../ui/deployClusterWizard/deployClusterWizardModel';
export const SqlServerMasterResource = 'master';
export const DataResource = 'data-0';
export const HdfsResource = 'storage-0';
@@ -11,15 +11,26 @@ export const NameNodeResource = 'nmnode-0';
export const SparkHeadResource = 'sparkhead';
export const ZooKeeperResource = 'zookeeper';
export const SparkResource = 'spark-0';
export const HadrEnabledSetting = 'hadr.enabled';
interface ServiceEndpoint {
port: number;
serviceType: ServiceType;
name: EndpointName;
dnsName?: string;
}
type ServiceType = 'NodePort' | 'LoadBalancer';
type EndpointName = 'Controller' | 'Master' | 'Knox' | 'MasterSecondary';
type EndpointName = 'Controller' | 'Master' | 'Knox' | 'MasterSecondary' | 'AppServiceProxy' | 'ServiceProxy';
export interface ActiveDirectorySettings {
organizationalUnit: string;
domainControllerFQDNs: string;
dnsIPAddresses: string;
domainDNSName: string;
clusterUsers: string;
clusterAdmins: string;
appReaders?: string;
appOwners?: string;
}
export class BigDataClusterDeploymentProfile {
constructor(private _profileName: string, private _bdcConfig: any, private _controlConfig: any) {
@@ -39,6 +50,30 @@ export class BigDataClusterDeploymentProfile {
this._bdcConfig.metadata.name = value;
}
public get registry(): string {
return this._controlConfig.spec.docker.registry;
}
public set registry(value: string) {
this._controlConfig.spec.docker.registry = value;
}
public get repository(): string {
return this._controlConfig.spec.docker.repository;
}
public set repository(value: string) {
this._controlConfig.spec.docker.repository = value;
}
public get imageTag(): string {
return this._controlConfig.spec.docker.imageTag;
}
public set imageTag(value: string) {
this._controlConfig.spec.docker.imageTag = value;
}
public get bdcConfig(): any {
return this._bdcConfig;
}
@@ -107,15 +142,6 @@ export class BigDataClusterDeploymentProfile {
return this._bdcConfig.spec.resources[SparkResource] ? this.getReplicas(SparkResource) : 0;
}
public get hadrEnabled(): boolean {
const value = this._bdcConfig.spec.resources[SqlServerMasterResource].spec.settings.sql[HadrEnabledSetting];
return value === true || value === 'true';
}
public set hadrEnabled(value: boolean) {
this._bdcConfig.spec.resources[SqlServerMasterResource].spec.settings.sql[HadrEnabledSetting] = value;
}
public get includeSpark(): boolean {
return <boolean>this._bdcConfig.spec.resources[HdfsResource].spec.settings.spark.includeSpark;
}
@@ -175,32 +201,48 @@ export class BigDataClusterDeploymentProfile {
return this.getEndpointPort(this._controlConfig.spec.endpoints, 'Controller', 30080);
}
public set controllerPort(port: number) {
this.setEndpointPort(this._controlConfig.spec.endpoints, 'Controller', port);
public setControllerEndpoint(port: number, dnsName?: string) {
this.setEndpoint(this._controlConfig.spec.endpoints, 'Controller', port, dnsName);
}
public get serviceProxyPort(): number {
return this.getEndpointPort(this._controlConfig.spec.endpoints, 'ServiceProxy', 30080);
}
public setServiceProxyEndpoint(port: number, dnsName?: string) {
this.setEndpoint(this._controlConfig.spec.endpoints, 'ServiceProxy', port, dnsName);
}
public get appServiceProxyPort(): number {
return this.getEndpointPort(this._bdcConfig.spec.resources.appproxy.spec.endpoints, 'AppServiceProxy', 30777);
}
public setAppServiceProxyEndpoint(port: number, dnsName?: string) {
this.setEndpoint(this._bdcConfig.spec.resources.appproxy.spec.endpoints, 'AppServiceProxy', port, dnsName);
}
public get sqlServerPort(): number {
return this.getEndpointPort(this._bdcConfig.spec.resources.master.spec.endpoints, 'Master', 31433);
}
public set sqlServerPort(port: number) {
this.setEndpointPort(this._bdcConfig.spec.resources.master.spec.endpoints, 'Master', port);
public setSqlServerEndpoint(port: number, dnsName?: string) {
this.setEndpoint(this._bdcConfig.spec.resources.master.spec.endpoints, 'Master', port, dnsName);
}
public get sqlServerReadableSecondaryPort(): number {
return this.getEndpointPort(this._bdcConfig.spec.resources.master.spec.endpoints, 'MasterSecondary', 31436);
}
public set sqlServerReadableSecondaryPort(port: number) {
this.setEndpointPort(this._bdcConfig.spec.resources.master.spec.endpoints, 'MasterSecondary', port);
public setSqlServerReadableSecondaryEndpoint(port: number, dnsName?: string) {
this.setEndpoint(this._bdcConfig.spec.resources.master.spec.endpoints, 'MasterSecondary', port, dnsName);
}
public get gatewayPort(): number {
return this.getEndpointPort(this._bdcConfig.spec.resources.gateway.spec.endpoints, 'Knox', 30443);
}
public set gatewayPort(port: number) {
this.setEndpointPort(this._bdcConfig.spec.resources.gateway.spec.endpoints, 'Knox', port);
public setGatewayEndpoint(port: number, dnsName?: string) {
this.setEndpoint(this._bdcConfig.spec.resources.gateway.spec.endpoints, 'Knox', port, dnsName);
}
public addSparkResource(replicas: number): void {
@@ -220,8 +262,32 @@ export class BigDataClusterDeploymentProfile {
}
public get activeDirectorySupported(): boolean {
// TODO: Implement AD authentication
return false;
// The profiles that highlight the AD authentication feature will have a security secion in the control.json for the AD settings.
return 'security' in this._controlConfig;
}
public setAuthenticationMode(mode: string): void {
// If basic authentication is picked, the security section must be removed
// otherwise azdata will throw validation error
if (mode === AuthenticationMode.Basic && 'security' in this._controlConfig) {
delete this._controlConfig.security;
}
}
public setActiveDirectorySettings(adSettings: ActiveDirectorySettings): void {
this._controlConfig.security.ouDistinguishedName = adSettings.organizationalUnit;
this._controlConfig.security.dnsIpAddresses = this.splitByComma(adSettings.dnsIPAddresses);
this._controlConfig.security.domainControllerFullyQualifiedDns = this.splitByComma(adSettings.domainControllerFQDNs);
this._controlConfig.security.domainDnsName = adSettings.domainDNSName;
this._controlConfig.security.realm = adSettings.domainDNSName.toUpperCase();
this._controlConfig.security.clusterAdmins = this.splitByComma(adSettings.clusterAdmins);
this._controlConfig.security.clusterUsers = this.splitByComma(adSettings.clusterUsers);
if (adSettings.appReaders) {
this._controlConfig.security.appReaders = this.splitByComma(adSettings.appReaders);
}
if (adSettings.appOwners) {
this._controlConfig.security.appOwners = this.splitByComma(adSettings.appOwners);
}
}
public getBdcJson(readable: boolean = true): string {
@@ -249,16 +315,27 @@ export class BigDataClusterDeploymentProfile {
return endpoint ? endpoint.port : defaultValue;
}
private setEndpointPort(endpoints: ServiceEndpoint[], name: EndpointName, port: number): void {
private setEndpoint(endpoints: ServiceEndpoint[], name: EndpointName, port: number, dnsName?: string): void {
const endpoint = endpoints.find(endpoint => endpoint.name === name);
if (endpoint) {
endpoint.port = port;
endpoint.dnsName = dnsName;
} else {
endpoints.push({
const newEndpoint: ServiceEndpoint = {
name: name,
serviceType: 'NodePort',
port: port
});
};
// for newly added endpoint, we cannot have blank value for the dnsName, only set it if it is not empty
if (dnsName) {
newEndpoint.dnsName = dnsName;
}
endpoints.push(newEndpoint);
}
}
private splitByComma(value: string): string[] {
// split by comma, then remove trailing spaces for each item and finally remove the empty values.
return value.split(',').map(v => v && v.trim()).filter(v => v !== '' && v !== undefined);
}
}

View File

@@ -10,10 +10,32 @@ import * as vscode from 'vscode';
import * as nls from 'vscode-nls';
import { IPlatformService } from './platformService';
import { NotebookInfo } from '../interfaces';
import { getErrorMessage, getDateTimeString } from '../utils';
const localize = nls.loadMessageBundle();
export interface Notebook {
cells: NotebookCell[];
}
export interface NotebookCell {
cell_type: 'code';
source: string[];
metadata: {};
outputs: string[];
execution_count: number;
}
export interface NotebookExecutionResult {
succeeded: boolean;
outputNotebook?: string;
errorMessage?: string;
}
export interface INotebookService {
launchNotebook(notebook: string | NotebookInfo): Thenable<azdata.nb.NotebookEditor>;
launchNotebookWithContent(title: string, content: string): Thenable<azdata.nb.NotebookEditor>;
getNotebook(notebook: string | NotebookInfo): Promise<Notebook>;
executeNotebook(notebook: any, env: NodeJS.ProcessEnv): Promise<NotebookExecutionResult>;
}
export class NotebookService implements INotebookService {
@@ -21,32 +43,89 @@ export class NotebookService implements INotebookService {
constructor(private platformService: IPlatformService, private extensionPath: string) { }
/**
* Copy the notebook to the user's home directory and launch the notebook from there.
* Launch notebook with file path
* @param notebook the path of the notebook
*/
launchNotebook(notebook: string | NotebookInfo): Thenable<azdata.nb.NotebookEditor> {
const notebookPath = this.getNotebook(notebook);
const notebookFullPath = path.join(this.extensionPath, notebookPath);
return this.platformService.fileExists(notebookPath).then((notebookPathExists) => {
if (notebookPathExists) {
return this.showNotebookAsUntitled(notebookPath);
} else {
return this.platformService.fileExists(notebookFullPath).then(notebookFullPathExists => {
if (notebookFullPathExists) {
return this.showNotebookAsUntitled(notebookFullPath);
} else {
throw localize('resourceDeployment.notebookNotFound', "The notebook {0} does not exist", notebookPath);
}
});
}
return this.getNotebookFullPath(notebook).then(notebookPath => {
return this.showNotebookAsUntitled(notebookPath);
});
}
/**
* Launch notebook with file path
* @param title the title of the notebook
* @param content the notebook content
*/
launchNotebookWithContent(title: string, content: string): Thenable<azdata.nb.NotebookEditor> {
const uri: vscode.Uri = vscode.Uri.parse(`untitled:${title}`);
return azdata.nb.showNotebookDocument(uri, {
connectionProfile: undefined,
preview: false,
initialContent: content,
initialDirtyState: false
});
}
async getNotebook(notebook: string | NotebookInfo): Promise<Notebook> {
const notebookPath = await this.getNotebookFullPath(notebook);
return <Notebook>JSON.parse(await this.platformService.readTextFile(notebookPath));
}
async executeNotebook(notebook: Notebook, env: NodeJS.ProcessEnv): Promise<NotebookExecutionResult> {
const content = JSON.stringify(notebook, undefined, 4);
const fileName = `nb-${getDateTimeString()}.ipynb`;
const workingDirectory = this.platformService.storagePath();
const notebookFullPath = path.join(workingDirectory, fileName);
const outputFullPath = path.join(workingDirectory, `output-${fileName}`);
try {
await this.platformService.saveTextFile(content, notebookFullPath);
await this.platformService.runCommand(`azdata notebook run --path "${notebookFullPath}" --output-path "${workingDirectory}" --timeout -1`,
{
additionalEnvironmentVariables: env,
workingDirectory: workingDirectory
});
return {
succeeded: true
};
}
catch (error) {
const outputExists = await this.platformService.fileExists(outputFullPath);
return {
succeeded: false,
outputNotebook: outputExists ? await this.platformService.readTextFile(outputFullPath) : undefined,
errorMessage: getErrorMessage(error)
};
} finally {
this.platformService.deleteFile(notebookFullPath);
this.platformService.deleteFile(outputFullPath);
}
}
async getNotebookFullPath(notebook: string | NotebookInfo): Promise<string> {
const notebookPath = this.getNotebookPath(notebook);
let notebookExists = await this.platformService.fileExists(notebookPath);
if (notebookExists) {
// this is for the scenarios when the provider is in a different extension, the full path will be passed in.
return notebookPath;
}
// this is for the scenarios in this extension, the notebook paths are relative path.
const absolutePath = path.join(this.extensionPath, notebookPath);
notebookExists = await this.platformService.fileExists(absolutePath);
if (notebookExists) {
return absolutePath;
} else {
throw new Error(localize('resourceDeployment.notebookNotFound', "The notebook {0} does not exist", notebookPath));
}
}
/**
* get the notebook path for current platform
* @param notebook the notebook path
*/
getNotebook(notebook: string | NotebookInfo): string {
getNotebookPath(notebook: string | NotebookInfo): string {
let notebookPath;
if (notebook && !isString(notebook)) {
const platform = this.platformService.platform();

View File

@@ -7,6 +7,7 @@ import * as fs from 'fs';
import * as vscode from 'vscode';
import * as azdata from 'azdata';
import * as cp from 'child_process';
import { getErrorMessage } from '../utils';
/**
* Abstract of platform dependencies
@@ -22,6 +23,8 @@ export interface IPlatformService {
makeDirectory(path: string): Promise<void>;
readTextFile(filePath: string): Promise<string>;
runCommand(command: string, options?: CommandOptions): Promise<string>;
saveTextFile(content: string, path: string): Promise<void>;
deleteFile(path: string, ignoreError?: boolean): Promise<void>;
}
export interface CommandOptions {
@@ -91,4 +94,24 @@ export class PlatformService implements IPlatformService {
});
});
}
saveTextFile(content: string, path: string): Promise<void> {
return fs.promises.writeFile(path, content, 'utf8');
}
async deleteFile(path: string, ignoreError: boolean = true): Promise<void> {
try {
const exists = await this.fileExists(path);
if (exists) {
fs.promises.unlink(path);
}
}
catch (error) {
if (ignoreError) {
console.error('Error occured deleting file: ', getErrorMessage(error));
} else {
throw error;
}
}
}
}

View File

@@ -29,7 +29,7 @@ export class DockerTool extends ToolBase {
}
get displayName(): string {
return localize('resourceDeployment.DockerDisplayName', 'Docker');
return localize('resourceDeployment.DockerDisplayName', 'docker');
}
get homePage(): string {