ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 10:58:30 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix Local keys for spark so that dup key error doesnt occur (#7634)

Browse Source
This commit is contained in:
Udeesha Gautam
2019-10-18 18:06:37 -07:00
committed by GitHub
parent ab31a7b964
commit a2f105a913
7 changed files with 55 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
extensions/mssql/src/localizedConstants.ts
View File

@@ -37,16 +37,16 @@ export const applyRecursivelyText = localize('mssql.applyRecursively', "Apply Re
export function errorApplyingAclChanges(errMsg: string): string { return localize('mssql.errorApplyingAclChanges', "Unexpected error occurred while applying changes : {0}", errMsg); }
// Spark Job Submission Constants //////////////////////////////////////////
export const sparkLocalFileDestinationHint = localize('sparkJobSubmission_LocalFileDestinationHint', 'Local file will be uploaded to HDFS. ');
export const sparkJobSubmissionEndMessage = localize('sparkJobSubmission_SubmissionEndMessage', '.......................... Submit Spark Job End ............................');
export function sparkJobSubmissionPrepareUploadingFile(localPath: string, clusterFolder: string): string { return localize('sparkJobSubmission_PrepareUploadingFile', 'Uploading file from local {0} to HDFS folder: {1}', localPath, clusterFolder); }
export const sparkJobSubmissionUploadingFileSucceeded = localize('sparkJobSubmission_UploadingFileSucceeded', 'Upload file to cluster Succeeded!');
export function sparkJobSubmissionUploadingFileFailed(err: string): string { return localize('sparkJobSubmission_UploadingFileFailed', 'Upload file to cluster Failed. {0}', err); }
export function sparkJobSubmissionPrepareSubmitJob(jobName: string): string { return localize('sparkJobSubmission_PrepareSubmitJob', 'Submitting job {0} ... ', jobName); }
export const sparkJobSubmissionSparkJobHasBeenSubmitted = localize('sparkJobSubmission_SubmitJobFinished', 'The Spark Job has been submitted.');
export function sparkJobSubmissionSubmitJobFailed(err: string): string { return localize('sparkJobSubmission_SubmitJobFailed', 'Spark Job Submission Failed. {0} ', err); }
export function sparkJobSubmissionYarnUIMessage(yarnUIURL: string): string { return localize('sparkJobSubmission_YarnUIMessage', 'YarnUI Url: {0} ', yarnUIURL); }
export function sparkJobSubmissionSparkHistoryLinkMessage(sparkHistoryLink: string): string { return localize('sparkJobSubmission_SparkHistoryLinkMessage', 'Spark History Url: {0} ', sparkHistoryLink); }
export function sparkJobSubmissionGetApplicationIdFailed(err: string): string { return localize('sparkJobSubmission_GetApplicationIdFailed', 'Get Application Id Failed. {0}', err); }
export function sparkJobSubmissionLocalFileNotExisted(path: string): string { return localize('sparkJobSubmission_LocalFileNotExisted', 'Local file {0} does not existed. ', path); }
export const sparkJobSubmissionNoSqlBigDataClusterFound = localize('sparkJobSubmission_NoSqlBigDataClusterFound', 'No SQL Server Big Data Cluster found.');
export const sparkLocalFileDestinationHint = localize('sparkJobSubmission.LocalFileDestinationHint', 'Local file will be uploaded to HDFS. ');
export const sparkJobSubmissionEndMessage = localize('sparkJobSubmission.SubmissionEndMessage', '.......................... Submit Spark Job End ............................');
export function sparkJobSubmissionPrepareUploadingFile(localPath: string, clusterFolder: string): string { return localize('sparkJobSubmission.PrepareUploadingFile', 'Uploading file from local {0} to HDFS folder: {1}', localPath, clusterFolder); }
export const sparkJobSubmissionUploadingFileSucceeded = localize('sparkJobSubmission.UploadingFileSucceeded', 'Upload file to cluster Succeeded!');
export function sparkJobSubmissionUploadingFileFailed(err: string): string { return localize('sparkJobSubmission.UploadingFileFailed', 'Upload file to cluster Failed. {0}', err); }
export function sparkJobSubmissionPrepareSubmitJob(jobName: string): string { return localize('sparkJobSubmission.PrepareSubmitJob', 'Submitting job {0} ... ', jobName); }
export const sparkJobSubmissionSparkJobHasBeenSubmitted = localize('sparkJobSubmission.SubmitJobFinished', 'The Spark Job has been submitted.');
export function sparkJobSubmissionSubmitJobFailed(err: string): string { return localize('sparkJobSubmission.SubmitJobFailed', 'Spark Job Submission Failed. {0} ', err); }
export function sparkJobSubmissionYarnUIMessage(yarnUIURL: string): string { return localize('sparkJobSubmission.YarnUIMessage', 'YarnUI Url: {0} ', yarnUIURL); }
export function sparkJobSubmissionSparkHistoryLinkMessage(sparkHistoryLink: string): string { return localize('sparkJobSubmission.SparkHistoryLinkMessage', 'Spark History Url: {0} ', sparkHistoryLink); }
export function sparkJobSubmissionGetApplicationIdFailed(err: string): string { return localize('sparkJobSubmission.GetApplicationIdFailed', 'Get Application Id Failed. {0}', err); }
export function sparkJobSubmissionLocalFileNotExisted(path: string): string { return localize('sparkJobSubmission.LocalFileNotExisted', 'Local file {0} does not existed. ', path); }
export const sparkJobSubmissionNoSqlBigDataClusterFound = localize('sparkJobSubmission.NoSqlBigDataClusterFound', 'No SQL Server Big Data Cluster found.');

6
extensions/mssql/src/sparkFeature/dialog/dialogCommands.ts
View File

@@ -76,7 +76,7 @@ export class OpenSparkJobSubmissionDialogCommand extends Command {
selectedHost = await vscode.window.showQuickPick(displayList, {
placeHolder:
localize('sparkJobSubmission_PleaseSelectSqlWithCluster',
localize('sparkJobSubmission.PleaseSelectSqlWithCluster',
"Please select SQL Server with Big Data Cluster.")
});
if (selectedHost === selectConnectionMsg) {
@@ -99,7 +99,7 @@ export class OpenSparkJobSubmissionDialogCommand extends Command {
}
}
let errorMsg = localize('sparkJobSubmission_NoSqlSelected', 'No SQL Server is selected.');
let errorMsg = localize('sparkJobSubmission.NoSqlSelected', 'No SQL Server is selected.');
if (!selectedHost) { throw new Error(errorMsg); }
let sqlConnection = connectionMap.get(selectedHost);
@@ -135,7 +135,7 @@ export class OpenSparkJobSubmissionDialogFromFileCommand extends Command {
return;
}
} catch (err) {
this.apiWrapper.showErrorMessage(localize('sparkJobSubmission_GetFilePathFromSelectedNodeFailed', 'Error Get File Path: {0}', err));
this.apiWrapper.showErrorMessage(localize('sparkJobSubmission.GetFilePathFromSelectedNodeFailed', 'Error Get File Path: {0}', err));
return;
}

14
extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkAdvancedTab.ts
View File

@@ -23,7 +23,7 @@ export class SparkAdvancedTab {
}
constructor(private appContext: AppContext) {
this._tab = this.apiWrapper.createTab(localize('sparkJobSubmission_AdvancedTabName', 'ADVANCED'));
this._tab = this.apiWrapper.createTab(localize('sparkJobSubmission.AdvancedTabName', 'ADVANCED'));
this._tab.registerContent(async (modelView) => {
let builder = modelView.modelBuilder;
@@ -37,11 +37,11 @@ export class SparkAdvancedTab {
this._referenceJARFilesInputBox = builder.inputBox().component();
formContainer.addFormItem({
component: this._referenceJARFilesInputBox,
title: localize('sparkJobSubmission_ReferenceJarList', 'Reference Jars')
title: localize('sparkJobSubmission.ReferenceJarList', 'Reference Jars')
},
Object.assign(
{
info: localize('sparkJobSubmission_ReferenceJarListToolTip',
info: localize('sparkJobSubmission.ReferenceJarListToolTip',
'Jars to be placed in executor working directory. The Jar path needs to be an HDFS Path. Multiple paths should be split by semicolon (;)')
},
parentLayout));
@@ -49,11 +49,11 @@ export class SparkAdvancedTab {
this._referencePyFilesInputBox = builder.inputBox().component();
formContainer.addFormItem({
component: this._referencePyFilesInputBox,
title: localize('sparkJobSubmission_ReferencePyList', 'Reference py Files')
title: localize('sparkJobSubmission.ReferencePyList', 'Reference py Files')
},
Object.assign(
{
info: localize('sparkJobSubmission_ReferencePyListTooltip',
info: localize('sparkJobSubmission.ReferencePyListTooltip',
'Py Files to be placed in executor working directory. The file path needs to be an HDFS Path. Multiple paths should be split by semicolon(;)')
},
parentLayout));
@@ -61,10 +61,10 @@ export class SparkAdvancedTab {
this._referenceFilesInputBox = builder.inputBox().component();
formContainer.addFormItem({
component: this._referenceFilesInputBox,
title: localize('sparkJobSubmission_ReferenceFilesList', 'Reference Files')
title: localize('sparkJobSubmission.ReferenceFilesList', 'Reference Files')
},
Object.assign({
info: localize('sparkJobSubmission_ReferenceFilesListTooltip',
info: localize('sparkJobSubmission.ReferenceFilesListTooltip',
'Files to be placed in executor working directory. The file path needs to be an HDFS Path. Multiple paths should be split by semicolon(;)')
}, parentLayout));

34
extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkConfigurationTab.ts
View File

@@ -41,7 +41,7 @@ export class SparkConfigurationTab {
// If path is specified, means the default source setting for this tab is HDFS file, otherwise, it would be local file.
constructor(private _dataModel: SparkJobSubmissionModel, private appContext: AppContext, private _path?: string) {
this._tab = this.apiWrapper.createTab(localize('sparkJobSubmission_GeneralTabName', 'GENERAL'));
this._tab = this.apiWrapper.createTab(localize('sparkJobSubmission.GeneralTabName', 'GENERAL'));
this._tab.registerContent(async (modelView) => {
let builder = modelView.modelBuilder;
@@ -53,13 +53,13 @@ export class SparkConfigurationTab {
let formContainer = builder.formContainer();
this._jobNameInputBox = builder.inputBox().withProperties({
placeHolder: localize('sparkJobSubmission_JobNamePlaceHolder', 'Enter a name ...'),
placeHolder: localize('sparkJobSubmission.JobNamePlaceHolder', 'Enter a name ...'),
value: (this._path) ? fspath.basename(this._path) : ''
}).component();
formContainer.addFormItem({
component: this._jobNameInputBox,
title: localize('sparkJobSubmission_JobName', 'Job Name'),
title: localize('sparkJobSubmission.JobName', 'Job Name'),
required: true
}, parentLayout);
@@ -68,7 +68,7 @@ export class SparkConfigurationTab {
}).component();
formContainer.addFormItem({
component: this._sparkContextLabel,
title: localize('sparkJobSubmission_SparkCluster', 'Spark Cluster')
title: localize('sparkJobSubmission.SparkCluster', 'Spark Cluster')
}, parentLayout);
this._fileSourceDropDown = builder.dropDown().withProperties<azdata.DropDownProperties>({
@@ -102,7 +102,7 @@ export class SparkConfigurationTab {
this._sparkSourceFileInputBox = builder.inputBox().withProperties({
required: true,
placeHolder: localize('sparkJobSubmission_FilePathPlaceHolder', 'Path to a .jar or .py file'),
placeHolder: localize('sparkJobSubmission.FilePathPlaceHolder', 'Path to a .jar or .py file'),
value: (this._path) ? this._path : ''
}).component();
this._sparkSourceFileInputBox.onTextChanged(text => {
@@ -110,7 +110,7 @@ export class SparkConfigurationTab {
this._dataModel.updateModelByLocalPath(text);
if (this._localUploadDestinationLabel) {
if (text) {
this._localUploadDestinationLabel.value = localize('sparkJobSubmission_LocalFileDestinationHintWithPath',
this._localUploadDestinationLabel.value = localize('sparkJobSubmission.LocalFileDestinationHintWithPath',
'The selected local file will be uploaded to HDFS: {0}', this._dataModel.hdfsSubmitFilePath);
} else {
this._localUploadDestinationLabel.value = LocalizedConstants.sparkLocalFileDestinationHint;
@@ -167,24 +167,24 @@ export class SparkConfigurationTab {
formContainer.addFormItem({
component: this._sourceFlexContainerWithHint,
title: localize('sparkJobSubmission_MainFilePath', 'JAR/py File'),
title: localize('sparkJobSubmission.MainFilePath', 'JAR/py File'),
required: true
}, parentLayout);
this._mainClassInputBox = builder.inputBox().component();
formContainer.addFormItem({
component: this._mainClassInputBox,
title: localize('sparkJobSubmission_MainClass', 'Main Class'),
title: localize('sparkJobSubmission.MainClass', 'Main Class'),
required: true
}, parentLayout);
this._argumentsInputBox = builder.inputBox().component();
formContainer.addFormItem({
component: this._argumentsInputBox,
title: localize('sparkJobSubmission_Arguments', 'Arguments')
title: localize('sparkJobSubmission.Arguments', 'Arguments')
},
Object.assign(
{ info: localize('sparkJobSubmission_ArgumentsTooltip', 'Command line arguments used in your main class, multiple arguments should be split by space.') },
{ info: localize('sparkJobSubmission.ArgumentsTooltip', 'Command line arguments used in your main class, multiple arguments should be split by space.') },
parentLayout));
await modelView.initializeModel(formContainer.component());
@@ -193,7 +193,7 @@ export class SparkConfigurationTab {
public async validate(): Promise<boolean> {
if (!this._jobNameInputBox.value) {
this._dataModel.showDialogError(localize('sparkJobSubmission_NotSpecifyJobName', 'Property Job Name is not specified.'));
this._dataModel.showDialogError(localize('sparkJobSubmission.NotSpecifyJobName', 'Property Job Name is not specified.'));
return false;
}
@@ -202,7 +202,7 @@ export class SparkConfigurationTab {
this._dataModel.isMainSourceFromLocal = true;
this._dataModel.updateModelByLocalPath(this._sparkSourceFileInputBox.value);
} else {
this._dataModel.showDialogError(localize('sparkJobSubmission_NotSpecifyJARPYPath', 'Property JAR/py File is not specified.'));
this._dataModel.showDialogError(localize('sparkJobSubmission.NotSpecifyJARPYPath', 'Property JAR/py File is not specified.'));
return false;
}
} else {
@@ -210,13 +210,13 @@ export class SparkConfigurationTab {
this._dataModel.isMainSourceFromLocal = false;
this._dataModel.hdfsSubmitFilePath = this._sparkSourceFileInputBox.value;
} else {
this._dataModel.showDialogError(localize('sparkJobSubmission_NotSpecifyJARPYPath', 'Property JAR/py File is not specified.'));
this._dataModel.showDialogError(localize('sparkJobSubmission.NotSpecifyJARPYPath', 'Property JAR/py File is not specified.'));
return false;
}
}
if (this._dataModel.isJarFile() && !this._mainClassInputBox.value) {
this._dataModel.showDialogError(localize('sparkJobSubmission_NotSpecifyMainClass', 'Property Main Class is not specified.'));
this._dataModel.showDialogError(localize('sparkJobSubmission.NotSpecifyMainClass', 'Property Main Class is not specified.'));
return false;
}
@@ -231,11 +231,11 @@ export class SparkConfigurationTab {
try {
let isFileExisted = await this._dataModel.isClusterFileExisted(this._dataModel.hdfsSubmitFilePath);
if (!isFileExisted) {
this._dataModel.showDialogError(localize('sparkJobSubmission_HDFSFileNotExistedWithPath', '{0} does not exist in Cluster or exception thrown. ', this._dataModel.hdfsSubmitFilePath));
this._dataModel.showDialogError(localize('sparkJobSubmission.HDFSFileNotExistedWithPath', '{0} does not exist in Cluster or exception thrown. ', this._dataModel.hdfsSubmitFilePath));
return false;
}
} catch (error) {
this._dataModel.showDialogError(localize('sparkJobSubmission_HDFSFileNotExisted', 'The specified HDFS file does not exist. '));
this._dataModel.showDialogError(localize('sparkJobSubmission.HDFSFileNotExisted', 'The specified HDFS file does not exist. '));
return false;
}
}
@@ -272,7 +272,7 @@ export class SparkConfigurationTab {
return undefined;
} catch (err) {
this.apiWrapper.showErrorMessage(localize('sparkJobSubmission_SelectFileError', 'Error in locating the file due to Error: {0}', utils.getErrorMessage(err)));
this.apiWrapper.showErrorMessage(localize('sparkJobSubmission.SelectFileError', 'Error in locating the file due to Error: {0}', utils.getErrorMessage(err)));
return undefined;
}
}

12
extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkJobSubmissionDialog.ts
View File

@@ -35,13 +35,13 @@ export class SparkJobSubmissionDialog {
private appContext: AppContext,
private outputChannel: vscode.OutputChannel) {
if (!this.sqlClusterConnection || !this.appContext || !this.outputChannel) {
throw new Error(localize('sparkJobSubmission_SparkJobSubmissionDialogInitializeError',
throw new Error(localize('sparkJobSubmission.SparkJobSubmissionDialogInitializeError',
'Parameters for SparkJobSubmissionDialog is illegal'));
}
}
public async openDialog(path?: string): Promise<void> {
this._dialog = this.apiWrapper.createDialog(localize('sparkJobSubmission_DialogTitleNewJob', 'New Job'));
this._dialog = this.apiWrapper.createDialog(localize('sparkJobSubmission.DialogTitleNewJob', 'New Job'));
this._dataModel = new SparkJobSubmissionModel(this.sqlClusterConnection, this._dialog, this.appContext);
@@ -50,9 +50,9 @@ export class SparkJobSubmissionDialog {
this._dialog.content = [this._sparkConfigTab.tab, this._sparkAdvancedTab.tab];
this._dialog.cancelButton.label = localize('sparkJobSubmission_DialogCancelButton', 'Cancel');
this._dialog.cancelButton.label = localize('sparkJobSubmission.DialogCancelButton', 'Cancel');
this._dialog.okButton.label = localize('sparkJobSubmission_DialogSubmitButton', 'Submit');
this._dialog.okButton.label = localize('sparkJobSubmission.DialogSubmitButton', 'Submit');
this._dialog.okButton.onClick(() => this.onClickOk());
this._dialog.registerCloseValidator(() => this.handleValidate());
@@ -61,7 +61,7 @@ export class SparkJobSubmissionDialog {
}
private onClickOk(): void {
let jobName = localize('sparkJobSubmission_SubmitSparkJob', '{0} Spark Job Submission:',
let jobName = localize('sparkJobSubmission.SubmitSparkJob', '{0} Spark Job Submission:',
this._sparkConfigTab.getInputValues()[0]);
this.apiWrapper.startBackgroundOperation(
{
@@ -79,7 +79,7 @@ export class SparkJobSubmissionDialog {
private async onSubmit(op: azdata.BackgroundOperation): Promise<void> {
try {
this.outputChannel.show();
let msg = localize('sparkJobSubmission_SubmissionStartMessage',
let msg = localize('sparkJobSubmission.SubmissionStartMessage',
'.......................... Submit Spark Job Start ..........................');
this.outputChannel.appendLine(msg);
// 1. Upload local file to HDFS for local source.

14
extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkJobSubmissionModel.ts
View File

@@ -41,7 +41,7 @@ export class SparkJobSubmissionModel {
requestService?: (args: any) => any) {
if (!this._sqlClusterConnection || !this._dialog || !this._appContext) {
throw new Error(localize('sparkJobSubmission_SparkJobSubmissionModelInitializeError',
throw new Error(localize('sparkJobSubmission.SparkJobSubmissionModelInitializeError',
'Parameters for SparkJobSubmissionModel is illegal'));
}
@@ -89,7 +89,7 @@ export class SparkJobSubmissionModel {
public async submitBatchJobByLivy(submissionArgs: SparkJobSubmissionInput): Promise<string> {
try {
if (!submissionArgs) {
return Promise.reject(localize('sparkJobSubmission_submissionArgsIsInvalid', 'submissionArgs is invalid. '));
return Promise.reject(localize('sparkJobSubmission.submissionArgsIsInvalid', 'submissionArgs is invalid. '));
}
submissionArgs.setSparkClusterInfo(this._sqlClusterConnection);
@@ -104,11 +104,11 @@ export class SparkJobSubmissionModel {
// TODO: whether set timeout as 15000ms
try {
if (!submissionArgs) {
return Promise.reject(localize('sparkJobSubmission_submissionArgsIsInvalid', 'submissionArgs is invalid. '));
return Promise.reject(localize('sparkJobSubmission.submissionArgsIsInvalid', 'submissionArgs is invalid. '));
}
if (!utils.isValidNumber(livyBatchId)) {
return Promise.reject(new Error(localize('sparkJobSubmission_LivyBatchIdIsInvalid', 'livyBatchId is invalid. ')));
return Promise.reject(new Error(localize('sparkJobSubmission.LivyBatchIdIsInvalid', 'livyBatchId is invalid. ')));
}
if (!retryTime) {
@@ -125,7 +125,7 @@ export class SparkJobSubmissionModel {
} while (response.appId === '' && timeOutCount < retryTime);
if (response.appId === '') {
return Promise.reject(localize('sparkJobSubmission_GetApplicationIdTimeOut', 'Get Application Id time out. {0}[Log] {1}', os.EOL, response.log));
return Promise.reject(localize('sparkJobSubmission.GetApplicationIdTimeOut', 'Get Application Id time out. {0}[Log] {1}', os.EOL, response.log));
} else {
return response.appId;
}
@@ -137,7 +137,7 @@ export class SparkJobSubmissionModel {
public async uploadFile(localFilePath: string, hdfsFolderPath: string): Promise<void> {
try {
if (!localFilePath || !hdfsFolderPath) {
return Promise.reject(localize('sparkJobSubmission_localFileOrFolderNotSpecified.', 'Property localFilePath or hdfsFolderPath is not specified. '));
return Promise.reject(localize('sparkJobSubmission.localFileOrFolderNotSpecified.', 'Property localFilePath or hdfsFolderPath is not specified. '));
}
if (!(await utils.exists(localFilePath))) {
@@ -154,7 +154,7 @@ export class SparkJobSubmissionModel {
public async isClusterFileExisted(path: string): Promise<boolean> {
try {
if (!path) {
return Promise.reject(localize('sparkJobSubmission_PathNotSpecified.', 'Property Path is not specified. '));
return Promise.reject(localize('sparkJobSubmission.PathNotSpecified.', 'Property Path is not specified. '));
}
let fileSource: IFileSource = await this._sqlClusterConnection.createHdfsFileSource();

4
extensions/mssql/src/sparkFeature/dialog/sparkJobSubmission/sparkJobSubmissionService.ts
View File

@@ -86,7 +86,7 @@ export class SparkJobSubmissionService {
return response.id;
}
return Promise.reject(new Error(localize('sparkJobSubmission_LivyNoBatchIdReturned',
return Promise.reject(new Error(localize('sparkJobSubmission.LivyNoBatchIdReturned',
'No Spark job batch id is returned from response.{0}[Error] {1}', os.EOL, JSON.stringify(response))));
} catch (error) {
return Promise.reject(error);
@@ -124,7 +124,7 @@ export class SparkJobSubmissionService {
return this.extractYarnAppIdFromLog(response.log);
}
return Promise.reject(localize('sparkJobSubmission_LivyNoLogReturned',
return Promise.reject(localize('sparkJobSubmission.LivyNoLogReturned',
'No log is returned within response.{0}[Error] {1}', os.EOL, JSON.stringify(response)));
} catch (error) {
return Promise.reject(error);