From d3e163a1d7b6b2cdbbe226fa3267b842ce11058f Mon Sep 17 00:00:00 2001 From: bnhoule <52506119+bnhoule@users.noreply.github.com> Date: Tue, 21 Sep 2021 17:11:00 -0500 Subject: [PATCH] Adding Derived Columns to ADS Flatfile Import (#16795) * Adding derived column boilerplate * brandan preliminary frontend changes * empty commit * added new param * updating contracts, dialogue changes * utils changes, saving timeout attempt * pushing for aasim * Cleaning up code and fixing the issue in theory * changing button, did not solve independent scroll * Fixing the scroll bar issue * updating flat file service * adding override keyword to overrriden method * improving UI * pushing changes associated with resolved comments * localizing strings, editing comments * all comments resolved * Fixing a test * updating import package Updating azure MFA bug * Clearing navigation validator Fixing broken table name change * fixed prose test * removing unused code from tests * Fixed PR comments * Fixing some PR comments * WIP * Fixing transformation code and create derived column dialog styling * removing unused code * Adding comment for console log * fixed table styling * Adding some aria labels * Fixed some code cleanup issues * update import service Co-authored-by: Aasim Khan Co-authored-by: bnhoule --- extensions/import/config.json | 2 +- extensions/import/package.json | 2 +- extensions/import/src/common/constants.ts | 19 +- .../import/src/dialogs/derivedColumnDialog.ts | 368 ++++++++++++++++++ extensions/import/src/services/contracts.ts | 42 ++ extensions/import/src/services/features.ts | 12 +- extensions/import/src/test/utils.test.ts | 12 +- .../wizard/pages/prosePreviewPage.test.ts | 4 +- extensions/import/src/wizard/api/basePage.ts | 2 +- extensions/import/src/wizard/api/models.ts | 2 + .../import/src/wizard/flatFileWizard.ts | 5 +- .../import/src/wizard/pages/fileConfigPage.ts | 101 +++-- .../src/wizard/pages/modifyColumnsPage.ts | 22 +- .../src/wizard/pages/prosePreviewPage.ts | 77 ++-- .../import/src/wizard/pages/summaryPage.ts | 7 +- 15 files changed, 569 insertions(+), 108 deletions(-) create mode 100644 extensions/import/src/dialogs/derivedColumnDialog.ts diff --git a/extensions/import/config.json b/extensions/import/config.json index 1e52232bf2..bd0d8ffa79 100644 --- a/extensions/import/config.json +++ b/extensions/import/config.json @@ -1,7 +1,7 @@ { "downloadUrl": "https://sqlopsextensions.blob.core.windows.net/extensions/import/service/{#version#}/{#fileName#}", "useDefaultLinuxRuntime": true, - "version": "0.0.8", + "version": "0.0.9", "downloadFileNames": { "Windows_64": "win-x64.zip", "Windows_86": "win-x86.zip", diff --git a/extensions/import/package.json b/extensions/import/package.json index e28c57d765..6e4bcc3f6e 100644 --- a/extensions/import/package.json +++ b/extensions/import/package.json @@ -2,7 +2,7 @@ "name": "import", "displayName": "SQL Server Import", "description": "SQL Server Import for Azure Data Studio supports importing CSV or JSON files into SQL Server.", - "version": "1.4.1", + "version": "1.5.0", "publisher": "Microsoft", "preview": true, "engines": { diff --git a/extensions/import/src/common/constants.ts b/extensions/import/src/common/constants.ts index 925c2f16cb..d9c26c5366 100644 --- a/extensions/import/src/common/constants.ts +++ b/extensions/import/src/common/constants.ts @@ -58,6 +58,23 @@ export const page2NameText = localize('flatFileImport.page2Name', "Preview Data" export const page3NameText = localize('flatFileImport.page3Name', "Modify Columns"); export const page4NameText = localize('flatFileImport.page4Name', "Summary"); export const importNewFileText = localize('flatFileImport.importNewFile', "Import new file"); - +export const createDerivedColumn = localize('flatFileImport.createDerivedColumn', "Create derived column"); +export const specifyDerivedColNameTitle = localize('flatFileImport.specifyDerivedColNameTitle', "Column Name"); +export const specifyTransformation = localize('flatFileImport.specifyTransformation', "Specify Transformation"); +export const previewTransformation = localize('flatFileImport.previewTransformation', "Preview Transformation"); +export const columnTableTitle = localize('flatFileImport.columnTableTitle', "Column"); +export const headerIntructionText = localize('flatFileImport.headerIntructionText', "Welcome to the Derived Column Tool! To get started, please follow the steps below:"); +export const deriverColumnInstruction1 = localize('flatFileImport.deriverColumnInstruction1', "Select the columns of data on the left required to derive your new column"); +export const deriverColumnInstruction2 = localize('flatFileImport.deriverColumnInstruction2', "Select a row and specify an example transformation that you would like applied to the rest of the column"); +export const deriverColumnInstruction3 = localize('flatFileImport.deriverColumnInstruction3', "Click \"Preview Transformation\" to preview the transformation"); +export const deriverColumnInstruction4 = localize('flatFileImport.deriverColumnInstruction4', "Refine your transformation until you have the desired column"); +export const deriverColumnInstruction5 = localize('flatFileImport.deriverColumnInstruction5', "Specify the new derived column\'s name and click \"Done\""); +export const selectAllColumns = localize('flatFileImport.selectAllColumns', "Select all columns"); +export function specifyTransformationForRow(rowIndex: number): string { + return localize('flatFileImport.specifyTransformationForRow', "Specify transformation for row {0}", rowIndex); +} +export function selectColumn(colName: string): string { + return localize('flatFileImport.selectColumn', "Select column {0}", colName); +} // SQL Queries export const selectSchemaQuery = `SELECT name FROM sys.schemas`; diff --git a/extensions/import/src/dialogs/derivedColumnDialog.ts b/extensions/import/src/dialogs/derivedColumnDialog.ts new file mode 100644 index 0000000000..57675b9506 --- /dev/null +++ b/extensions/import/src/dialogs/derivedColumnDialog.ts @@ -0,0 +1,368 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as azdata from 'azdata'; +import { ImportDataModel } from '../wizard/api/models'; +import * as EventEmitter from 'events'; +import { FlatFileProvider } from '../services/contracts'; +import * as constants from '../common/constants'; + +const headerLeft: azdata.CssStyles = { + 'border': 'none', + 'text-align': 'left', + 'white-space': 'nowrap', + 'text-overflow': 'ellipsis', + 'overflow': 'hidden', + 'border-bottom': '2px solid' +}; + +const styleLeft: azdata.CssStyles = { + 'border': 'none', + 'text-align': 'left', + 'white-space': 'nowrap', + 'text-overflow': 'ellipsis', + 'overflow': 'hidden', +}; + +export class DerivedColumnDialog { + private _dialogObject: azdata.window.Dialog; + private _doneEmitter: EventEmitter = new EventEmitter(); + private _currentTransformation: string[] = []; + private _currentDerivedColumnName!: string; + private _view!: azdata.ModelView; + private _specifyTransformations: azdata.InputBoxComponent[] = []; + private _headerInstructionText: azdata.TextComponent; + private _bodyInstructionText: azdata.TextComponent; + + private _applyButton!: azdata.window.Button; + private _transformationTable!: azdata.DeclarativeTableComponent; + private _transformationContainer!: azdata.FlexContainer; + private _specifyDerivedColumnNameContainer!: azdata.FlexContainer; + + constructor(private _model: ImportDataModel, private _provider: FlatFileProvider) { + } + + public createDerivedColumn(): Promise { + this._applyButton = azdata.window.createButton(constants.previewTransformation); + this._applyButton.enabled = false; + this._dialogObject = azdata.window.createModelViewDialog( + constants.createDerivedColumn, + 'DerivedColumnDialog', + 'wide' + ); + this._dialogObject.customButtons = [this._applyButton]; + this._applyButton.hidden = false; + + let tab = azdata.window.createTab(''); + tab.registerContent(async (view: azdata.ModelView) => { + const columnTableData: azdata.DeclarativeTableCellValue[][] = []; + this._model.originalProseColumns.forEach(c => { + const tableRow: azdata.DeclarativeTableCellValue[] = []; + tableRow.push( + { value: false, ariaLabel: constants.selectColumn(c.columnName) }, + { value: c.columnName } + ); + columnTableData.push(tableRow); + }); + this._view = view; + const columnTable = view.modelBuilder.declarativeTable().withProps({ + columns: [ + { + displayName: '', + ariaLabel: constants.selectAllColumns, + valueType: azdata.DeclarativeDataType.boolean, + isReadOnly: false, + showCheckAll: true, + width: '20px', + headerCssStyles: headerLeft, + rowCssStyles: styleLeft, + }, + { + displayName: constants.columnTableTitle, + valueType: azdata.DeclarativeDataType.string, + isReadOnly: true, + width: '140px', + headerCssStyles: headerLeft, + rowCssStyles: styleLeft, + ariaLabel: constants.columnTableTitle + } + ], + dataValues: columnTableData, + CSSStyles: { + 'table-layout': 'fixed' + } + }).component(); + + + columnTable.onDataChanged(e => { + if (e.value) { + // Adding newly selected column to transformation table + this._transformationTable.columns.push({ + displayName: this._model.proseColumns[e.row].columnName, + valueType: azdata.DeclarativeDataType.string, + isReadOnly: true, + width: '100px', + headerCssStyles: headerLeft, + rowCssStyles: styleLeft + }); + + this._model.proseDataPreview.forEach((v, i) => { + this._transformationTable.dataValues[i].push({ value: v[e.row] }); + }); + } + else { + // Removing unselected column from transformation table + let removeIndex = this._transformationTable.columns.findIndex(v => this._model.proseColumns[e.row].columnName === v.displayName); + this._transformationTable.columns.splice(removeIndex, 1); + for (let index = 0; index < this._model.proseDataPreview.length; index++) { + this._transformationTable.dataValues[index].splice(removeIndex, 1); + } + } + const isColumnAdded = this._transformationTable.columns.length > 1; + this.clearAndAddTransformationContainerComponents(isColumnAdded); + this._applyButton.enabled = isColumnAdded; + }); + + + const columnContainer = view.modelBuilder.flexContainer().withLayout({ + flexFlow: 'column', + height: '100%' + }).withProps({ + CSSStyles: { + 'border-right': 'solid 1px' + } + }).component(); + columnContainer.addItem(columnTable, { flex: '1 1 auto', CSSStyles: { 'overflow-y': 'hidden' } }); + + const transformationTableData: azdata.DeclarativeTableCellValue[][] = []; + for (let index = 0; index < this._model.proseDataPreview.length; index++) { + this._specifyTransformations.push(this._view.modelBuilder.inputBox().withProps({ + value: '', + placeHolder: constants.specifyTransformation, + ariaLabel: constants.specifyTransformationForRow(index) + }).component()); + transformationTableData.push([{ + value: this._specifyTransformations[index] + }]); + } + + this._transformationTable = view.modelBuilder.declarativeTable().withProps({ + columns: [ + { + displayName: constants.specifyTransformation, + ariaLabel: constants.specifyTransformation, + valueType: azdata.DeclarativeDataType.component, + isReadOnly: false, + width: '200px', + headerCssStyles: headerLeft, + rowCssStyles: styleLeft + } + ], + CSSStyles: { + 'table-layout': 'fixed', + 'overflow': 'scroll', + }, + width: '800px', + dataValues: transformationTableData + }).component(); + + this._applyButton.onClick(async e => { + const requiredColNames = this._transformationTable.columns.map(v => v.displayName); + requiredColNames.splice(0, 1); // Removing specify transformation column + const transExamples: string[] = []; + const transExampleIndices: number[] = []; + + // Getting all the example transformations specified by the user + this._transformationTable.dataValues.forEach((v, index) => { + const example = (v[0].value).value as string; + if (example !== '') { + transExamples.push(example); + transExampleIndices.push(index); + } + }); + + if (transExamples.length > 0) { + try { + const response = await this._provider.sendLearnTransformationRequest({ + columnNames: requiredColNames, + transformationExamples: transExamples, + transformationExampleRowIndices: transExampleIndices + }); + this._currentTransformation = response.transformationPreview; + this._currentTransformation.forEach((v, i) => { + (this._transformationTable.dataValues[i][0].value).placeHolder = v; + }); + this.clearAndAddTransformationContainerComponents(true); + } catch (e) { + this._dialogObject.message = { + text: e.toString(), + level: azdata.window.MessageLevel.Error + }; + } + } + this.validatePage(); + }); + + const columnNameText = view.modelBuilder.text().withProps({ + value: constants.specifyDerivedColNameTitle, + requiredIndicator: true, + CSSStyles: { + 'font-size': '13px', + 'font-weight': 'bold' + } + }).component(); + + const columnNameInput = view.modelBuilder.inputBox().withProps({ + ariaLabel: constants.specifyDerivedColNameTitle, + required: true + }).component(); + + columnNameInput.onTextChanged(e => { + if (e) { + this._currentDerivedColumnName = e; + this.validatePage(); + } + }); + + this._specifyDerivedColumnNameContainer = view.modelBuilder.flexContainer().withItems([ + columnNameText, + columnNameInput + ]).withLayout({ + width: '500px' + }).component(); + + this._transformationContainer = view.modelBuilder.flexContainer().withLayout({ + flexFlow: 'column', + height: '100%', + }).withProps({ + CSSStyles: { + 'overflow-y': 'auto', + 'margin-left': '10px', + } + }).component(); + + this._headerInstructionText = this._view.modelBuilder.text() + .withProps({ + value: constants.headerIntructionText, + CSSStyles: { + 'font-size': 'x-large', + 'line-height': '22pt', + 'margin-bottom': '0.7em' + } + }).component(); + + this._bodyInstructionText = this._view.modelBuilder.text() + .withProps({ + value: [ + constants.deriverColumnInstruction1, + constants.deriverColumnInstruction2, + constants.deriverColumnInstruction3, + constants.deriverColumnInstruction4, + constants.deriverColumnInstruction5, + ], + textType: azdata.TextType.OrderedList, + CSSStyles: { + 'font-size': 'large', + 'line-height': '22pt', + 'margin-left': '1em', + 'margin-top': '0em' + } + }).component(); + + this.clearAndAddTransformationContainerComponents(false); + + const flexGrid = view.modelBuilder.flexContainer().withLayout({ + flexFlow: 'row', + height: '100%', + width: '100%' + }).component(); + + /** + * Setting height of the div based on the total viewport height after removing dialog + * header and footer heights. With this the div will occupy the entire page space of the dialog. + */ + flexGrid.addItem(columnContainer, { + flex: '0 0 auto', + CSSStyles: { + 'min-height': 'calc(100vh - 160px)' + } + }); + flexGrid.addItem(this._transformationContainer, { + flex: '0 0 auto', + CSSStyles: { + 'overflow': 'scroll', + 'padding-right': '10px', + 'width': '900px', + 'max-height': 'calc(100vh - 160px)' + } + }); + + const formBuilder = view.modelBuilder.formContainer().withFormItems( + [ + { + component: flexGrid + } + ], + { + horizontal: false + } + ); + const form = formBuilder.withLayout({ width: '100%' }).component(); + return view.initializeModel(form); + }); + + this._dialogObject.okButton.onClick(e => { + this._doneEmitter.emit('done'); + }); + + this._dialogObject.cancelButton.onClick(e => { + this._doneEmitter.emit('close'); + }); + + this._dialogObject.content = [tab]; + azdata.window.openDialog(this._dialogObject); + return new Promise((resolve) => { + this._doneEmitter.once('done', async () => { + try { + await this._provider.sendSaveTransformationRequest({ + derivedColumnName: this._currentDerivedColumnName + }); + resolve({ + derivedColumnName: this._currentDerivedColumnName, + derivedColumnDataPreview: this._currentTransformation + }); + } catch (e) { + console.log(e); // Need to have better error handling for saved transformation.However this seems to be mostly a non-issue. + + resolve(undefined); + } + }); + + this._doneEmitter.once('close', async () => { + resolve(undefined); + }); + }); + } + + private clearAndAddTransformationContainerComponents(addTable: boolean): void { + this._transformationContainer.clearItems(); + if (addTable) { + this._transformationContainer.addItem(this._specifyDerivedColumnNameContainer, { flex: '0 0 auto' }); + this._transformationContainer.addItem(this._transformationTable, { flex: '1 1 auto', CSSStyles: { 'overflow': 'scroll' } }); + } + else { + this._transformationContainer.addItem(this._headerInstructionText, { flex: '0 0 auto' }); + this._transformationContainer.addItem(this._bodyInstructionText, { flex: '0 0 auto' }); + } + } + + private validatePage(): void { + this._dialogObject.okButton.enabled = this._currentDerivedColumnName !== undefined && this._currentTransformation.length !== 0; + } +} + +export interface DerivedColumnDialogResult { + derivedColumnName?: string; + derivedColumnDataPreview?: string[]; +} diff --git a/extensions/import/src/services/contracts.ts b/extensions/import/src/services/contracts.ts index e4ac76a1a6..b53f78d498 100644 --- a/extensions/import/src/services/contracts.ts +++ b/extensions/import/src/services/contracts.ts @@ -52,6 +52,38 @@ export interface ColumnInfo { } +/** + * LearnTransformationRequest + * Send this request to learn a transformation and preview it + */ +const learnTransformationRequestName = 'flatfile/learnTransformation'; + +export interface LearnTransformationParams { + columnNames: string[]; + transformationExamples: string[]; + transformationExampleRowIndices: number[]; +} + +export interface LearnTransformationResponse { + transformationPreview: string[]; +} + + +/** +* SaveTransformationRequest +* Send this request to save a transformation to be applied on insertion into database +*/ +const saveTransformationRequestName = 'flatfile/saveTransformation'; + +export interface SaveTransformationParams { + derivedColumnName: string; +} + +export interface SaveTransformationResponse { + numTransformations: number; +} + + /** * PROSEDiscoveryRequest * Send this request to create a new PROSE session with a new file and preview it @@ -139,6 +171,14 @@ export namespace ChangeColumnSettingsRequest { export const type = new RequestType(changeColumnSettingsRequestName); } +export namespace LearnTransformationRequest { + export const type = new RequestType(learnTransformationRequestName); +} + +export namespace SaveTransformationRequest { + export const type = new RequestType(saveTransformationRequestName); +} + export interface FlatFileProvider { providerId?: string; @@ -147,4 +187,6 @@ export interface FlatFileProvider { sendInsertDataRequest(params: InsertDataParams): Thenable; sendGetColumnInfoRequest(params: GetColumnInfoParams): Thenable; sendChangeColumnSettingsRequest(params: ChangeColumnSettingsParams): Thenable; + sendLearnTransformationRequest(params: LearnTransformationParams): Thenable; + sendSaveTransformationRequest(params: SaveTransformationParams): Thenable; } diff --git a/extensions/import/src/services/features.ts b/extensions/import/src/services/features.ts index 3ffd5f418b..9b59f9de9c 100644 --- a/extensions/import/src/services/features.ts +++ b/extensions/import/src/services/features.ts @@ -85,12 +85,22 @@ export class FlatFileImportFeature extends SqlOpsFeature { return requestSender(Contracts.ChangeColumnSettingsRequest.type, params); }; + let sendLearnTransformationRequest = (params: Contracts.LearnTransformationParams): Thenable => { + return requestSender(Contracts.LearnTransformationRequest.type, params); + }; + + let sendSaveTransformationRequest = (params: Contracts.SaveTransformationParams): Thenable => { + return requestSender(Contracts.SaveTransformationRequest.type, params); + }; + return managerInstance.registerApi(ApiType.FlatFileProvider, { providerId: client.providerId, sendPROSEDiscoveryRequest, sendChangeColumnSettingsRequest, sendGetColumnInfoRequest, - sendInsertDataRequest + sendInsertDataRequest, + sendLearnTransformationRequest, + sendSaveTransformationRequest }); } } diff --git a/extensions/import/src/test/utils.test.ts b/extensions/import/src/test/utils.test.ts index e3cb1bd0d4..9c6d3389c8 100644 --- a/extensions/import/src/test/utils.test.ts +++ b/extensions/import/src/test/utils.test.ts @@ -6,7 +6,7 @@ import * as azdata from 'azdata'; import * as vscode from 'vscode'; import { ImportDataModel, ColumnMetadata } from '../wizard/api/models'; -import { FlatFileProvider, PROSEDiscoveryParams, InsertDataParams, GetColumnInfoParams, ChangeColumnSettingsParams, PROSEDiscoveryResponse, InsertDataResponse, ChangeColumnSettingsResponse, GetColumnInfoResponse } from '../services/contracts'; +import { FlatFileProvider, PROSEDiscoveryParams, InsertDataParams, GetColumnInfoParams, ChangeColumnSettingsParams, PROSEDiscoveryResponse, InsertDataResponse, ChangeColumnSettingsResponse, GetColumnInfoResponse, LearnTransformationParams, LearnTransformationResponse, SaveTransformationParams, SaveTransformationResponse } from '../services/contracts'; export class ImportTestUtils { @@ -170,6 +170,10 @@ export class TestImportDataModel implements ImportDataModel { schema: string; filePath: string; fileType: string; + transPreviews: string[][]; + originalProseColumns: ColumnMetadata[]; + derivedColumnName: string; + newFileSelected: boolean; } export class TestFlatFileProvider implements FlatFileProvider { @@ -186,6 +190,12 @@ export class TestFlatFileProvider implements FlatFileProvider { sendChangeColumnSettingsRequest(params: ChangeColumnSettingsParams): Thenable { throw new Error('Method not implemented.'); } + sendLearnTransformationRequest(params: LearnTransformationParams): Thenable { + throw new Error('Method not implemented.'); + } + sendSaveTransformationRequest(params: SaveTransformationParams): Thenable { + throw new Error('Method not implemented.'); + } } diff --git a/extensions/import/src/test/wizard/pages/prosePreviewPage.test.ts b/extensions/import/src/test/wizard/pages/prosePreviewPage.test.ts index 7f451ceb81..788aebe0a7 100644 --- a/extensions/import/src/test/wizard/pages/prosePreviewPage.test.ts +++ b/extensions/import/src/test/wizard/pages/prosePreviewPage.test.ts @@ -12,7 +12,6 @@ import { ImportDataModel } from '../../../wizard/api/models'; import { TestImportDataModel } from '../../utils.test'; import { ImportPage } from '../../../wizard/api/importPage'; import { ProsePreviewPage } from '../../../wizard/pages/prosePreviewPage'; - describe('import extension prose preview tests', function () { // declaring mock variables @@ -44,10 +43,10 @@ describe('import extension prose preview tests', function () { await new Promise(function (resolve) { page.registerContent(async (view) => { prosePreviewPage = new ProsePreviewPage(mockFlatFileWizard.object, page, mockImportModel.object, view, TypeMoq.It.isAny()); + mockFlatFileWizard.object.createDerivedColumnButton = azdata.window.createButton('TestButton'); pages.set(1, prosePreviewPage); await prosePreviewPage.start(); prosePreviewPage.setupNavigationValidator(); - await prosePreviewPage.onPageEnter(); resolve(); }); wizard.generateScriptButton.hidden = true; @@ -57,7 +56,6 @@ describe('import extension prose preview tests', function () { // checking if all the required components are correctly initialized should.notEqual(prosePreviewPage.table, undefined, 'table should not be undefined'); - should.notEqual(prosePreviewPage.refresh, undefined, 'refresh should not be undefined'); should.notEqual(prosePreviewPage.loading, undefined, 'loading should not be undefined'); should.notEqual(prosePreviewPage.form, undefined, 'form should not be undefined'); should.notEqual(prosePreviewPage.resultTextComponent, undefined, 'resultTextComponent should not be undefined'); diff --git a/extensions/import/src/wizard/api/basePage.ts b/extensions/import/src/wizard/api/basePage.ts index 4b5563107e..40034cb4d0 100644 --- a/extensions/import/src/wizard/api/basePage.ts +++ b/extensions/import/src/wizard/api/basePage.ts @@ -91,7 +91,7 @@ export abstract class BasePage { return values; } - public async getDatabaseValues(): Promise<{ displayName: string, name: string }[]> { + public async getDatabaseValues(): Promise { let idx = -1; let count = -1; let values = (await azdata.connection.listDatabases(this.model.server.connectionId)).map(db => { diff --git a/extensions/import/src/wizard/api/models.ts b/extensions/import/src/wizard/api/models.ts index 0d98fba37d..6f04b7c489 100644 --- a/extensions/import/src/wizard/api/models.ts +++ b/extensions/import/src/wizard/api/models.ts @@ -19,6 +19,8 @@ export interface ImportDataModel { schema: string; filePath: string; fileType: string; + originalProseColumns: ColumnMetadata[]; + newFileSelected: boolean; } /** diff --git a/extensions/import/src/wizard/flatFileWizard.ts b/extensions/import/src/wizard/flatFileWizard.ts index e4d00c5357..e7bb1240ed 100644 --- a/extensions/import/src/wizard/flatFileWizard.ts +++ b/extensions/import/src/wizard/flatFileWizard.ts @@ -23,6 +23,7 @@ export class FlatFileWizard { public page3: azdata.window.WizardPage; public page4: azdata.window.WizardPage; + public createDerivedColumnButton: azdata.window.Button; private importAnotherFileButton: azdata.window.Button; constructor( @@ -89,6 +90,8 @@ export class FlatFileWizard { await summaryPage.start(); }); + this.createDerivedColumnButton = azdata.window.createButton(constants.createDerivedColumn); + this.createDerivedColumnButton.hidden = true; this.importAnotherFileButton = azdata.window.createButton(constants.importNewFileText); this.importAnotherFileButton.onClick(() => { @@ -99,7 +102,7 @@ export class FlatFileWizard { }); this.importAnotherFileButton.hidden = true; - this.wizard.customButtons = [this.importAnotherFileButton]; + this.wizard.customButtons = [this.importAnotherFileButton, this.createDerivedColumnButton]; this.wizard.onPageChanged(async (event) => { let newPageIdx = event.newPage; let lastPageIdx = event.lastPage; diff --git a/extensions/import/src/wizard/pages/fileConfigPage.ts b/extensions/import/src/wizard/pages/fileConfigPage.ts index d191d1b871..b106620c4d 100644 --- a/extensions/import/src/wizard/pages/fileConfigPage.ts +++ b/extensions/import/src/wizard/pages/fileConfigPage.ts @@ -7,7 +7,7 @@ import * as azdata from 'azdata'; import * as vscode from 'vscode'; import { ImportPage } from '../api/importPage'; import * as constants from '../../common/constants'; -import * as fs from 'fs'; +import { promises as fs } from 'fs'; export class FileConfigPage extends ImportPage { @@ -85,7 +85,6 @@ export class FileConfigPage extends ImportPage { this._schemaLoader = schemaLoader; } - private tableNames: string[] = []; async start(): Promise { @@ -144,6 +143,7 @@ export class FileConfigPage extends ImportPage { this.model.server = connectionValue.connection; await this.populateDatabaseDropdown(); } + await this.populateDatabaseDropdown(); }); return { @@ -198,33 +198,35 @@ export class FileConfigPage extends ImportPage { this.databaseDropdown.values = []; this.schemaDropdown.values = []; - if (!this.model.server) { - //TODO handle error case - this.databaseDropdown.loading = false; - return false; - } - - let defaultServerDatabase = this.model.server.options.database; - - let values: any[]; try { - values = await this.getDatabaseValues(); - } catch (error) { - // This code is used in case of contained databases when the query will return an error. - console.log(error); - values = [{ displayName: defaultServerDatabase, name: defaultServerDatabase }]; - this.databaseDropdown.editable = false; + if (!this.model.server) { + //TODO handle error case + this.databaseDropdown.loading = false; + return false; + } + + let defaultServerDatabase = this.model.server.options.database; + + let values: azdata.CategoryValue[]; + try { + values = await this.getDatabaseValues(); + } catch (error) { + // This code is used in case of contained databases when the query will return an error. + console.log(error); + values = [{ displayName: defaultServerDatabase, name: defaultServerDatabase }]; + this.databaseDropdown.editable = false; + } + + this.model.database = defaultServerDatabase; + + this.databaseDropdown.updateProperties({ + values: values + }); + + this.databaseDropdown.value = { displayName: this.model.database, name: this.model.database }; + } finally { + this.databaseDropdown.loading = false; } - - this.model.database = defaultServerDatabase; - - this.databaseDropdown.updateProperties({ - values: values - }); - - this.databaseDropdown.value = { displayName: this.model.database, name: this.model.database }; - this.databaseDropdown.loading = false; - return true; } @@ -232,16 +234,29 @@ export class FileConfigPage extends ImportPage { this.fileTextBox = this.view.modelBuilder.inputBox().withProps({ required: true, validationErrorMessage: constants.invalidFileLocationError - }).withValidation((component) => { - return fs.existsSync(component.value); + }).withValidation(async (component) => { + if (component.value) { + try { + await fs.access(component.value); + return true; + } catch (e) { + return false; + } + } + return false; }).component(); + this.fileTextBox.onTextChanged(e => { + this.model.newFileSelected = true; + }); + this.fileButton = this.view.modelBuilder.button().withProps({ label: constants.browseFilesText, secondary: true }).component(); this.fileButton.onDidClick(async (click) => { + this.model.newFileSelected = true; let fileUris = await vscode.window.showOpenDialog( { canSelectFiles: true, @@ -324,6 +339,7 @@ export class FileConfigPage extends ImportPage { }).component(); this.tableNameTextBox.onTextChanged((tableName) => { + this.model.newFileSelected = true; this.model.table = tableName; }); @@ -415,33 +431,6 @@ export class FileConfigPage extends ImportPage { delete this.model.database; delete this.model.schema; } - - // private async populateTableNames(): Promise { - // this.tableNames = []; - // let databaseName = (this.databaseDropdown.value).name; - // - // if (!databaseName || databaseName.length === 0) { - // this.tableNames = []; - // return false; - // } - // - // let connectionUri = await azdata.connection.getUriForConnection(this.model.server.connectionId); - // let queryProvider = azdata.dataprotocol.getProvider(this.model.server.providerName, azdata.DataProviderType.QueryProvider); - // let results: azdata.SimpleExecuteResult; - // - // try { - // //let query = sqlstring.format('USE ?; SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = \'BASE TABLE\'', [databaseName]); - // //results = await queryProvider.runQueryAndReturn(connectionUri, query); - // } catch (e) { - // return false; - // } - // - // this.tableNames = results.rows.map(row => { - // return row[0].displayValue; - // }); - // - // return true; - // } } diff --git a/extensions/import/src/wizard/pages/modifyColumnsPage.ts b/extensions/import/src/wizard/pages/modifyColumnsPage.ts index abb1d356d2..095f07ec0e 100644 --- a/extensions/import/src/wizard/pages/modifyColumnsPage.ts +++ b/extensions/import/src/wizard/pages/modifyColumnsPage.ts @@ -104,7 +104,6 @@ export class ModifyColumnsPage extends ImportPage { }); }); - this.form = this.view.modelBuilder.formContainer() .withFormItems( [ @@ -131,15 +130,28 @@ export class ModifyColumnsPage extends ImportPage { await this.populateTable(); this.instance.changeNextButtonLabel(constants.importDataText); this.loading.loading = false; - + this.instance.registerNavigationValidator((info) => { + return this.table.data && this.table.data.length > 0; + }); return true; } override async onPageLeave(): Promise { + await this.emptyTable(); this.instance.changeNextButtonLabel(constants.nextText); + this.instance.registerNavigationValidator((info) => { + return true; + }); return undefined; } + private emptyTable() { + this.table.updateProperties({ + data: [], + columns: [] + }); + } + override async cleanup(): Promise { delete this.model.proseColumns; this.instance.changeNextButtonLabel(constants.nextText); @@ -147,12 +159,6 @@ export class ModifyColumnsPage extends ImportPage { return true; } - public override setupNavigationValidator() { - this.instance.registerNavigationValidator((info) => { - return this.table.data && this.table.data.length > 0; - }); - } - private async populateTable() { let data: ColumnMetadataArray[] = []; diff --git a/extensions/import/src/wizard/pages/prosePreviewPage.ts b/extensions/import/src/wizard/pages/prosePreviewPage.ts index 5844b1fee1..5e6ead4546 100644 --- a/extensions/import/src/wizard/pages/prosePreviewPage.ts +++ b/extensions/import/src/wizard/pages/prosePreviewPage.ts @@ -2,17 +2,17 @@ * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the Source EULA. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ - import * as azdata from 'azdata'; import { ImportPage } from '../api/importPage'; import * as constants from '../../common/constants'; +import { DerivedColumnDialog } from '../../dialogs/derivedColumnDialog'; +import * as vscode from 'vscode'; export class ProsePreviewPage extends ImportPage { private _table: azdata.TableComponent; private _loading: azdata.LoadingComponent; private _form: azdata.FormContainer; - private _refresh: azdata.ButtonComponent; private _resultTextComponent: azdata.TextComponent; private _isSuccess: boolean; @@ -40,14 +40,6 @@ export class ProsePreviewPage extends ImportPage { this._form = form; } - public get refresh(): azdata.ButtonComponent { - return this._refresh; - } - - public set refresh(refresh: azdata.ButtonComponent) { - this._refresh = refresh; - } - public get resultTextComponent(): azdata.TextComponent { return this._resultTextComponent; } @@ -70,14 +62,22 @@ export class ProsePreviewPage extends ImportPage { columns: undefined, forceFitColumns: azdata.ColumnSizingMode.DataFit }).component(); - this.refresh = this.view.modelBuilder.button().withProps({ - label: constants.refreshText, - isFile: false, - secondary: true - }).component(); - this.refresh.onDidClick(async () => { - await this.onPageEnter(); + this.instance.createDerivedColumnButton.onClick(async (e) => { + const derivedColumnDialog = new DerivedColumnDialog(this.model, this.provider); + const response = await derivedColumnDialog.createDerivedColumn(); + if (response) { + this.model.proseColumns.push({ + columnName: response.derivedColumnName, + dataType: 'nvarchar(MAX)', + primaryKey: false, + nullable: true + }); + response.derivedColumnDataPreview.forEach((v, i) => { + this.model.proseDataPreview[i].push(v); + }); + this.populateTable(this.model.proseDataPreview, this.model.proseColumns.map(c => c.columnName)); + } }); this.loading = this.view.modelBuilder.loadingComponent().component(); @@ -94,9 +94,9 @@ export class ProsePreviewPage extends ImportPage { }, { component: this.table, - title: '', - actions: [this.refresh] + title: '' } + ]).component(); this.loading.component = this.form; @@ -107,22 +107,30 @@ export class ProsePreviewPage extends ImportPage { } async onPageEnter(): Promise { - this.loading.loading = true; let proseResult: boolean; let error: string; - try { - proseResult = await this.handleProse(); - } catch (ex) { - error = ex.toString(); + const enablePreviewFeatures = vscode.workspace.getConfiguration('workbench').get('enablePreviewFeatures'); + if (this.model.newFileSelected) { + this.loading.loading = true; + try { + proseResult = await this.learnFile(); + } catch (ex) { + error = ex.toString(); + this.instance.wizard.message = { + level: azdata.window.MessageLevel.Error, + text: error + }; + } + this.model.newFileSelected = false; + this.loading.loading = false; } - - this.loading.loading = false; - if (proseResult) { + if (!this.model.newFileSelected || proseResult) { await this.populateTable(this.model.proseDataPreview, this.model.proseColumns.map(c => c.columnName)); this.isSuccess = true; if (this.form) { this.resultTextComponent.value = constants.successTitleText; } + this.instance.createDerivedColumnButton.hidden = !enablePreviewFeatures; return true; } else { await this.populateTable([], []); @@ -135,7 +143,7 @@ export class ProsePreviewPage extends ImportPage { } override async onPageLeave(): Promise { - await this.emptyTable(); + this.instance.createDerivedColumnButton.hidden = true; return true; } @@ -156,7 +164,7 @@ export class ProsePreviewPage extends ImportPage { }); } - private async handleProse(): Promise { + private async learnFile(): Promise { const response = await this.provider.sendPROSEDiscoveryRequest({ filePath: this.model.filePath, tableName: this.model.table, @@ -170,6 +178,7 @@ export class ProsePreviewPage extends ImportPage { } this.model.proseColumns = []; + this.model.originalProseColumns = []; if (response.columnInfo) { response.columnInfo.forEach((column) => { this.model.proseColumns.push({ @@ -178,6 +187,12 @@ export class ProsePreviewPage extends ImportPage { primaryKey: false, nullable: column.isNullable }); + this.model.originalProseColumns.push({ + columnName: column.name, + dataType: column.sqlType, + primaryKey: false, + nullable: column.isNullable + }); }); return true; } @@ -203,8 +218,4 @@ export class ProsePreviewPage extends ImportPage { width: '700', }); } - - private async emptyTable() { - this.table.updateProperties([]); - } } diff --git a/extensions/import/src/wizard/pages/summaryPage.ts b/extensions/import/src/wizard/pages/summaryPage.ts index ac121362b9..ec8557fdeb 100644 --- a/extensions/import/src/wizard/pages/summaryPage.ts +++ b/extensions/import/src/wizard/pages/summaryPage.ts @@ -50,7 +50,12 @@ export class SummaryPage extends ImportPage { async start(): Promise { this.table = this.view.modelBuilder.table().component(); - this.statusText = this.view.modelBuilder.text().component(); + this.statusText = this.view.modelBuilder.text().withProps({ + CSSStyles: { + 'user-select': 'text', + 'font-size': '13px' + } + }).component(); this.loading = this.view.modelBuilder.loadingComponent().withItem(this.statusText).component(); this.form = this.view.modelBuilder.formContainer().withFormItems(