Adding Derived Columns to ADS Flatfile Import (#16795)

* Adding derived column boilerplate * brandan preliminary frontend changes * empty commit * added new param * updating contracts, dialogue changes * utils changes, saving timeout attempt * pushing for aasim * Cleaning up code and fixing the issue in theory * changing button, did not solve independent scroll * Fixing the scroll bar issue * updating flat file service * adding override keyword to overrriden method * improving UI * pushing changes associated with resolved comments * localizing strings, editing comments * all comments resolved * Fixing a test * updating import package Updating azure MFA bug * Clearing navigation validator Fixing broken table name change * fixed prose test * removing unused code from tests * Fixed PR comments * Fixing some PR comments * WIP * Fixing transformation code and create derived column dialog styling * removing unused code * Adding comment for console log * fixed table styling * Adding some aria labels * Fixed some code cleanup issues * update import service Co-authored-by: Aasim Khan <aasimkhan30@gmail.com> Co-authored-by: bnhoule <t-bhoule@microsoft.com>
2026-02-07 17:23:56 -05:00 · 2021-09-21 17:11:00 -05:00
parent fad2963202
commit d3e163a1d7
15 changed files with 569 additions and 108 deletions
--- a/extensions/import/config.json
+++ b/extensions/import/config.json
@@ -1,7 +1,7 @@
 {
 	"downloadUrl": "https://sqlopsextensions.blob.core.windows.net/extensions/import/service/{#version#}/{#fileName#}",
 	"useDefaultLinuxRuntime": true,
-	"version": "0.0.8",
+	"version": "0.0.9",
 	"downloadFileNames": {
 		"Windows_64": "win-x64.zip",
 		"Windows_86": "win-x86.zip",
--- a/extensions/import/package.json
+++ b/extensions/import/package.json
@@ -2,7 +2,7 @@
 	"name": "import",
 	"displayName": "SQL Server Import",
 	"description": "SQL Server Import for Azure Data Studio supports importing CSV or JSON files into SQL Server.",
-	"version": "1.4.1",
+	"version": "1.5.0",
 	"publisher": "Microsoft",
 	"preview": true,
 	"engines": {
--- a/extensions/import/src/common/constants.ts
+++ b/extensions/import/src/common/constants.ts
@@ -58,6 +58,23 @@ export const page2NameText = localize('flatFileImport.page2Name', "Preview Data"
 export const page3NameText = localize('flatFileImport.page3Name', "Modify Columns");
 export const page4NameText = localize('flatFileImport.page4Name', "Summary");
 export const importNewFileText = localize('flatFileImport.importNewFile', "Import new file");
-
+export const createDerivedColumn = localize('flatFileImport.createDerivedColumn', "Create derived column");
+export const specifyDerivedColNameTitle = localize('flatFileImport.specifyDerivedColNameTitle', "Column Name");
+export const specifyTransformation = localize('flatFileImport.specifyTransformation', "Specify Transformation");
+export const previewTransformation = localize('flatFileImport.previewTransformation', "Preview Transformation");
+export const columnTableTitle = localize('flatFileImport.columnTableTitle', "Column");
+export const headerIntructionText = localize('flatFileImport.headerIntructionText', "Welcome to the Derived Column Tool! To get started, please follow the steps below:");
+export const deriverColumnInstruction1 = localize('flatFileImport.deriverColumnInstruction1', "Select the columns of data on the left required to derive your new column");
+export const deriverColumnInstruction2 = localize('flatFileImport.deriverColumnInstruction2', "Select a row and specify an example transformation that you would like applied to the rest of the column");
+export const deriverColumnInstruction3 = localize('flatFileImport.deriverColumnInstruction3', "Click \"Preview Transformation\" to preview the transformation");
+export const deriverColumnInstruction4 = localize('flatFileImport.deriverColumnInstruction4', "Refine your transformation until you have the desired column");
+export const deriverColumnInstruction5 = localize('flatFileImport.deriverColumnInstruction5', "Specify the new derived column\'s name and click \"Done\"");
+export const selectAllColumns = localize('flatFileImport.selectAllColumns', "Select all columns");
+export function specifyTransformationForRow(rowIndex: number): string {
+	return localize('flatFileImport.specifyTransformationForRow', "Specify transformation for row {0}", rowIndex);
+}
+export function selectColumn(colName: string): string {
+	return localize('flatFileImport.selectColumn', "Select column {0}", colName);
+}
 // SQL Queries
 export const selectSchemaQuery = `SELECT name FROM sys.schemas`;
--- a/extensions/import/src/dialogs/derivedColumnDialog.ts
+++ b/extensions/import/src/dialogs/derivedColumnDialog.ts
@@ -0,0 +1,368 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import * as azdata from 'azdata';
+import { ImportDataModel } from '../wizard/api/models';
+import * as EventEmitter from 'events';
+import { FlatFileProvider } from '../services/contracts';
+import * as constants from '../common/constants';
+
+const headerLeft: azdata.CssStyles = {
+	'border': 'none',
+	'text-align': 'left',
+	'white-space': 'nowrap',
+	'text-overflow': 'ellipsis',
+	'overflow': 'hidden',
+	'border-bottom': '2px solid'
+};
+
+const styleLeft: azdata.CssStyles = {
+	'border': 'none',
+	'text-align': 'left',
+	'white-space': 'nowrap',
+	'text-overflow': 'ellipsis',
+	'overflow': 'hidden',
+};
+
+export class DerivedColumnDialog {
+	private _dialogObject: azdata.window.Dialog;
+	private _doneEmitter: EventEmitter = new EventEmitter();
+	private _currentTransformation: string[] = [];
+	private _currentDerivedColumnName!: string;
+	private _view!: azdata.ModelView;
+	private _specifyTransformations: azdata.InputBoxComponent[] = [];
+	private _headerInstructionText: azdata.TextComponent;
+	private _bodyInstructionText: azdata.TextComponent;
+
+	private _applyButton!: azdata.window.Button;
+	private _transformationTable!: azdata.DeclarativeTableComponent;
+	private _transformationContainer!: azdata.FlexContainer;
+	private _specifyDerivedColumnNameContainer!: azdata.FlexContainer;
+
+	constructor(private _model: ImportDataModel, private _provider: FlatFileProvider) {
+	}
+
+	public createDerivedColumn(): Promise<DerivedColumnDialogResult | undefined> {
+		this._applyButton = azdata.window.createButton(constants.previewTransformation);
+		this._applyButton.enabled = false;
+		this._dialogObject = azdata.window.createModelViewDialog(
+			constants.createDerivedColumn,
+			'DerivedColumnDialog',
+			'wide'
+		);
+		this._dialogObject.customButtons = [this._applyButton];
+		this._applyButton.hidden = false;
+
+		let tab = azdata.window.createTab('');
+		tab.registerContent(async (view: azdata.ModelView) => {
+			const columnTableData: azdata.DeclarativeTableCellValue[][] = [];
+			this._model.originalProseColumns.forEach(c => {
+				const tableRow: azdata.DeclarativeTableCellValue[] = [];
+				tableRow.push(
+					{ value: false, ariaLabel: constants.selectColumn(c.columnName) },
+					{ value: c.columnName }
+				);
+				columnTableData.push(tableRow);
+			});
+			this._view = view;
+			const columnTable = view.modelBuilder.declarativeTable().withProps({
+				columns: [
+					{
+						displayName: '',
+						ariaLabel: constants.selectAllColumns,
+						valueType: azdata.DeclarativeDataType.boolean,
+						isReadOnly: false,
+						showCheckAll: true,
+						width: '20px',
+						headerCssStyles: headerLeft,
+						rowCssStyles: styleLeft,
+					},
+					{
+						displayName: constants.columnTableTitle,
+						valueType: azdata.DeclarativeDataType.string,
+						isReadOnly: true,
+						width: '140px',
+						headerCssStyles: headerLeft,
+						rowCssStyles: styleLeft,
+						ariaLabel: constants.columnTableTitle
+					}
+				],
+				dataValues: columnTableData,
+				CSSStyles: {
+					'table-layout': 'fixed'
+				}
+			}).component();
+
+
+			columnTable.onDataChanged(e => {
+				if (e.value) {
+					// Adding newly selected column to transformation table
+					this._transformationTable.columns.push({
+						displayName: this._model.proseColumns[e.row].columnName,
+						valueType: azdata.DeclarativeDataType.string,
+						isReadOnly: true,
+						width: '100px',
+						headerCssStyles: headerLeft,
+						rowCssStyles: styleLeft
+					});
+
+					this._model.proseDataPreview.forEach((v, i) => {
+						this._transformationTable.dataValues[i].push({ value: v[e.row] });
+					});
+				}
+				else {
+					// Removing unselected column from transformation table
+					let removeIndex = this._transformationTable.columns.findIndex(v => this._model.proseColumns[e.row].columnName === v.displayName);
+					this._transformationTable.columns.splice(removeIndex, 1);
+					for (let index = 0; index < this._model.proseDataPreview.length; index++) {
+						this._transformationTable.dataValues[index].splice(removeIndex, 1);
+					}
+				}
+				const isColumnAdded = this._transformationTable.columns.length > 1;
+				this.clearAndAddTransformationContainerComponents(isColumnAdded);
+				this._applyButton.enabled = isColumnAdded;
+			});
+
+
+			const columnContainer = view.modelBuilder.flexContainer().withLayout({
+				flexFlow: 'column',
+				height: '100%'
+			}).withProps({
+				CSSStyles: {
+					'border-right': 'solid 1px'
+				}
+			}).component();
+			columnContainer.addItem(columnTable, { flex: '1 1 auto', CSSStyles: { 'overflow-y': 'hidden' } });
+
+			const transformationTableData: azdata.DeclarativeTableCellValue[][] = [];
+			for (let index = 0; index < this._model.proseDataPreview.length; index++) {
+				this._specifyTransformations.push(this._view.modelBuilder.inputBox().withProps({
+					value: '',
+					placeHolder: constants.specifyTransformation,
+					ariaLabel: constants.specifyTransformationForRow(index)
+				}).component());
+				transformationTableData.push([{
+					value: this._specifyTransformations[index]
+				}]);
+			}
+
+			this._transformationTable = view.modelBuilder.declarativeTable().withProps({
+				columns: [
+					{
+						displayName: constants.specifyTransformation,
+						ariaLabel: constants.specifyTransformation,
+						valueType: azdata.DeclarativeDataType.component,
+						isReadOnly: false,
+						width: '200px',
+						headerCssStyles: headerLeft,
+						rowCssStyles: styleLeft
+					}
+				],
+				CSSStyles: {
+					'table-layout': 'fixed',
+					'overflow': 'scroll',
+				},
+				width: '800px',
+				dataValues: transformationTableData
+			}).component();
+
+			this._applyButton.onClick(async e => {
+				const requiredColNames = this._transformationTable.columns.map(v => v.displayName);
+				requiredColNames.splice(0, 1); // Removing specify transformation column
+				const transExamples: string[] = [];
+				const transExampleIndices: number[] = [];
+
+				// Getting all the example transformations specified by the user
+				this._transformationTable.dataValues.forEach((v, index) => {
+					const example = (<azdata.InputBoxComponent>v[0].value).value as string;
+					if (example !== '') {
+						transExamples.push(example);
+						transExampleIndices.push(index);
+					}
+				});
+
+				if (transExamples.length > 0) {
+					try {
+						const response = await this._provider.sendLearnTransformationRequest({
+							columnNames: requiredColNames,
+							transformationExamples: transExamples,
+							transformationExampleRowIndices: transExampleIndices
+						});
+						this._currentTransformation = response.transformationPreview;
+						this._currentTransformation.forEach((v, i) => {
+							(<azdata.InputBoxComponent>this._transformationTable.dataValues[i][0].value).placeHolder = v;
+						});
+						this.clearAndAddTransformationContainerComponents(true);
+					} catch (e) {
+						this._dialogObject.message = {
+							text: e.toString(),
+							level: azdata.window.MessageLevel.Error
+						};
+					}
+				}
+				this.validatePage();
+			});
+
+			const columnNameText = view.modelBuilder.text().withProps({
+				value: constants.specifyDerivedColNameTitle,
+				requiredIndicator: true,
+				CSSStyles: {
+					'font-size': '13px',
+					'font-weight': 'bold'
+				}
+			}).component();
+
+			const columnNameInput = view.modelBuilder.inputBox().withProps({
+				ariaLabel: constants.specifyDerivedColNameTitle,
+				required: true
+			}).component();
+
+			columnNameInput.onTextChanged(e => {
+				if (e) {
+					this._currentDerivedColumnName = e;
+					this.validatePage();
+				}
+			});
+
+			this._specifyDerivedColumnNameContainer = view.modelBuilder.flexContainer().withItems([
+				columnNameText,
+				columnNameInput
+			]).withLayout({
+				width: '500px'
+			}).component();
+
+			this._transformationContainer = view.modelBuilder.flexContainer().withLayout({
+				flexFlow: 'column',
+				height: '100%',
+			}).withProps({
+				CSSStyles: {
+					'overflow-y': 'auto',
+					'margin-left': '10px',
+				}
+			}).component();
+
+			this._headerInstructionText = this._view.modelBuilder.text()
+				.withProps({
+					value: constants.headerIntructionText,
+					CSSStyles: {
+						'font-size': 'x-large',
+						'line-height': '22pt',
+						'margin-bottom': '0.7em'
+					}
+				}).component();
+
+			this._bodyInstructionText = this._view.modelBuilder.text()
+				.withProps({
+					value: [
+						constants.deriverColumnInstruction1,
+						constants.deriverColumnInstruction2,
+						constants.deriverColumnInstruction3,
+						constants.deriverColumnInstruction4,
+						constants.deriverColumnInstruction5,
+					],
+					textType: azdata.TextType.OrderedList,
+					CSSStyles: {
+						'font-size': 'large',
+						'line-height': '22pt',
+						'margin-left': '1em',
+						'margin-top': '0em'
+					}
+				}).component();
+
+			this.clearAndAddTransformationContainerComponents(false);
+
+			const flexGrid = view.modelBuilder.flexContainer().withLayout({
+				flexFlow: 'row',
+				height: '100%',
+				width: '100%'
+			}).component();
+
+			/**
+			 * Setting height of the div based on the total viewport height after removing dialog
+			 * header and footer heights. With this the div will occupy the entire page space of the dialog.
+			 */
+			flexGrid.addItem(columnContainer, {
+				flex: '0 0 auto',
+				CSSStyles: {
+					'min-height': 'calc(100vh - 160px)'
+				}
+			});
+			flexGrid.addItem(this._transformationContainer, {
+				flex: '0 0 auto',
+				CSSStyles: {
+					'overflow': 'scroll',
+					'padding-right': '10px',
+					'width': '900px',
+					'max-height': 'calc(100vh - 160px)'
+				}
+			});
+
+			const formBuilder = view.modelBuilder.formContainer().withFormItems(
+				[
+					{
+						component: flexGrid
+					}
+				],
+				{
+					horizontal: false
+				}
+			);
+			const form = formBuilder.withLayout({ width: '100%' }).component();
+			return view.initializeModel(form);
+		});
+
+		this._dialogObject.okButton.onClick(e => {
+			this._doneEmitter.emit('done');
+		});
+
+		this._dialogObject.cancelButton.onClick(e => {
+			this._doneEmitter.emit('close');
+		});
+
+		this._dialogObject.content = [tab];
+		azdata.window.openDialog(this._dialogObject);
+		return new Promise((resolve) => {
+			this._doneEmitter.once('done', async () => {
+				try {
+					await this._provider.sendSaveTransformationRequest({
+						derivedColumnName: this._currentDerivedColumnName
+					});
+					resolve({
+						derivedColumnName: this._currentDerivedColumnName,
+						derivedColumnDataPreview: this._currentTransformation
+					});
+				} catch (e) {
+					console.log(e); // Need to have better error handling for saved transformation.However this seems to be mostly a non-issue.
+
+					resolve(undefined);
+				}
+			});
+
+			this._doneEmitter.once('close', async () => {
+				resolve(undefined);
+			});
+		});
+	}
+
+	private clearAndAddTransformationContainerComponents(addTable: boolean): void {
+		this._transformationContainer.clearItems();
+		if (addTable) {
+			this._transformationContainer.addItem(this._specifyDerivedColumnNameContainer, { flex: '0 0 auto' });
+			this._transformationContainer.addItem(this._transformationTable, { flex: '1 1 auto', CSSStyles: { 'overflow': 'scroll' } });
+		}
+		else {
+			this._transformationContainer.addItem(this._headerInstructionText, { flex: '0 0 auto' });
+			this._transformationContainer.addItem(this._bodyInstructionText, { flex: '0 0 auto' });
+		}
+	}
+
+	private validatePage(): void {
+		this._dialogObject.okButton.enabled = this._currentDerivedColumnName !== undefined && this._currentTransformation.length !== 0;
+	}
+}
+
+export interface DerivedColumnDialogResult {
+	derivedColumnName?: string;
+	derivedColumnDataPreview?: string[];
+}
--- a/extensions/import/src/services/contracts.ts
+++ b/extensions/import/src/services/contracts.ts
@@ -52,6 +52,38 @@ export interface ColumnInfo {
 }


+/**
+ * LearnTransformationRequest
+ * Send this request to learn a transformation and preview it
+ */
+const learnTransformationRequestName = 'flatfile/learnTransformation';
+
+export interface LearnTransformationParams {
+	columnNames: string[];
+	transformationExamples: string[];
+	transformationExampleRowIndices: number[];
+}
+
+export interface LearnTransformationResponse {
+	transformationPreview: string[];
+}
+
+
+/**
+* SaveTransformationRequest
+* Send this request to save a transformation to be applied on insertion into database
+*/
+const saveTransformationRequestName = 'flatfile/saveTransformation';
+
+export interface SaveTransformationParams {
+	derivedColumnName: string;
+}
+
+export interface SaveTransformationResponse {
+	numTransformations: number;
+}
+
+
 /**
 * PROSEDiscoveryRequest
 * Send this request to create a new PROSE session with a new file and preview it
@@ -139,6 +171,14 @@ export namespace ChangeColumnSettingsRequest {
 	export const type = new RequestType<ChangeColumnSettingsParams, ChangeColumnSettingsResponse, void, void>(changeColumnSettingsRequestName);
 }

+export namespace LearnTransformationRequest {
+	export const type = new RequestType<LearnTransformationParams, LearnTransformationResponse, void, void>(learnTransformationRequestName);
+}
+
+export namespace SaveTransformationRequest {
+	export const type = new RequestType<SaveTransformationParams, SaveTransformationResponse, void, void>(saveTransformationRequestName);
+}
+

 export interface FlatFileProvider {
 	providerId?: string;
@@ -147,4 +187,6 @@ export interface FlatFileProvider {
 	sendInsertDataRequest(params: InsertDataParams): Thenable<InsertDataResponse>;
 	sendGetColumnInfoRequest(params: GetColumnInfoParams): Thenable<GetColumnInfoResponse>;
 	sendChangeColumnSettingsRequest(params: ChangeColumnSettingsParams): Thenable<ChangeColumnSettingsResponse>;
+	sendLearnTransformationRequest(params: LearnTransformationParams): Thenable<LearnTransformationResponse>;
+	sendSaveTransformationRequest(params: SaveTransformationParams): Thenable<SaveTransformationResponse>;
 }
--- a/extensions/import/src/services/features.ts
+++ b/extensions/import/src/services/features.ts
@@ -85,12 +85,22 @@ export class FlatFileImportFeature extends SqlOpsFeature<undefined> {
 			return requestSender(Contracts.ChangeColumnSettingsRequest.type, params);
 		};

+		let sendLearnTransformationRequest = (params: Contracts.LearnTransformationParams): Thenable<Contracts.LearnTransformationResponse> => {
+			return requestSender(Contracts.LearnTransformationRequest.type, params);
+		};
+
+		let sendSaveTransformationRequest = (params: Contracts.SaveTransformationParams): Thenable<Contracts.SaveTransformationResponse> => {
+			return requestSender(Contracts.SaveTransformationRequest.type, params);
+		};
+
 		return managerInstance.registerApi<Contracts.FlatFileProvider>(ApiType.FlatFileProvider, {
 			providerId: client.providerId,
 			sendPROSEDiscoveryRequest,
 			sendChangeColumnSettingsRequest,
 			sendGetColumnInfoRequest,
-			sendInsertDataRequest
+			sendInsertDataRequest,
+			sendLearnTransformationRequest,
+			sendSaveTransformationRequest
 		});
 	}
 }
--- a/extensions/import/src/test/utils.test.ts
+++ b/extensions/import/src/test/utils.test.ts
@@ -6,7 +6,7 @@
 import * as azdata from 'azdata';
 import * as vscode from 'vscode';
 import { ImportDataModel, ColumnMetadata } from '../wizard/api/models';
-import { FlatFileProvider, PROSEDiscoveryParams, InsertDataParams, GetColumnInfoParams, ChangeColumnSettingsParams, PROSEDiscoveryResponse, InsertDataResponse, ChangeColumnSettingsResponse, GetColumnInfoResponse } from '../services/contracts';
+import { FlatFileProvider, PROSEDiscoveryParams, InsertDataParams, GetColumnInfoParams, ChangeColumnSettingsParams, PROSEDiscoveryResponse, InsertDataResponse, ChangeColumnSettingsResponse, GetColumnInfoResponse, LearnTransformationParams, LearnTransformationResponse, SaveTransformationParams, SaveTransformationResponse } from '../services/contracts';

 export class ImportTestUtils {

@@ -170,6 +170,10 @@ export class TestImportDataModel implements ImportDataModel {
 	schema: string;
 	filePath: string;
 	fileType: string;
+	transPreviews: string[][];
+	originalProseColumns: ColumnMetadata[];
+	derivedColumnName: string;
+	newFileSelected: boolean;
 }

 export class TestFlatFileProvider implements FlatFileProvider {
@@ -186,6 +190,12 @@ export class TestFlatFileProvider implements FlatFileProvider {
 	sendChangeColumnSettingsRequest(params: ChangeColumnSettingsParams): Thenable<ChangeColumnSettingsResponse> {
 		throw new Error('Method not implemented.');
 	}
+	sendLearnTransformationRequest(params: LearnTransformationParams): Thenable<LearnTransformationResponse> {
+		throw new Error('Method not implemented.');
+	}
+	sendSaveTransformationRequest(params: SaveTransformationParams): Thenable<SaveTransformationResponse> {
+		throw new Error('Method not implemented.');
+	}

 }

--- a/extensions/import/src/test/wizard/pages/prosePreviewPage.test.ts
+++ b/extensions/import/src/test/wizard/pages/prosePreviewPage.test.ts
@@ -12,7 +12,6 @@ import { ImportDataModel } from '../../../wizard/api/models';
 import { TestImportDataModel } from '../../utils.test';
 import { ImportPage } from '../../../wizard/api/importPage';
 import { ProsePreviewPage } from '../../../wizard/pages/prosePreviewPage';
-
 describe('import extension prose preview tests', function () {

 	// declaring mock variables
@@ -44,10 +43,10 @@ describe('import extension prose preview tests', function () {
 		await new Promise<void>(function (resolve) {
 			page.registerContent(async (view) => {
 				prosePreviewPage = new ProsePreviewPage(mockFlatFileWizard.object, page, mockImportModel.object, view, TypeMoq.It.isAny());
+				mockFlatFileWizard.object.createDerivedColumnButton = azdata.window.createButton('TestButton');
 				pages.set(1, prosePreviewPage);
 				await prosePreviewPage.start();
 				prosePreviewPage.setupNavigationValidator();
-				await prosePreviewPage.onPageEnter();
 				resolve();
 			});
 			wizard.generateScriptButton.hidden = true;
@@ -57,7 +56,6 @@ describe('import extension prose preview tests', function () {

 		// checking if all the required components are correctly initialized
 		should.notEqual(prosePreviewPage.table, undefined, 'table should not be undefined');
-		should.notEqual(prosePreviewPage.refresh, undefined, 'refresh should not be undefined');
 		should.notEqual(prosePreviewPage.loading, undefined, 'loading should not be undefined');
 		should.notEqual(prosePreviewPage.form, undefined, 'form should not be undefined');
 		should.notEqual(prosePreviewPage.resultTextComponent, undefined, 'resultTextComponent should not be undefined');
--- a/extensions/import/src/wizard/api/basePage.ts
+++ b/extensions/import/src/wizard/api/basePage.ts
@@ -91,7 +91,7 @@ export abstract class BasePage {
 		return values;
 	}

-	public async getDatabaseValues(): Promise<{ displayName: string, name: string }[]> {
+	public async getDatabaseValues(): Promise<azdata.CategoryValue[]> {
 		let idx = -1;
 		let count = -1;
 		let values = (await azdata.connection.listDatabases(this.model.server.connectionId)).map(db => {
--- a/extensions/import/src/wizard/api/models.ts
+++ b/extensions/import/src/wizard/api/models.ts
@@ -19,6 +19,8 @@ export interface ImportDataModel {
 	schema: string;
 	filePath: string;
 	fileType: string;
+	originalProseColumns: ColumnMetadata[];
+	newFileSelected: boolean;
 }

 /**
--- a/extensions/import/src/wizard/flatFileWizard.ts
+++ b/extensions/import/src/wizard/flatFileWizard.ts
@@ -23,6 +23,7 @@ export class FlatFileWizard {
 	public page3: azdata.window.WizardPage;
 	public page4: azdata.window.WizardPage;

+	public createDerivedColumnButton: azdata.window.Button;
 	private importAnotherFileButton: azdata.window.Button;

 	constructor(
@@ -89,6 +90,8 @@ export class FlatFileWizard {
 			await summaryPage.start();
 		});

+		this.createDerivedColumnButton = azdata.window.createButton(constants.createDerivedColumn);
+		this.createDerivedColumnButton.hidden = true;

 		this.importAnotherFileButton = azdata.window.createButton(constants.importNewFileText);
 		this.importAnotherFileButton.onClick(() => {
@@ -99,7 +102,7 @@ export class FlatFileWizard {
 		});

 		this.importAnotherFileButton.hidden = true;
-		this.wizard.customButtons = [this.importAnotherFileButton];
+		this.wizard.customButtons = [this.importAnotherFileButton, this.createDerivedColumnButton];
 		this.wizard.onPageChanged(async (event) => {
 			let newPageIdx = event.newPage;
 			let lastPageIdx = event.lastPage;
--- a/extensions/import/src/wizard/pages/fileConfigPage.ts
+++ b/extensions/import/src/wizard/pages/fileConfigPage.ts
@@ -7,7 +7,7 @@ import * as azdata from 'azdata';
 import * as vscode from 'vscode';
 import { ImportPage } from '../api/importPage';
 import * as constants from '../../common/constants';
-import * as fs from 'fs';
+import { promises as fs } from 'fs';

 export class FileConfigPage extends ImportPage {

@@ -85,7 +85,6 @@ export class FileConfigPage extends ImportPage {
 		this._schemaLoader = schemaLoader;
 	}

-
 	private tableNames: string[] = [];

 	async start(): Promise<boolean> {
@@ -144,6 +143,7 @@ export class FileConfigPage extends ImportPage {
 				this.model.server = connectionValue.connection;
 				await this.populateDatabaseDropdown();
 			}
+			await this.populateDatabaseDropdown();
 		});

 		return {
@@ -198,33 +198,35 @@ export class FileConfigPage extends ImportPage {
 		this.databaseDropdown.values = [];
 		this.schemaDropdown.values = [];

-		if (!this.model.server) {
-			//TODO handle error case
-			this.databaseDropdown.loading = false;
-			return false;
-		}
-
-		let defaultServerDatabase = this.model.server.options.database;
-
-		let values: any[];
 		try {
-			values = await this.getDatabaseValues();
-		} catch (error) {
-			// This code is used in case of contained databases when the query will return an error.
-			console.log(error);
-			values = [{ displayName: defaultServerDatabase, name: defaultServerDatabase }];
-			this.databaseDropdown.editable = false;
+			if (!this.model.server) {
+				//TODO handle error case
+				this.databaseDropdown.loading = false;
+				return false;
+			}
+
+			let defaultServerDatabase = this.model.server.options.database;
+
+			let values: azdata.CategoryValue[];
+			try {
+				values = await this.getDatabaseValues();
+			} catch (error) {
+				// This code is used in case of contained databases when the query will return an error.
+				console.log(error);
+				values = [{ displayName: defaultServerDatabase, name: defaultServerDatabase }];
+				this.databaseDropdown.editable = false;
+			}
+
+			this.model.database = defaultServerDatabase;
+
+			this.databaseDropdown.updateProperties({
+				values: values
+			});
+
+			this.databaseDropdown.value = { displayName: this.model.database, name: this.model.database };
+		} finally {
+			this.databaseDropdown.loading = false;
 		}
-
-		this.model.database = defaultServerDatabase;
-
-		this.databaseDropdown.updateProperties({
-			values: values
-		});
-
-		this.databaseDropdown.value = { displayName: this.model.database, name: this.model.database };
-		this.databaseDropdown.loading = false;
-
 		return true;
 	}

@@ -232,16 +234,29 @@ export class FileConfigPage extends ImportPage {
 		this.fileTextBox = this.view.modelBuilder.inputBox().withProps({
 			required: true,
 			validationErrorMessage: constants.invalidFileLocationError
-		}).withValidation((component) => {
-			return fs.existsSync(component.value);
+		}).withValidation(async (component) => {
+			if (component.value) {
+				try {
+					await fs.access(component.value);
+					return true;
+				} catch (e) {
+					return false;
+				}
+			}
+			return false;
 		}).component();

+		this.fileTextBox.onTextChanged(e => {
+			this.model.newFileSelected = true;
+		});
+
 		this.fileButton = this.view.modelBuilder.button().withProps({
 			label: constants.browseFilesText,
 			secondary: true
 		}).component();

 		this.fileButton.onDidClick(async (click) => {
+			this.model.newFileSelected = true;
 			let fileUris = await vscode.window.showOpenDialog(
 				{
 					canSelectFiles: true,
@@ -324,6 +339,7 @@ export class FileConfigPage extends ImportPage {
 		}).component();

 		this.tableNameTextBox.onTextChanged((tableName) => {
+			this.model.newFileSelected = true;
 			this.model.table = tableName;
 		});

@@ -415,33 +431,6 @@ export class FileConfigPage extends ImportPage {
 		delete this.model.database;
 		delete this.model.schema;
 	}
-
-	// private async populateTableNames(): Promise<boolean> {
-	// 	this.tableNames = [];
-	// 	let databaseName = (<azdata.CategoryValue>this.databaseDropdown.value).name;
-	//
-	// 	if (!databaseName || databaseName.length === 0) {
-	// 		this.tableNames = [];
-	// 		return false;
-	// 	}
-	//
-	// 	let connectionUri = await azdata.connection.getUriForConnection(this.model.server.connectionId);
-	// 	let queryProvider = azdata.dataprotocol.getProvider<azdata.QueryProvider>(this.model.server.providerName, azdata.DataProviderType.QueryProvider);
-	// 	let results: azdata.SimpleExecuteResult;
-	//
-	// 	try {
-	// 		//let query = sqlstring.format('USE ?; SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = \'BASE TABLE\'', [databaseName]);
-	// 		//results = await queryProvider.runQueryAndReturn(connectionUri, query);
-	// 	} catch (e) {
-	// 		return false;
-	// 	}
-	//
-	// 	this.tableNames = results.rows.map(row => {
-	// 		return row[0].displayValue;
-	// 	});
-	//
-	// 	return true;
-	// }
 }


--- a/extensions/import/src/wizard/pages/modifyColumnsPage.ts
+++ b/extensions/import/src/wizard/pages/modifyColumnsPage.ts
@@ -104,7 +104,6 @@ export class ModifyColumnsPage extends ImportPage {
 			});
 		});

-
 		this.form = this.view.modelBuilder.formContainer()
 			.withFormItems(
 				[
@@ -131,15 +130,28 @@ export class ModifyColumnsPage extends ImportPage {
 		await this.populateTable();
 		this.instance.changeNextButtonLabel(constants.importDataText);
 		this.loading.loading = false;
-
+		this.instance.registerNavigationValidator((info) => {
+			return this.table.data && this.table.data.length > 0;
+		});
 		return true;
 	}

 	override async onPageLeave(): Promise<boolean> {
+		await this.emptyTable();
 		this.instance.changeNextButtonLabel(constants.nextText);
+		this.instance.registerNavigationValidator((info) => {
+			return true;
+		});
 		return undefined;
 	}

+	private emptyTable() {
+		this.table.updateProperties({
+			data: [],
+			columns: []
+		});
+	}
+
 	override async cleanup(): Promise<boolean> {
 		delete this.model.proseColumns;
 		this.instance.changeNextButtonLabel(constants.nextText);
@@ -147,12 +159,6 @@ export class ModifyColumnsPage extends ImportPage {
 		return true;
 	}

-	public override setupNavigationValidator() {
-		this.instance.registerNavigationValidator((info) => {
-			return this.table.data && this.table.data.length > 0;
-		});
-	}
-
 	private async populateTable() {
 		let data: ColumnMetadataArray[] = [];

--- a/extensions/import/src/wizard/pages/prosePreviewPage.ts
+++ b/extensions/import/src/wizard/pages/prosePreviewPage.ts
@@ -2,17 +2,17 @@
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the Source EULA. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
-
 import * as azdata from 'azdata';
 import { ImportPage } from '../api/importPage';
 import * as constants from '../../common/constants';
+import { DerivedColumnDialog } from '../../dialogs/derivedColumnDialog';
+import * as vscode from 'vscode';

 export class ProsePreviewPage extends ImportPage {

 	private _table: azdata.TableComponent;
 	private _loading: azdata.LoadingComponent;
 	private _form: azdata.FormContainer;
-	private _refresh: azdata.ButtonComponent;
 	private _resultTextComponent: azdata.TextComponent;
 	private _isSuccess: boolean;

@@ -40,14 +40,6 @@ export class ProsePreviewPage extends ImportPage {
 		this._form = form;
 	}

-	public get refresh(): azdata.ButtonComponent {
-		return this._refresh;
-	}
-
-	public set refresh(refresh: azdata.ButtonComponent) {
-		this._refresh = refresh;
-	}
-
 	public get resultTextComponent(): azdata.TextComponent {
 		return this._resultTextComponent;
 	}
@@ -70,14 +62,22 @@ export class ProsePreviewPage extends ImportPage {
 			columns: undefined,
 			forceFitColumns: azdata.ColumnSizingMode.DataFit
 		}).component();
-		this.refresh = this.view.modelBuilder.button().withProps({
-			label: constants.refreshText,
-			isFile: false,
-			secondary: true
-		}).component();

-		this.refresh.onDidClick(async () => {
-			await this.onPageEnter();
+		this.instance.createDerivedColumnButton.onClick(async (e) => {
+			const derivedColumnDialog = new DerivedColumnDialog(this.model, this.provider);
+			const response = await derivedColumnDialog.createDerivedColumn();
+			if (response) {
+				this.model.proseColumns.push({
+					columnName: response.derivedColumnName,
+					dataType: 'nvarchar(MAX)',
+					primaryKey: false,
+					nullable: true
+				});
+				response.derivedColumnDataPreview.forEach((v, i) => {
+					this.model.proseDataPreview[i].push(v);
+				});
+				this.populateTable(this.model.proseDataPreview, this.model.proseColumns.map(c => c.columnName));
+			}
 		});

 		this.loading = this.view.modelBuilder.loadingComponent().component();
@@ -94,9 +94,9 @@ export class ProsePreviewPage extends ImportPage {
 			},
 			{
 				component: this.table,
-				title: '',
-				actions: [this.refresh]
+				title: ''
 			}
+
 		]).component();

 		this.loading.component = this.form;
@@ -107,22 +107,30 @@ export class ProsePreviewPage extends ImportPage {
 	}

 	async onPageEnter(): Promise<boolean> {
-		this.loading.loading = true;
 		let proseResult: boolean;
 		let error: string;
-		try {
-			proseResult = await this.handleProse();
-		} catch (ex) {
-			error = ex.toString();
+		const enablePreviewFeatures = vscode.workspace.getConfiguration('workbench').get('enablePreviewFeatures');
+		if (this.model.newFileSelected) {
+			this.loading.loading = true;
+			try {
+				proseResult = await this.learnFile();
+			} catch (ex) {
+				error = ex.toString();
+				this.instance.wizard.message = {
+					level: azdata.window.MessageLevel.Error,
+					text: error
+				};
+			}
+			this.model.newFileSelected = false;
+			this.loading.loading = false;
 		}
-
-		this.loading.loading = false;
-		if (proseResult) {
+		if (!this.model.newFileSelected || proseResult) {
 			await this.populateTable(this.model.proseDataPreview, this.model.proseColumns.map(c => c.columnName));
 			this.isSuccess = true;
 			if (this.form) {
 				this.resultTextComponent.value = constants.successTitleText;
 			}
+			this.instance.createDerivedColumnButton.hidden = !enablePreviewFeatures;
 			return true;
 		} else {
 			await this.populateTable([], []);
@@ -135,7 +143,7 @@ export class ProsePreviewPage extends ImportPage {
 	}

 	override async onPageLeave(): Promise<boolean> {
-		await this.emptyTable();
+		this.instance.createDerivedColumnButton.hidden = true;
 		return true;
 	}

@@ -156,7 +164,7 @@ export class ProsePreviewPage extends ImportPage {
 		});
 	}

-	private async handleProse(): Promise<boolean> {
+	private async learnFile(): Promise<boolean> {
 		const response = await this.provider.sendPROSEDiscoveryRequest({
 			filePath: this.model.filePath,
 			tableName: this.model.table,
@@ -170,6 +178,7 @@ export class ProsePreviewPage extends ImportPage {
 		}

 		this.model.proseColumns = [];
+		this.model.originalProseColumns = [];
 		if (response.columnInfo) {
 			response.columnInfo.forEach((column) => {
 				this.model.proseColumns.push({
@@ -178,6 +187,12 @@ export class ProsePreviewPage extends ImportPage {
 					primaryKey: false,
 					nullable: column.isNullable
 				});
+				this.model.originalProseColumns.push({
+					columnName: column.name,
+					dataType: column.sqlType,
+					primaryKey: false,
+					nullable: column.isNullable
+				});
 			});
 			return true;
 		}
@@ -203,8 +218,4 @@ export class ProsePreviewPage extends ImportPage {
 			width: '700',
 		});
 	}
-
-	private async emptyTable() {
-		this.table.updateProperties([]);
-	}
 }
--- a/extensions/import/src/wizard/pages/summaryPage.ts
+++ b/extensions/import/src/wizard/pages/summaryPage.ts
@@ -50,7 +50,12 @@ export class SummaryPage extends ImportPage {

 	async start(): Promise<boolean> {
 		this.table = this.view.modelBuilder.table().component();
-		this.statusText = this.view.modelBuilder.text().component();
+		this.statusText = this.view.modelBuilder.text().withProps({
+			CSSStyles: {
+				'user-select': 'text',
+				'font-size': '13px'
+			}
+		}).component();
 		this.loading = this.view.modelBuilder.loadingComponent().withItem(this.statusText).component();

 		this.form = this.view.modelBuilder.formContainer().withFormItems(