.sqlproj and datasources.json file parsing (#8921)

* Checkpoint

* Adding mock contents for tree

* added open sqlproj dialog

* reading files from directory

* Added directory traversal

* Adding tree sorting by folder vs file and label

* Improved auto-unfolding of tree based on node type

* replacing fs with fs.promise alias

* PR feedback

* added activation event for when workspace contains sqlproj files

* Returning after displaying error

* Fixing linter errors

* Reworked tree

* Fixing missing grandchildren

* Correcting tree URI construction

* Refactoring to isolate tree item responsibilities from data model responsibilities

* project file parsing

* constructing tree from project files rather than filesystem

* Fixing double-initialization

* Changing projectEntry to take enum for file type

* Correct node type for project item

* Parsing datasources.json

* Child nodes for sql data source

* Localizing strings

* Checkpoint

* Adding mock contents for tree

* added open sqlproj dialog

* reading files from directory

* Added directory traversal

* Adding tree sorting by folder vs file and label

* Improved auto-unfolding of tree based on node type

* replacing fs with fs.promise alias

* PR feedback

* added activation event for when workspace contains sqlproj files

* Returning after displaying error

* Fixing linter errors

* Reworked tree

* Fixing missing grandchildren

* Correcting tree URI construction

* Refactoring to isolate tree item responsibilities from data model responsibilities

* project file parsing

* constructing tree from project files rather than filesystem

* Fixing double-initialization

* Changing projectEntry to take enum for file type

* Correct node type for project item

* Parsing datasources.json

* Child nodes for sql data source

* Localizing strings

* missed file in merge

* changed extension method to helper

* cleanup

* Adding docstrings

extensions/sql-database-projects/src/models/dataSources/dataSourceJson.ts

@@ -0,0 +1,26 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/**
+ * JSON format for datasources.json
+ */
+interface DataSourceFileJson {
+	version: string;
+	datasources: DataSourceJson[];
+}
+
+/**
+ * JSON format for a datasource entry in datasources.json
+ */
+interface DataSourceJson {
+	name: string;
+	type: string;
+	version: string;
+
+	/**
+	 * contents for concrete datasource implementation
+	 */
+	data: string;
+}

extensions/sql-database-projects/src/models/dataSources/dataSources.ts

@@ -0,0 +1,71 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { promises as fs } from 'fs';
+import * as constants from '../../common/constants';
+import { SqlConnectionDataSource } from './sqlConnectionStringSource';
+
+/**
+ * Abstract class for a datasource in a project
+ */
+export abstract class DataSource {
+	public name: string;
+	public abstract get type(): string;
+	public abstract get friendlyName(): string;
+
+	constructor(name: string) {
+		this.name = name;
+	}
+}
+
+/**
+ * parses the specified file to load DataSource objects
+ */
+export async function load(dataSourcesFilePath: string): Promise<DataSource[]> {
+	let fileContents;
+
+	try {
+		fileContents = await fs.readFile(dataSourcesFilePath);
+	}
+	catch (err) {
+		throw new Error(constants.noDataSourcesFile);
+	}
+
+	const rawJsonContents = JSON.parse(fileContents.toString());
+
+	if (rawJsonContents.version === undefined) {
+		throw new Error(constants.missingVersion);
+	}
+
+	const output: DataSource[] = [];
+
+	// TODO: do we have a construct for parsing version numbers?
+	switch (rawJsonContents.version) {
+		case '0.0.0':
+			const dataSources: DataSourceFileJson = rawJsonContents as DataSourceFileJson;
+
+			for (const source of dataSources.datasources) {
+				output.push(createDataSource(source));
+			}
+
+			break;
+		default:
+			throw new Error(constants.unrecognizedDataSourcesVersion + rawJsonContents.version);
+	}
+
+	return output;
+}
+
+/**
+ * Creates DataSource object from JSON
+ */
+function createDataSource(json: DataSourceJson): DataSource {
+	switch (json.type) {
+		case SqlConnectionDataSource.type:
+			return SqlConnectionDataSource.fromJson(json);
+		default:
+			throw new Error(constants.unknownDataSourceType + json.type);
+	}
+}

extensions/sql-database-projects/src/models/dataSources/sqlConnectionStringSource.ts

@@ -0,0 +1,55 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { DataSource } from './dataSources';
+import * as constants from '../../common/constants';
+
+/**
+ * Contains information about a SQL connection string data source`
+ */
+export class SqlConnectionDataSource extends DataSource {
+	readonly connectionString: string;
+	readonly connectionStringComponents: { [id: string]: string } = {};
+
+	public static get type() {
+		return 'sql_connection_string';
+	}
+
+	public get type(): string {
+		return SqlConnectionDataSource.type;
+	}
+
+	public get friendlyName(): string {
+		return constants.sqlConnectionStringFriendly;
+	}
+
+	constructor(name: string, connectionString: string) {
+		super(name);
+
+		// TODO: do we have a common construct for connection strings?
+		this.connectionString = connectionString;
+
+		for (const component of this.connectionString.split(';')) {
+			const split = component.split('=');
+
+			if (split.length !== 2) {
+				throw new Error(constants.invalidSqlConnectionString);
+			}
+
+			this.connectionStringComponents[split[0]] = split[1];
+		}
+	}
+
+	public static fromJson(json: DataSourceJson): SqlConnectionDataSource {
+		return new SqlConnectionDataSource(json.name, (json.data as unknown as SqlConnectionDataSourceJson).connectionString);
+	}
+}
+
+/**
+ * JSON structure for a SQL connection string data source
+ */
+interface SqlConnectionDataSourceJson {
+	connectionString: string;
+}

extensions/sql-database-projects/src/models/project.ts

@@ -0,0 +1,91 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as xml2js from 'xml2js';
+import * as path from 'path';
+import { promises as fs } from 'fs';
+import { DataSource } from './dataSources/dataSources';
+
+/**
+ * Class representing a Project, and providing functions for operating on it
+ */
+export class Project {
+	public projectFile: string;
+	public files: ProjectEntry[] = [];
+	public dataSources: DataSource[] = [];
+
+	constructor(projectFilePath: string) {
+		this.projectFile = projectFilePath;
+	}
+
+	/**
+	 * Reads the project setting and contents from the file
+	 */
+	public async readProjFile() {
+		let projFileContents = await fs.readFile(this.projectFile);
+
+		const parser = new xml2js.Parser({
+			explicitArray: true,
+			explicitCharkey: false,
+			explicitRoot: false
+		});
+
+		let result;
+
+		try {
+			result = await parser.parseStringPromise(projFileContents.toString());
+		}
+		catch (err) {
+			vscode.window.showErrorMessage(err);
+			return;
+		}
+
+		// find all folders and files to include
+
+		for (const itemGroup of result['ItemGroup']) {
+			if (itemGroup['Build'] !== undefined) {
+				for (const fileEntry of itemGroup['Build']) {
+					this.files.push(this.createProjectEntry(fileEntry.$['Include'], EntryType.File));
+				}
+			}
+
+			if (itemGroup['Folder'] !== undefined) {
+				for (const folderEntry of itemGroup['Folder']) {
+					this.files.push(this.createProjectEntry(folderEntry.$['Include'], EntryType.Folder));
+				}
+			}
+		}
+	}
+
+	private createProjectEntry(relativePath: string, entryType: EntryType): ProjectEntry {
+		return new ProjectEntry(vscode.Uri.file(path.join(this.projectFile, relativePath)), entryType);
+	}
+}
+
+/**
+ * Represents an entry in a project file
+ */
+export class ProjectEntry {
+	/**
+	 * Absolute file system URI
+	 */
+	uri: vscode.Uri;
+	type: EntryType;
+
+	constructor(uri: vscode.Uri, type: EntryType) {
+		this.uri = uri;
+		this.type = type;
+	}
+
+	public toString(): string {
+		return this.uri.path;
+	}
+}
+
+export enum EntryType {
+	File,
+	Folder
+}

extensions/sql-database-projects/src/models/tree/baseTreeItem.ts

@@ -0,0 +1,54 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as path from 'path';
+
+/**
+ * Base class for an item that appears in the ADS project tree
+ */
+export abstract class BaseProjectTreeItem {
+	uri: vscode.Uri;
+	parent?: BaseProjectTreeItem;
+
+	constructor(uri: vscode.Uri, parent?: BaseProjectTreeItem) {
+		this.uri = uri;
+		this.parent = parent;
+	}
+
+	abstract get children(): BaseProjectTreeItem[];
+
+	abstract get treeItem(): vscode.TreeItem;
+
+	public get root() {
+		let node: BaseProjectTreeItem = this;
+
+		while (node.parent !== undefined) {
+			node = node.parent;
+		}
+
+		return node;
+	}
+}
+
+/**
+ * Leaf tree item that just displays text for messaging purposes
+ */
+export class MessageTreeItem extends BaseProjectTreeItem {
+	private message: string;
+
+	constructor(message: string, parent?: BaseProjectTreeItem) {
+		super(vscode.Uri.file(path.join(parent?.uri.path ?? 'Message', message)), parent);
+		this.message = message;
+	}
+
+	public get children(): BaseProjectTreeItem[] {
+		return [];
+	}
+
+	public get treeItem(): vscode.TreeItem {
+		return new vscode.TreeItem(this.message, vscode.TreeItemCollapsibleState.None);
+	}
+}

extensions/sql-database-projects/src/models/tree/dataSourceTreeItem.ts

@@ -0,0 +1,85 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as path from 'path';
+import { BaseProjectTreeItem, MessageTreeItem } from './baseTreeItem';
+import * as constants from '../../common/constants';
+import { ProjectRootTreeItem } from './projectTreeItem';
+import { DataSource } from '../dataSources/dataSources';
+import { SqlConnectionDataSource } from '../dataSources/sqlConnectionStringSource';
+
+/**
+ * Folder for containing DataSource nodes in the tree
+ */
+export class DataSourcesTreeItem extends BaseProjectTreeItem {
+	private dataSources: DataSourceTreeItem[] = [];
+
+	constructor(project: ProjectRootTreeItem) {
+		super(vscode.Uri.file(path.join(project.uri.path, constants.dataSourcesNodeName)), project);
+
+		this.construct();
+	}
+
+	private construct() {
+		for (const dataSource of (this.parent as ProjectRootTreeItem).project.dataSources) {
+			this.dataSources.push(constructDataSourceTreeItem(dataSource, this));
+		}
+	}
+
+	public get children(): BaseProjectTreeItem[] {
+		return this.dataSources;
+	}
+
+	public get treeItem(): vscode.TreeItem {
+		return new vscode.TreeItem(this.uri, vscode.TreeItemCollapsibleState.Collapsed);
+	}
+}
+
+abstract class DataSourceTreeItem extends BaseProjectTreeItem { }
+
+/**
+ * Tree item representing a SQL connection string data source
+ */
+export class SqlConnectionDataSourceTreeItem extends DataSourceTreeItem {
+	private dataSource: SqlConnectionDataSource;
+
+	constructor(dataSource: SqlConnectionDataSource, dataSourcesNode: DataSourcesTreeItem) {
+		super(vscode.Uri.file(path.join(dataSourcesNode.uri.path, dataSource.name)), dataSourcesNode);
+		this.dataSource = dataSource;
+	}
+
+	public get treeItem(): vscode.TreeItem {
+		let item = new vscode.TreeItem(this.uri, vscode.TreeItemCollapsibleState.Collapsed);
+		item.label = `${this.dataSource.name} (${this.dataSource.friendlyName})`;
+
+		return item;
+	}
+
+	/**
+	 * SQL connection string components, displayed as key-value pairs
+	 */
+	public get children(): BaseProjectTreeItem[] {
+		const result: MessageTreeItem[] = [];
+
+		for (const comp of Object.keys(this.dataSource.connectionStringComponents).sort()) {
+			result.push(new MessageTreeItem(`${comp}: ${this.dataSource.connectionStringComponents[comp]}`, this));
+		}
+
+		return result;
+	}
+}
+
+/**
+ * Constructs a new TreeItem for the specific given DataSource type
+ */
+export function constructDataSourceTreeItem(dataSource: DataSource, dataSourcesNode: DataSourcesTreeItem): DataSourceTreeItem {
+	switch (dataSource.type) {
+		case SqlConnectionDataSource.type:
+			return new SqlConnectionDataSourceTreeItem(dataSource as SqlConnectionDataSource, dataSourcesNode);
+		default:
+			throw new Error(constants.unknownDataSourceType + dataSource.type); // TODO: elegant handling of unknown dataSource type instead of failure
+	}
+}

extensions/sql-database-projects/src/models/tree/fileFolderTreeItem.ts

@@ -0,0 +1,73 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as path from 'path';
+import { BaseProjectTreeItem } from './baseTreeItem';
+import { ProjectRootTreeItem } from './projectTreeItem';
+import { Project } from '../project';
+
+/**
+ * Node representing a folder in a project
+ */
+export class FolderNode extends BaseProjectTreeItem {
+	public fileChildren: { [childName: string]: (FolderNode | FileNode) } = {};
+	public fileSystemUri: vscode.Uri;
+
+	constructor(folderPath: vscode.Uri, parent: FolderNode | ProjectRootTreeItem) {
+		super(fsPathToProjectUri(folderPath, parent.root as ProjectRootTreeItem), parent);
+		this.fileSystemUri = folderPath;
+	}
+
+	public get children(): BaseProjectTreeItem[] {
+		return Object.values(this.fileChildren).sort();
+	}
+
+	public get treeItem(): vscode.TreeItem {
+		return new vscode.TreeItem(this.uri, vscode.TreeItemCollapsibleState.Expanded);
+	}
+
+	public get project(): Project {
+		return (<FolderNode | ProjectRootTreeItem>this.parent).project;
+	}
+}
+
+/**
+ * Node representing a file in a project
+ */
+export class FileNode extends BaseProjectTreeItem {
+	public fileSystemUri: vscode.Uri;
+
+	constructor(filePath: vscode.Uri, parent: FolderNode | ProjectRootTreeItem) {
+		super(fsPathToProjectUri(filePath, parent.root as ProjectRootTreeItem), parent);
+		this.fileSystemUri = filePath;
+	}
+
+	public get children(): BaseProjectTreeItem[] {
+		return [];
+	}
+
+	public get treeItem(): vscode.TreeItem {
+		return new vscode.TreeItem(this.uri, vscode.TreeItemCollapsibleState.None);
+	}
+}
+
+/**
+ * Converts a full filesystem URI to a project-relative URI that's compatible with the project tree
+ */
+function fsPathToProjectUri(fileSystemUri: vscode.Uri, projectNode: ProjectRootTreeItem): vscode.Uri {
+	const projBaseDir = path.dirname(projectNode.project.projectFile);
+	let localUri = '';
+
+	if (fileSystemUri.fsPath.startsWith(projBaseDir)) {
+		localUri = fileSystemUri.fsPath.substring(projBaseDir.length);
+	}
+	else {
+		vscode.window.showErrorMessage('Project pointing to file outside of directory');
+		throw new Error('Project pointing to file outside of directory');
+	}
+
+	return vscode.Uri.file(path.join(projectNode.uri.path, localUri));
+}

extensions/sql-database-projects/src/models/tree/projectTreeItem.ts

@@ -0,0 +1,100 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as path from 'path';
+import { DataSourcesTreeItem } from './dataSourceTreeItem';
+import { BaseProjectTreeItem } from './baseTreeItem';
+import * as fileTree from './fileFolderTreeItem';
+import { Project, ProjectEntry, EntryType } from '../project';
+import * as utils from '../../common/utils';
+
+/**
+ * TreeNode root that represents an entire project
+ */
+export class ProjectRootTreeItem extends BaseProjectTreeItem {
+	dataSourceNode: DataSourcesTreeItem;
+	fileChildren: { [childName: string]: (fileTree.FolderNode | fileTree.FileNode) } = {};
+	project: Project;
+
+	constructor(project: Project) {
+		super(vscode.Uri.parse(path.basename(project.projectFile)), undefined);
+
+		this.project = project;
+		this.dataSourceNode = new DataSourcesTreeItem(this);
+
+		this.construct();
+	}
+
+	public get children(): BaseProjectTreeItem[] {
+		const output: BaseProjectTreeItem[] = [];
+		output.push(this.dataSourceNode);
+
+		// sort children so that folders come first, then alphabetical
+		const sortedChildren = Object.values(this.fileChildren).sort((a: (fileTree.FolderNode | fileTree.FileNode), b: (fileTree.FolderNode | fileTree.FileNode)) => {
+			if (a instanceof fileTree.FolderNode && !(b instanceof fileTree.FolderNode)) { return -1; }
+			else if (!(a instanceof fileTree.FolderNode) && b instanceof fileTree.FolderNode) { return 1; }
+			else { return a.uri.fsPath.localeCompare(b.uri.fsPath); }
+		});
+
+		return output.concat(sortedChildren);
+	}
+
+	public get treeItem(): vscode.TreeItem {
+		return new vscode.TreeItem(this.uri, vscode.TreeItemCollapsibleState.Expanded);
+	}
+
+	/**
+	 * Processes the list of files in a project file to constructs the tree
+	 */
+	private construct() {
+		for (const entry of this.project.files) {
+			const parentNode = this.getEntryParentNode(entry);
+
+			let newNode: fileTree.FolderNode | fileTree.FileNode;
+
+			switch (entry.type) {
+				case EntryType.File:
+					newNode = new fileTree.FileNode(entry.uri, parentNode);
+					break;
+				case EntryType.Folder:
+					newNode = new fileTree.FolderNode(entry.uri, parentNode);
+					break;
+				default:
+					throw new Error(`Unknown EntryType: '${entry.type}'`);
+			}
+
+			parentNode.fileChildren[path.basename(entry.uri.path)] = newNode;
+		}
+	}
+
+	/**
+	 * Gets the immediate parent tree node for an entry in a project file
+	 */
+	private getEntryParentNode(entry: ProjectEntry): fileTree.FolderNode | ProjectRootTreeItem {
+		const relativePathParts = utils.trimChars(utils.trimUri(vscode.Uri.file(this.project.projectFile), entry.uri), '/').split('/').slice(0, -1); // remove the last part because we only care about the parent
+
+		if (relativePathParts.length === 0) {
+			return this; // if nothing left after trimming the entry itself, must been root
+		}
+
+		let current: fileTree.FolderNode | ProjectRootTreeItem = this;
+
+		for (const part of relativePathParts) {
+			if (current.fileChildren[part] === undefined) {
+				current.fileChildren[part] = new fileTree.FolderNode(vscode.Uri.file(path.join(path.dirname(this.project.projectFile), part)), current);
+			}
+
+			if (current.fileChildren[part] instanceof fileTree.FileNode) {
+				return current;
+			}
+			else {
+				current = current.fileChildren[part] as fileTree.FolderNode | ProjectRootTreeItem;
+			}
+		}
+
+		return current;
+	}
+}