Feature/outer paths for project (#11445)

* allow relative paths in project file outside of project folder

* Adding some tests

* Adding error string to loc strings

* Fixed test

* fix error message

* PR comments and some more fixes

extensions/sql-database-projects/src/common/constants.ts

@@ -108,6 +108,7 @@ export const invalidDataSchemaProvider = localize('invalidDataSchemaProvider', "
 export const invalidDatabaseReference = localize('invalidDatabaseReference', "Invalid database reference in .sqlproj file");
 export const databaseSelectionRequired = localize('databaseSelectionRequired', "Database selection is required to import a project");
 export const databaseReferenceAlreadyExists = localize('databaseReferenceAlreadyExists', "A reference to this database already exists in this project");
+export const ousiderFolderPath = localize('outsideFolderPath', "Items with absolute path outside project folder are not supported. Please make sure the paths in the project file are relative to project folder.");
 export function projectAlreadyOpened(path: string) { return localize('projectAlreadyOpened', "Project '{0}' is already opened.", path); }
 export function projectAlreadyExists(name: string, path: string) { return localize('projectAlreadyExists', "A project named {0} already exists in {1}.", name, path); }
 export function noFileExist(fileName: string) { return localize('noFileExist', "File {0} doesn't exist", fileName); }
@@ -162,6 +163,7 @@ export const SuppressMissingDependenciesErrors = 'SuppressMissingDependenciesErr
 export const DatabaseVariableLiteralValue = 'DatabaseVariableLiteralValue';
 export const DSP = 'DSP';
 export const Properties = 'Properties';
+export const RelativeOuterPath = '..';
 
 // SqlProj File targets
 export const NetCoreTargets = '$(NETCoreTargetsPath)\\Microsoft.Data.Tools.Schema.SqlTasks.targets';

extensions/sql-database-projects/src/common/utils.ts

@@ -7,6 +7,7 @@ import * as vscode from 'vscode';
 import * as os from 'os';
 import * as constants from './constants';
 import { promises as fs } from 'fs';
+import * as path from 'path';
 
 /**
  * Consolidates on the error message string
@@ -19,7 +20,8 @@ export function getErrorMessage(error: any): string {
 
 /**
  * removes any leading portion shared between the two URIs from outerUri.
- * e.g. [@param innerUri: 'this\is'; @param outerUri: '\this\is\my\path'] => 'my\path'
+ * e.g. [@param innerUri: 'this\is'; @param outerUri: '\this\is\my\path'] => 'my\path' OR
+ * e.g. [@param innerUri: 'this\was'; @param outerUri: '\this\is\my\path'] => '..\my\path'
  * @param innerUri the URI that will be cut away from the outer URI
  * @param outerUri the URI that will have any shared beginning portion removed
  */
@@ -27,11 +29,22 @@ export function trimUri(innerUri: vscode.Uri, outerUri: vscode.Uri): string {
 	let innerParts = innerUri.path.split('/');
 	let outerParts = outerUri.path.split('/');
 
+	if (path.isAbsolute(outerUri.path)
+		&& innerParts.length > 0 && outerParts.length > 0
+		&& innerParts[0].toLowerCase() !== outerParts[0].toLowerCase()) {
+		throw new Error(constants.ousiderFolderPath);
+	}
+
 	while (innerParts.length > 0 && outerParts.length > 0 && innerParts[0].toLocaleLowerCase() === outerParts[0].toLocaleLowerCase()) {
 		innerParts = innerParts.slice(1);
 		outerParts = outerParts.slice(1);
 	}
 
+	while (innerParts.length > 1) {
+		outerParts.unshift(constants.RelativeOuterPath);
+		innerParts = innerParts.slice(1);
+	}
+
 	return outerParts.join('/');
 }
 
@@ -68,24 +81,24 @@ export async function exists(path: string): Promise<boolean> {
  * get quoted path to be used in any commandline argument
  * @param filePath
  */
-export function getSafePath(filePath: string): string {
+export function getQuotedPath(filePath: string): string {
 	return (os.platform() === 'win32') ?
-		getSafeWindowsPath(filePath) :
-		getSafeNonWindowsPath(filePath);
+		getQuotedWindowsPath(filePath) :
+		getQuotedNonWindowsPath(filePath);
 }
 
 /**
- * ensure that path with spaces are handles correctly
+ * ensure that path with spaces are handles correctly (return quoted path)
  */
-export function getSafeWindowsPath(filePath: string): string {
+function getQuotedWindowsPath(filePath: string): string {
 	filePath = filePath.split('\\').join('\\\\').split('"').join('');
 	return '"' + filePath + '"';
 }
 
 /**
- * ensure that path with spaces are handles correctly
+ * ensure that path with spaces are handles correctly (return quoted path)
  */
-export function getSafeNonWindowsPath(filePath: string): string {
+function getQuotedNonWindowsPath(filePath: string): string {
 	filePath = filePath.split('\\').join('/').split('"').join('');
 	return '"' + filePath + '"';
 }

extensions/sql-database-projects/src/controllers/projectController.ts

@@ -380,6 +380,13 @@ export class ProjectsController {
 	}
 
 	private getProjectEntry(project: Project, context: BaseProjectTreeItem): ProjectEntry | undefined {
+		const root = context.root as ProjectRootTreeItem;
+		const fileOrFolder = context as FileNode ? context as FileNode : context as FolderNode;
+
+		if (root && fileOrFolder) {
+			// use relative path and not tree paths for files and folder
+			return project.files.find(x => utils.getPlatformSafeFileEntryPath(x.relativePath) === utils.getPlatformSafeFileEntryPath(utils.trimUri(root.fileSystemUri, fileOrFolder.fileSystemUri)));
+		}
 		return project.files.find(x => utils.getPlatformSafeFileEntryPath(x.relativePath) === utils.getPlatformSafeFileEntryPath(utils.trimUri(context.root.uri, context.uri)));
 	}
 

extensions/sql-database-projects/src/models/tree/fileFolderTreeItem.ts

@@ -9,6 +9,7 @@ import { BaseProjectTreeItem } from './baseTreeItem';
 import { ProjectRootTreeItem } from './projectTreeItem';
 import { Project } from '../project';
 import { DatabaseProjectItemType } from '../../common/constants';
+import * as utils from '../../common/utils';
 
 /**
  * Node representing a folder in a project
@@ -44,7 +45,7 @@ export class FileNode extends BaseProjectTreeItem {
 	public fileSystemUri: vscode.Uri;
 
 	constructor(filePath: vscode.Uri, parent: FolderNode | ProjectRootTreeItem) {
-		super(fsPathToProjectUri(filePath, parent.root as ProjectRootTreeItem), parent);
+		super(fsPathToProjectUri(filePath, parent.root as ProjectRootTreeItem, true), parent);
 		this.fileSystemUri = filePath;
 	}
 
@@ -87,15 +88,18 @@ export function sortFileFolderNodes(a: (FolderNode | FileNode), b: (FolderNode |
 /**
  * Converts a full filesystem URI to a project-relative URI that's compatible with the project tree
  */
-function fsPathToProjectUri(fileSystemUri: vscode.Uri, projectNode: ProjectRootTreeItem): vscode.Uri {
+function fsPathToProjectUri(fileSystemUri: vscode.Uri, projectNode: ProjectRootTreeItem, isFile?: boolean): vscode.Uri {
 	const projBaseDir = projectNode.project.projectFolderPath;
 	let localUri = '';
 
 	if (fileSystemUri.fsPath.startsWith(projBaseDir)) {
 		localUri = fileSystemUri.fsPath.substring(projBaseDir.length);
 	}
-	else {
-		throw new Error(`Project (${projBaseDir}) pointing to file outside of directory (${fileSystemUri.fsPath})`);
+	else if (isFile) {
+		// if file is outside the folder add add at top level in tree
+		// this is not true for folders otherwise the outside files will not be directly inside the top level
+		let parts = utils.getPlatformSafeFileEntryPath(fileSystemUri.fsPath).split('/');
+		localUri = parts[parts.length - 1];
 	}
 
 	return vscode.Uri.file(path.join(projectNode.uri.path, localUri));

extensions/sql-database-projects/src/models/tree/projectTreeItem.ts

@@ -11,7 +11,7 @@ import * as fileTree from './fileFolderTreeItem';
 import { Project, ProjectEntry, EntryType } from '../project';
 import * as utils from '../../common/utils';
 import { DatabaseReferencesTreeItem } from './databaseReferencesTreeItem';
-import { DatabaseProjectItemType } from '../../common/constants';
+import { DatabaseProjectItemType, RelativeOuterPath } from '../../common/constants';
 
 /**
  * TreeNode root that represents an entire project
@@ -21,11 +21,13 @@ export class ProjectRootTreeItem extends BaseProjectTreeItem {
 	databaseReferencesNode: DatabaseReferencesTreeItem;
 	fileChildren: { [childName: string]: (fileTree.FolderNode | fileTree.FileNode) } = {};
 	project: Project;
+	fileSystemUri: vscode.Uri;
 
 	constructor(project: Project) {
 		super(vscode.Uri.parse(path.basename(project.projectFilePath)), undefined);
 
 		this.project = project;
+		this.fileSystemUri = vscode.Uri.file(project.projectFilePath);
 		this.dataSourceNode = new DataSourcesTreeItem(this);
 		this.databaseReferencesNode = new DatabaseReferencesTreeItem(this);
 
@@ -51,6 +53,10 @@ export class ProjectRootTreeItem extends BaseProjectTreeItem {
 	 */
 	private construct() {
 		for (const entry of this.project.files) {
+			if (entry.type !== EntryType.File && entry.relativePath.startsWith(RelativeOuterPath)) {
+				continue;
+			}
+
 			const parentNode = this.getEntryParentNode(entry);
 
 			if (Object.keys(parentNode.fileChildren).includes(path.basename(entry.fsUri.path))) {
@@ -84,6 +90,10 @@ export class ProjectRootTreeItem extends BaseProjectTreeItem {
 			return this; // if nothing left after trimming the entry itself, must been root
 		}
 
+		if (relativePathParts[0] === RelativeOuterPath) {
+			return this;
+		}
+
 		let current: fileTree.FolderNode | ProjectRootTreeItem = this;
 
 		for (const part of relativePathParts) {

extensions/sql-database-projects/src/test/baselines/openSqlProjectBaseline.xml

@@ -68,6 +68,7 @@
     <Build Include="Tables\Users.sql" />
     <Build Include="Tables\Action History.sql" />
     <Build Include="Views\Maintenance\Database Performance.sql" />
+    <Build Include="..\Test\Test.sql" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="Views\User" />

extensions/sql-database-projects/src/test/netCoreTool.test.ts

@@ -9,7 +9,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as vscode from 'vscode';
 import { NetCoreTool, DBProjectConfigurationKey, NetCoreInstallLocationKey, NextCoreNonWindowsDefaultPath } from '../tools/netcoreTool';
-import { getSafePath } from '../common/utils';
+import { getQuotedPath } from '../common/utils';
 import { isNullOrUndefined } from 'util';
 import { generateTestFolderPath } from './testUtils';
 
@@ -52,7 +52,7 @@ describe.skip('NetCoreTool: Net core tests', function (): void {
 		const outputChannel = vscode.window.createOutputChannel('db project test');
 
 		try {
-			await netcoreTool.runStreamedCommand('echo test > ' + getSafePath(dummyFile), outputChannel, undefined);
+			await netcoreTool.runStreamedCommand('echo test > ' + getQuotedPath(dummyFile), outputChannel, undefined);
 			const text = await fs.promises.readFile(dummyFile);
 			should(text.toString().trim()).equal('test');
 		}

extensions/sql-database-projects/src/test/project.test.ts

@@ -29,11 +29,12 @@ describe('Project: sqlproj content operations', function (): void {
 		const project: Project = await Project.openProject(projFilePath);
 
 		// Files and folders
-		should(project.files.filter(f => f.type === EntryType.File).length).equal(4);
+		should(project.files.filter(f => f.type === EntryType.File).length).equal(5);
 		should(project.files.filter(f => f.type === EntryType.Folder).length).equal(4);
 
 		should(project.files.find(f => f.type === EntryType.Folder && f.relativePath === 'Views\\User')).not.equal(undefined); // mixed ItemGroup folder
 		should(project.files.find(f => f.type === EntryType.File && f.relativePath === 'Views\\User\\Profile.sql')).not.equal(undefined); // mixed ItemGroup file
+		should(project.files.find(f => f.type === EntryType.File && f.relativePath === '..\\Test\\Test.sql')).not.equal(undefined); // mixed ItemGroup file
 
 		// SqlCmdVariables
 		should(Object.keys(project.sqlCmdVariables).length).equal(2);

extensions/sql-database-projects/src/test/projectController.test.ts

@@ -24,7 +24,7 @@ import { PublishDatabaseDialog } from '../dialogs/publishDatabaseDialog';
 import { IPublishSettings, IGenerateScriptSettings } from '../models/IPublishSettings';
 import { exists } from '../common/utils';
 import { ProjectRootTreeItem } from '../models/tree/projectTreeItem';
-import { FolderNode } from '../models/tree/fileFolderTreeItem';
+import { FolderNode, FileNode } from '../models/tree/fileFolderTreeItem';
 import { BaseProjectTreeItem } from '../models/tree/baseTreeItem';
 
 let testContext: TestContext;
@@ -46,9 +46,12 @@ const mockConnectionProfile: azdata.IConnectionProfile = {
 	options: undefined as any
 };
 
+describe('ProjectsController', function (): void {
+	before(async function (): Promise<void> {
+		await templates.loadTemplates(path.join(__dirname, '..', '..', 'resources', 'templates'));
+		await baselines.loadBaselines();
+	});
 
-
-describe ('ProjectsController', function(): void {
 	beforeEach(function (): void {
 		testContext = createContext();
 	});
@@ -58,11 +61,6 @@ describe ('ProjectsController', function(): void {
 	});
 
 	describe('project controller operations', function (): void {
-		before(async function (): Promise<void> {
-			await templates.loadTemplates(path.join(__dirname, '..', '..', 'resources', 'templates'));
-			await baselines.loadBaselines();
-		});
-
 		describe('Project file operations and prompting', function (): void {
 			it('Should create new sqlproj file with correct values', async function (): Promise<void> {
 				const projController = new ProjectsController(new SqlDatabaseProjectTreeViewProvider());
@@ -85,7 +83,7 @@ describe ('ProjectsController', function(): void {
 
 				const project = await projController.openProject(vscode.Uri.file(sqlProjPath));
 
-				should(project.files.length).equal(8); // detailed sqlproj tests in their own test file
+				should(project.files.length).equal(9); // detailed sqlproj tests in their own test file
 				should(project.dataSources.length).equal(2); // detailed datasources tests in their own test file
 			});
 
@@ -201,6 +199,7 @@ describe ('ProjectsController', function(): void {
 				const projController = new ProjectsController(new SqlDatabaseProjectTreeViewProvider());
 
 				await projController.delete(projTreeRoot.children.find(x => x.friendlyName === 'UpperFolder')!.children[0] /* LowerFolder */);
+				await projController.delete(projTreeRoot.children.find(x => x.friendlyName === 'anotherScript.sql')!);
 
 				proj = await Project.openProject(proj.projectFilePath); // reload edited sqlproj from disk
 
@@ -219,6 +218,7 @@ describe ('ProjectsController', function(): void {
 				const projController = new ProjectsController(new SqlDatabaseProjectTreeViewProvider());
 
 				await projController.exclude(<FolderNode>projTreeRoot.children.find(x => x.friendlyName === 'UpperFolder')!.children[0] /* LowerFolder */);
+				await projController.exclude(<FileNode>projTreeRoot.children.find(x => x.friendlyName === 'anotherScript.sql')!);
 
 				proj = await Project.openProject(proj.projectFilePath); // reload edited sqlproj from disk
 
@@ -447,11 +447,11 @@ describe ('ProjectsController', function(): void {
 
 			let result = await projController.getModelFromContext(undefined);
 
-			should(result).deepEqual({database: mockDbSelection, serverId: connectionId});
+			should(result).deepEqual({ database: mockDbSelection, serverId: connectionId });
 
 			// test launch via Object Explorer context
 			result = await projController.getModelFromContext(mockConnectionProfile);
-			should(result).deepEqual({database: 'My Database', serverId: 'My Id'});
+			should(result).deepEqual({ database: 'My Database', serverId: 'My Id' });
 		});
 	});
 
@@ -575,12 +575,13 @@ async function setupDeleteExcludeTest(proj: Project): Promise<[ProjectEntry, Pro
 	await proj.addFolderItem('UpperFolder/LowerFolder');
 	const scriptEntry = await proj.addScriptItem('UpperFolder/LowerFolder/someScript.sql', 'not a real script');
 	await proj.addScriptItem('UpperFolder/LowerFolder/someOtherScript.sql', 'Also not a real script');
+	await proj.addScriptItem('../anotherScript.sql', 'Also not a real script');
 
 	const projTreeRoot = new ProjectRootTreeItem(proj);
 	sinon.stub(vscode.window, 'showWarningMessage').returns(<any>Promise.resolve(constants.yesString));
 
 	// confirm setup
-	should(proj.files.length).equal(4, 'number of file/folder entries');
+	should(proj.files.length).equal(5, 'number of file/folder entries');
 	should(path.parse(scriptEntry.fsUri.fsPath).base).equal('someScript.sql');
 	should((await fs.readFile(scriptEntry.fsUri.fsPath)).toString()).equal('not a real script');
 

extensions/sql-database-projects/src/test/projectTree.test.ts

@@ -116,4 +116,22 @@ describe.skip('Project Tree tests', function (): void {
 			'MyNestedFolder2',
 			'MyFile2.sql']);
 	});
+
+	it('Should be able to parse and include relative paths outside project folder', function (): void {
+		const root = os.platform() === 'win32' ? 'Z:\\Level1\\Level2\\' : '/Root/Level1/Level2';
+		const proj = new Project(vscode.Uri.file(`${root}TestProj.sqlproj`).fsPath);
+
+		// nested entries before explicit top-level folder entry
+		// also, ordering of files/folders at all levels
+		proj.files.push(proj.createProjectEntry('..\\someFolder1\\MyNestedFolder1\\MyFile1.sql', EntryType.File));
+		proj.files.push(proj.createProjectEntry('..\\..\\someFolder2\\MyFile2.sql', EntryType.File));
+		proj.files.push(proj.createProjectEntry('..\\..\\someFolder3', EntryType.Folder)); // folder should not be counted (same as SSDT)
+
+		const tree = new ProjectRootTreeItem(proj);
+		should(tree.children.map(x => x.uri.path)).deepEqual([
+			'/TestProj.sqlproj/Data Sources',
+			'/TestProj.sqlproj/Database References',
+			'/TestProj.sqlproj/MyFile1.sql',
+			'/TestProj.sqlproj/MyFile2.sql']);
+	});
 });

extensions/sql-database-projects/src/test/utils.test.ts

@@ -5,10 +5,12 @@
 
 import * as should from 'should';
 import * as path from 'path';
-import {createDummyFileStructure} from './testUtils';
-import { exists} from '../common/utils';
+import * as os from 'os';
+import { createDummyFileStructure } from './testUtils';
+import { exists, trimUri } from '../common/utils';
+import { Uri } from 'vscode';
 
-describe('Tests to verify exists function', function (): void {
+describe('Tests to verify utils functions', function (): void {
 	it('Should determine existence of files/folders', async () => {
 		let testFolderPath = await createDummyFileStructure();
 
@@ -16,8 +18,25 @@ describe('Tests to verify exists function', function (): void {
 		should(await exists(path.join(testFolderPath, 'file1.sql'))).equal(true);
 		should(await exists(path.join(testFolderPath, 'folder2'))).equal(true);
 		should(await exists(path.join(testFolderPath, 'folder4'))).equal(false);
-		should(await exists(path.join(testFolderPath, 'folder2','file4.sql'))).equal(true);
-		should(await exists(path.join(testFolderPath, 'folder4','file2.sql'))).equal(false);
+		should(await exists(path.join(testFolderPath, 'folder2', 'file4.sql'))).equal(true);
+		should(await exists(path.join(testFolderPath, 'folder4', 'file2.sql'))).equal(false);
+	});
+
+	it('Should get correct relative paths of files/folders', async () => {
+		const root = os.platform() === 'win32' ? 'Z:\\' : '/';
+		let projectUri = Uri.file(path.join(root, 'project', 'folder', 'project.sqlproj'));
+		let fileUri = Uri.file(path.join(root, 'project', 'folder', 'file.sql'));
+		should(trimUri(projectUri, fileUri)).equal('file.sql');
+
+		fileUri = Uri.file(path.join(root, 'project', 'file.sql'));
+		let urifile = trimUri(projectUri, fileUri);
+		should(urifile).equal('../file.sql');
+
+		fileUri = Uri.file(path.join(root, 'project', 'forked', 'file.sql'));
+		should(trimUri(projectUri, fileUri)).equal('../forked/file.sql');
+
+		fileUri = Uri.file(path.join(root, 'forked', 'from', 'top', 'file.sql'));
+		should(trimUri(projectUri, fileUri)).equal('../../forked/from/top/file.sql');
 	});
 });
 

extensions/sql-database-projects/src/tools/buildHelper.ts

@@ -84,8 +84,8 @@ export class BuildHelper {
 	}
 
 	public constructBuildArguments(projectPath: string, buildDirPath: string): string {
-		projectPath = utils.getSafePath(projectPath);
-		buildDirPath = utils.getSafePath(buildDirPath);
+		projectPath = utils.getQuotedPath(projectPath);
+		buildDirPath = utils.getQuotedPath(buildDirPath);
 		return ` build ${projectPath} /p:NetCoreBuild=true /p:NETCoreTargetsPath=${buildDirPath}`;
 	}
 }

extensions/sql-database-projects/src/tools/netcoreTool.ts

@@ -100,7 +100,7 @@ export class NetCoreTool {
 			throw new Error(NetCoreInstallationConfirmation);
 		}
 
-		const dotnetPath = utils.getSafePath(path.join(this.netcoreInstallLocation, dotnet));
+		const dotnetPath = utils.getQuotedPath(path.join(this.netcoreInstallLocation, dotnet));
 		const command = dotnetPath + ' ' + options.argument;
 
 		try {