Add support for showing files for glob style sql projects (#17518)

* use glob to get files for new style msbuild sdk sqlproj

* add tests

* cleanup

* fix test

* don't show bin and obj files and folders

* handle other glob patterns

* fix duplicate entries getting added for glob patterns in project's folder
This commit is contained in:
Kim Santiago
2021-11-05 13:29:47 -07:00
committed by GitHub
parent c9be45b9c7
commit b8ea493f8c
8 changed files with 398 additions and 7 deletions

View File

@@ -529,3 +529,52 @@ export async function showInfoMessageWithOutputChannel(message: string, outputCh
}
}
/**
* Returns the results of the glob pattern
* @param pattern Glob pattern to search for
*/
export async function globWithPattern(pattern: string): Promise<string[]> {
const forwardSlashPattern = pattern.replace(/\\/g, '/');
return await glob(forwardSlashPattern);
}
/**
* Recursively gets all the sql files at any depth in a folder
* @param folderPath
* @param ignoreBinObj ignore sql files in bin and obj folders
*/
export async function getSqlFilesInFolder(folderPath: string, ignoreBinObj?: boolean): Promise<string[]> {
// path needs to use forward slashes for glob to work
folderPath = folderPath.replace(/\\/g, '/');
const sqlFilter = path.posix.join(folderPath, '**', '*.sql');
if (ignoreBinObj) {
// don't add files in bin and obj folders
const binIgnore = path.posix.join(folderPath, 'bin', '**', '*.sql');
const objIgnore = path.posix.join(folderPath, 'obj', '**', '*.sql');
return await glob(sqlFilter, { ignore: [binIgnore, objIgnore] });
} else {
return await glob(sqlFilter);
}
}
/**
* Recursively gets all the folders at any depth in the given folder
* @param folderPath
* @param ignoreBinObj ignore bin and obj folders
*/
export async function getFoldersInFolder(folderPath: string, ignoreBinObj?: boolean): Promise<string[]> {
// path needs to use forward slashes for glob to work
const escapedPath = glob.escapePath(folderPath.replace(/\\/g, '/'));
const folderFilter = path.posix.join(escapedPath, '/**');
if (ignoreBinObj) {
// don't add bin and obj folders
const binIgnore = path.posix.join(escapedPath, 'bin');
const objIgnore = path.posix.join(escapedPath, 'obj');
return await glob(folderFilter, { onlyDirectories: true, ignore: [binIgnore, objIgnore] });
} else {
return await glob(folderFilter, { onlyDirectories: true });
}
}

View File

@@ -129,14 +129,48 @@ export class Project implements ISqlProject {
console.error(utils.getErrorMessage(e));
}
// find all folders and files to include
for (let ig = 0; ig < this.projFileXmlDoc!.documentElement.getElementsByTagName(constants.ItemGroup).length; ig++) {
const itemGroup = this.projFileXmlDoc!.documentElement.getElementsByTagName(constants.ItemGroup)[ig];
// glob style getting files and folders for new msbuild sdk style projects
if (this._isMsbuildSdkStyleProject) {
const files = await utils.getSqlFilesInFolder(this.projectFolderPath, true);
files.forEach(f => {
this._files.push(this.createFileProjectEntry(utils.trimUri(Uri.file(this.projectFilePath), Uri.file(f)), EntryType.File));
});
const folders = await utils.getFoldersInFolder(this.projectFolderPath, true);
folders.forEach(f => {
this._files.push(this.createFileProjectEntry(utils.trimUri(Uri.file(this.projectFilePath), Uri.file(f)), EntryType.Folder));
});
}
for (let ig = 0; ig < this.projFileXmlDoc.documentElement.getElementsByTagName(constants.ItemGroup).length; ig++) {
const itemGroup = this.projFileXmlDoc.documentElement.getElementsByTagName(constants.ItemGroup)[ig];
// find all folders and files to include that are specified in the project file
try {
const buildElements = itemGroup.getElementsByTagName(constants.Build);
for (let b = 0; b < buildElements.length; b++) {
this._files.push(this.createFileProjectEntry(buildElements[b].getAttribute(constants.Include)!, EntryType.File, buildElements[b].getAttribute(constants.Type)!));
const relativePath = buildElements[b].getAttribute(constants.Include)!;
const fullPath = path.join(utils.getPlatformSafeFileEntryPath(this.projectFolderPath), utils.getPlatformSafeFileEntryPath(relativePath));
// msbuild sdk style projects can handle other globbing patterns like <Build Include="folder1\*.sql" /> and <Build Include="Production*.sql" />
if (this._isMsbuildSdkStyleProject && !(await utils.exists(fullPath))) {
// add files from the glob pattern
const globFiles = await utils.globWithPattern(fullPath);
globFiles.forEach(gf => {
const newFileRelativePath = utils.convertSlashesForSqlProj(utils.trimUri(Uri.file(this.projectFilePath), Uri.file(gf)));
if (!this._files.find(f => f.relativePath === newFileRelativePath)) {
this._files.push(this.createFileProjectEntry(utils.trimUri(Uri.file(this.projectFilePath), Uri.file(gf)), EntryType.File));
}
});
// TODO: add support for <Build Remove="file.sql">
} else {
// only add file if it wasn't already added
if (!this._files.find(f => f.relativePath === relativePath)) {
this._files.push(this.createFileProjectEntry(relativePath, EntryType.File, buildElements[b].getAttribute(constants.Type)!));
}
}
}
} catch (e) {
void window.showErrorMessage(constants.errorReadingProject(constants.BuildElements, this.projectFilePath));
@@ -146,9 +180,10 @@ export class Project implements ISqlProject {
try {
const folderElements = itemGroup.getElementsByTagName(constants.Folder);
for (let f = 0; f < folderElements.length; f++) {
// don't add Properties folder since it isn't supported for now
if (folderElements[f].getAttribute(constants.Include) !== constants.Properties) {
this._files.push(this.createFileProjectEntry(folderElements[f].getAttribute(constants.Include)!, EntryType.Folder));
const relativePath = folderElements[f].getAttribute(constants.Include)!;
// don't add Properties folder since it isn't supported for now and don't add if the folder was already added
if (relativePath !== constants.Properties && !this._files.find(f => f.relativePath === utils.trimChars(relativePath, '\\'))) {
this._files.push(this.createFileProjectEntry(relativePath, EntryType.Folder));
}
}
} catch (e) {

View File

@@ -30,6 +30,9 @@ export let sqlProjectInvalidCollationBaseline: string;
export let newStyleProjectSdkNodeBaseline: string;
export let newStyleProjectSdkProjectAttributeBaseline: string;
export let newStyleProjectSdkImportAttributeBaseline: string;
export let openNewStyleSqlProjectBaseline: string;
export let openNewStyleSqlProjectWithFilesSpecifiedBaseline: string;
export let openNewStyleSqlProjectWithGlobsSpecifiedBaseline: string;
const baselineFolderPath = __dirname;
@@ -57,6 +60,9 @@ export async function loadBaselines() {
newStyleProjectSdkNodeBaseline = await loadBaseline(baselineFolderPath, 'newStyleSqlProjectSdkNodeBaseline.xml');
newStyleProjectSdkProjectAttributeBaseline = await loadBaseline(baselineFolderPath, 'newStyleSqlProjectSdkProjectAttributeBaseline.xml');
newStyleProjectSdkImportAttributeBaseline = await loadBaseline(baselineFolderPath, 'newStyleSqlProjectSdkImportAttributeBaseline.xml');
openNewStyleSqlProjectBaseline = await loadBaseline(baselineFolderPath, 'openNewStyleSqlProjectBaseline.xml');
openNewStyleSqlProjectWithFilesSpecifiedBaseline = await loadBaseline(baselineFolderPath, 'openNewStyleSqlProjectWithFilesSpecifiedBaseline.xml');
openNewStyleSqlProjectWithGlobsSpecifiedBaseline = await loadBaseline(baselineFolderPath, 'openNewStyleSqlProjectWithGlobsSpecifiedBaseline.xml');
}
async function loadBaseline(baselineFolderPath: string, fileName: string): Promise<string> {

View File

@@ -0,0 +1,41 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build">
<Sdk Name="Microsoft.Build.Sql" Version="1.0.0" />
<PropertyGroup>
<Name>TestProjectName</Name>
<ProjectGuid>{2C283C5D-9E4A-4313-8FF9-4E0CEE20B063}</ProjectGuid>
<DSP>Microsoft.Data.Tools.Schema.Sql.Sql150DatabaseSchemaProvider</DSP>
<ModelCollation>1033, CI</ModelCollation>
</PropertyGroup>
<Target Name="BeforeBuild">
<Delete Files="$(BaseIntermediateOutputPath)\project.assets.json" />
</Target>
<ItemGroup>
<SqlCmdVariable Include="ProdDatabaseName">
<DefaultValue>MyProdDatabase</DefaultValue>
<Value>$(SqlCmdVar__1)</Value>
</SqlCmdVariable>
<SqlCmdVariable Include="BackupDatabaseName">
<DefaultValue>MyBackupDatabase</DefaultValue>
<Value>$(SqlCmdVar__2)</Value>
</SqlCmdVariable>
</ItemGroup>
<ItemGroup>
<PreDeploy Include="Script.PreDeployment1.sql" />
<None Include="Script.PreDeployment2.sql" />
</ItemGroup>
<ItemGroup>
<PostDeploy Include="Script.PostDeployment1.sql" />
<None Include="folder1\Script.PostDeployment2.sql" />
</ItemGroup>
<ItemGroup>
<ArtifactReference Condition="'$(NetCoreBuild)' == 'true'" Include="$(NETCoreTargetsPath)\SystemDacpacs\150\master.dacpac">
<SuppressMissingDependenciesErrors>False</SuppressMissingDependenciesErrors>
<DatabaseVariableLiteralValue>master</DatabaseVariableLiteralValue>
</ArtifactReference>
<ArtifactReference Condition="'$(NetCoreBuild)' != 'true'" Include="$(DacPacRootPath)\Extensions\Microsoft\SQLDB\Extensions\SqlServer\150\SqlSchemas\master.dacpac">
<SuppressMissingDependenciesErrors>False</SuppressMissingDependenciesErrors>
<DatabaseVariableLiteralValue>master</DatabaseVariableLiteralValue>
</ArtifactReference>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build">
<Sdk Name="Microsoft.Build.Sql" Version="1.0.0" />
<PropertyGroup>
<Name>TestProjectName</Name>
<ProjectGuid>{2C283C5D-9E4A-4313-8FF9-4E0CEE20B063}</ProjectGuid>
<DSP>Microsoft.Data.Tools.Schema.Sql.Sql150DatabaseSchemaProvider</DSP>
<ModelCollation>1033, CI</ModelCollation>
</PropertyGroup>
<Target Name="BeforeBuild">
<Delete Files="$(BaseIntermediateOutputPath)\project.assets.json" />
</Target>
<ItemGroup>
<Folder Include="Properties" />
<Folder Include="folder1" />
</ItemGroup>
<ItemGroup>
<Build Include="folder1\file2.sql" />
<Build Include="file1.sql" />
</ItemGroup>
<ItemGroup>
<SqlCmdVariable Include="ProdDatabaseName">
<DefaultValue>MyProdDatabase</DefaultValue>
<Value>$(SqlCmdVar__1)</Value>
</SqlCmdVariable>
<SqlCmdVariable Include="BackupDatabaseName">
<DefaultValue>MyBackupDatabase</DefaultValue>
<Value>$(SqlCmdVar__2)</Value>
</SqlCmdVariable>
</ItemGroup>
<ItemGroup>
<PreDeploy Include="Script.PreDeployment1.sql" />
<None Include="Script.PreDeployment2.sql" />
</ItemGroup>
<ItemGroup>
<PostDeploy Include="Script.PostDeployment1.sql" />
<None Include="folder1\Script.PostDeployment1.sql" />
</ItemGroup>
<ItemGroup>
<ArtifactReference Condition="'$(NetCoreBuild)' == 'true'" Include="$(NETCoreTargetsPath)\SystemDacpacs\150\master.dacpac">
<SuppressMissingDependenciesErrors>False</SuppressMissingDependenciesErrors>
<DatabaseVariableLiteralValue>master</DatabaseVariableLiteralValue>
</ArtifactReference>
<ArtifactReference Condition="'$(NetCoreBuild)' != 'true'" Include="$(DacPacRootPath)\Extensions\Microsoft\SQLDB\Extensions\SqlServer\150\SqlSchemas\master.dacpac">
<SuppressMissingDependenciesErrors>False</SuppressMissingDependenciesErrors>
<DatabaseVariableLiteralValue>master</DatabaseVariableLiteralValue>
</ArtifactReference>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build">
<Sdk Name="Microsoft.Build.Sql" Version="1.0.0" />
<PropertyGroup>
<Name>TestProjectName</Name>
<ProjectGuid>{2C283C5D-9E4A-4313-8FF9-4E0CEE20B063}</ProjectGuid>
<DSP>Microsoft.Data.Tools.Schema.Sql.Sql150DatabaseSchemaProvider</DSP>
<ModelCollation>1033, CI</ModelCollation>
</PropertyGroup>
<Target Name="BeforeBuild">
<Delete Files="$(BaseIntermediateOutputPath)\project.assets.json" />
</Target>
<ItemGroup>
<Folder Include="Properties" />
</ItemGroup>
<ItemGroup>
<Build Include="..\other\folder1\file*.sql" />
<Build Include="..\other\folder1\test?.sql" />
<Build Include="..\other\folder2\*.sql" />
<Build Include="folder1\*.sql" />
</ItemGroup>
<ItemGroup>
<ArtifactReference Condition="'$(NetCoreBuild)' == 'true'" Include="$(NETCoreTargetsPath)\SystemDacpacs\150\master.dacpac">
<SuppressMissingDependenciesErrors>False</SuppressMissingDependenciesErrors>
<DatabaseVariableLiteralValue>master</DatabaseVariableLiteralValue>
</ArtifactReference>
<ArtifactReference Condition="'$(NetCoreBuild)' != 'true'" Include="$(DacPacRootPath)\Extensions\Microsoft\SQLDB\Extensions\SqlServer\150\SqlSchemas\master.dacpac">
<SuppressMissingDependenciesErrors>False</SuppressMissingDependenciesErrors>
<DatabaseVariableLiteralValue>master</DatabaseVariableLiteralValue>
</ArtifactReference>
</ItemGroup>
</Project>

View File

@@ -827,6 +827,95 @@ describe('Project: sqlproj content operations', function (): void {
});
});
describe('Project: Msbuild sdk style project content operations', function (): void {
before(async function (): Promise<void> {
await baselines.loadBaselines();
});
beforeEach(function (): void {
sinon.restore();
});
it('Should read project from sqlproj and files and folders by globbing', async function (): Promise<void> {
projFilePath = await testUtils.createTestSqlProjFile(baselines.openNewStyleSqlProjectBaseline);
await testUtils.createDummyFileStructureWithPrePostDeployScripts(false, undefined, path.dirname(projFilePath));
const project: Project = await Project.openProject(projFilePath);
// Files and folders
should(project.files.filter(f => f.type === EntryType.Folder).length).equal(2);
should(project.files.filter(f => f.type === EntryType.File).length).equal(15);
// SqlCmdVariables
should(Object.keys(project.sqlCmdVariables).length).equal(2);
should(project.sqlCmdVariables['ProdDatabaseName']).equal('MyProdDatabase');
should(project.sqlCmdVariables['BackupDatabaseName']).equal('MyBackupDatabase');
// Database references
// should only have one database reference even though there are two master.dacpac references (1 for ADS and 1 for SSDT)
should(project.databaseReferences.length).equal(1);
should(project.databaseReferences[0].databaseName).containEql(constants.master);
should(project.databaseReferences[0] instanceof SystemDatabaseReferenceProjectEntry).equal(true);
// // Pre-post deployment scripts
should(project.preDeployScripts.length).equal(1);
should(project.postDeployScripts.length).equal(1);
should(project.noneDeployScripts.length).equal(2);
should(project.preDeployScripts.find(f => f.type === EntryType.File && f.relativePath === 'Script.PreDeployment1.sql')).not.equal(undefined, 'File Script.PreDeployment1.sql not read');
should(project.noneDeployScripts.find(f => f.type === EntryType.File && f.relativePath === 'Script.PreDeployment2.sql')).not.equal(undefined, 'File Script.PreDeployment2.sql not read');
should(project.postDeployScripts.find(f => f.type === EntryType.File && f.relativePath === 'Script.PostDeployment1.sql')).not.equal(undefined, 'File Script.PostDeployment1.sql not read');
should(project.noneDeployScripts.find(f => f.type === EntryType.File && f.relativePath === 'folder1\\Script.PostDeployment2.sql')).not.equal(undefined, 'File folder1\\Script.PostDeployment2.sql not read');
});
it('Should handle files listed in sqlproj', async function (): Promise<void> {
projFilePath = await testUtils.createTestSqlProjFile(baselines.openNewStyleSqlProjectWithFilesSpecifiedBaseline);
await testUtils.createDummyFileStructure(false, undefined, path.dirname(projFilePath));
const project: Project = await Project.openProject(projFilePath);
// Files and folders
should(project.files.filter(f => f.type === EntryType.Folder).length).equal(2);
should(project.files.filter(f => f.type === EntryType.File).length).equal(11);
// these are also listed in the sqlproj, but there shouldn't be duplicate entries for them
should(project.files.filter(f => f.relativePath === 'folder1\\file2.sql').length).equal(1);
should(project.files.filter(f => f.relativePath === 'file1.sql').length).equal(1);
should(project.files.filter(f => f.relativePath === 'folder1').length).equal(1);
});
it('Should handle globbing patterns listed in sqlproj', async function (): Promise<void> {
const testFolderPath = await testUtils.generateTestFolderPath();
const mainProjectPath = path.join(testFolderPath, 'project');
const otherFolderPath = path.join(testFolderPath, 'other');
projFilePath = await testUtils.createTestSqlProjFile(baselines.openNewStyleSqlProjectWithGlobsSpecifiedBaseline, mainProjectPath);
await testUtils.createDummyFileStructure(false, undefined, path.dirname(projFilePath));
// create files outside of project folder that are included in the project file
await fs.mkdir(otherFolderPath);
await testUtils.createOtherDummyFiles(otherFolderPath);
const project: Project = await Project.openProject(projFilePath);
should(project.files.filter(f => f.type === EntryType.File).length).equal(17);
// make sure all the correct files from the globbing patterns were included
// ..\other\folder1\test?.sql
should(project.files.filter(f => f.relativePath === '..\\other\\folder1\\test1.sql').length).equal(1);
should(project.files.filter(f => f.relativePath === '..\\other\\folder1\\test2.sql').length).equal(1);
should(project.files.filter(f => f.relativePath === '..\\other\\folder1\\testLongerName.sql').length).equal(0);
// ..\other\folder1\file*.sql
should(project.files.filter(f => f.relativePath === '..\\other\\folder1\\file1.sql').length).equal(1);
should(project.files.filter(f => f.relativePath === '..\\other\\folder1\\file2.sql').length).equal(1);
// ..\other\folder2\*.sql
should(project.files.filter(f => f.relativePath === '..\\other\\folder2\\file1.sql').length).equal(1);
should(project.files.filter(f => f.relativePath === '..\\other\\folder2\\file2.sql').length).equal(1);
// make sure no duplicates from folder1\*.sql
should(project.files.filter(f => f.relativePath === 'folder1\\file1.sql').length).equal(1);
});
});
describe('Project: add SQLCMD Variables', function (): void {
before(async function (): Promise<void> {
await baselines.loadBaselines();

View File

@@ -115,6 +115,58 @@ export async function createDummyFileStructure(createList?: boolean, list?: Uri[
return testFolderPath;
}
/**
* TestFolder directory structure
* - file1.sql
* - folder1
* -file1.sql
* -file2.sql
* -file3.sql
* -file4.sql
* -file5.sql
* -Script.PostDeployment2.sql
* - folder2
* -file1.sql
* -file2.sql
* -file3.sql
* -file4.sql
* -file5.sql
* - file2.txt
* - Script.PreDeployment1.sql
* - Script.PreDeployment2.sql
* - Script.PostDeployment1.sql
*
* @param createList Boolean specifying to create a list of the files and folders been created
* @param list List of files and folders that are been created
*/
export async function createDummyFileStructureWithPrePostDeployScripts(createList?: boolean, list?: Uri[], testFolderPath?: string): Promise<string> {
testFolderPath = await createDummyFileStructure(createList, list, testFolderPath);
// add pre-deploy scripts
const predeployscript1 = path.join(testFolderPath, 'Script.PreDeployment1.sql');
await fs.writeFile(predeployscript1, '');
const predeployscript2 = path.join(testFolderPath, 'Script.PreDeployment2.sql');
await fs.writeFile(predeployscript2, '');
if (createList) {
list?.push(Uri.file(predeployscript1));
list?.push(Uri.file(predeployscript2));
}
// add post-deploy scripts
const postdeployscript1 = path.join(testFolderPath, 'Script.PostDeployment1.sql');
await fs.writeFile(postdeployscript1, '');
const postdeployscript2 = path.join(testFolderPath, 'folder1', 'Script.PostDeployment2.sql');
await fs.writeFile(postdeployscript2, '');
if (createList) {
list?.push(Uri.file(postdeployscript1));
list?.push(Uri.file(postdeployscript2));
}
return testFolderPath;
}
export async function createListOfFiles(filePath?: string): Promise<Uri[]> {
let fileFolderList: Uri[] = [];
@@ -122,3 +174,41 @@ export async function createListOfFiles(filePath?: string): Promise<Uri[]> {
return fileFolderList;
}
/**
* TestFolder directory structure
* - file1.sql
* - folder1
* -file1.sql
* -file2.sql
* -test1.sql
* -test2.sql
* -testLongerName.sql
* - folder2
* -file1.sql
* -file2.sql
*
*/
export async function createOtherDummyFiles(testFolderPath: string): Promise<string> {
let filePath = path.join(testFolderPath, 'file1.sql');
await fs.writeFile(filePath, '');
for (let dirCount = 1; dirCount <= 2; dirCount++) {
let dirName = path.join(testFolderPath, `folder${dirCount}`);
await fs.mkdir(dirName, { recursive: true });
for (let fileCount = 1; fileCount <= 2; fileCount++) {
let fileName = path.join(dirName, `file${fileCount}.sql`);
await fs.writeFile(fileName, '');
}
}
const test1 = path.join(testFolderPath, 'folder1', 'test1.sql');
await fs.writeFile(test1, '');
const test2 = path.join(testFolderPath, 'folder1', 'test2.sql');
await fs.writeFile(test2, '');
const testLongerName = path.join(testFolderPath, 'folder1', 'testLongerName.sql');
await fs.writeFile(testLongerName, '');
return testFolderPath;
}