Adding External Streaming Job I/O validation (#13195)

* Added Tools Service call for ValidateStreamingJob

* Partial addition of ESJ

* adding test mocks

* Validation working

* Modifying command visibility logic to submatch ESJs in addition to files

* Changed string literal to constant, corrected attribute order

* Added tests

* correcting casing that's causing test failures on linux

* Swapping Thenable for Promise

* excluded validate from command palette

extensions/sql-database-projects/src/common/constants.ts

@@ -149,6 +149,7 @@ export const ousiderFolderPath = localize('outsideFolderPath', "Items with absol
 export const parentTreeItemUnknown = localize('parentTreeItemUnknown', "Cannot access parent of provided tree item");
 export const prePostDeployCount = localize('prePostDeployCount', "To successfully build, update the project to have one pre-deployment script and/or one post-deployment script");
 export const invalidProjectReload = localize('invalidProjectReload', "Cannot access provided database project. Only valid, open database projects can be reloaded.");
+export const externalStreamingJobValidationPassed = localize('externalStreamingJobValidationPassed', "Validation of external streaming job passed.");
 export function projectAlreadyOpened(path: string) { return localize('projectAlreadyOpened', "Project '{0}' is already opened.", path); }
 export function projectAlreadyExists(name: string, path: string) { return localize('projectAlreadyExists', "A project named {0} already exists in {1}.", name, path); }
 export function noFileExist(fileName: string) { return localize('noFileExist', "File {0} doesn't exist", fileName); }
@@ -184,6 +185,7 @@ export const scriptFriendlyName = localize('scriptFriendlyName', "Script");
 export const tableFriendlyName = localize('tableFriendlyName', "Table");
 export const viewFriendlyName = localize('viewFriendlyName', "View");
 export const storedProcedureFriendlyName = localize('storedProcedureFriendlyName', "Stored Procedure");
+export const externalStreamingJobFriendlyName = localize('externalStreamingJobFriendlyName', "External Streaming Job");
 export const preDeployScriptFriendlyName = localize('preDeployScriptFriendlyName', "Script.PreDeployment");
 export const postDeployScriptFriendlyName = localize('postDeployScriptFriendlyName', "Script.PostDeployment");
 
@@ -223,6 +225,8 @@ export const True = 'True';
 export const False = 'False';
 export const Private = 'Private';
 export const ProjectGuid = 'ProjectGuid';
+export const Type = 'Type';
+export const ExternalStreamingJob: string = 'ExternalStreamingJob';
 
 // SqlProj File targets
 export const NetCoreTargets = '$(NETCoreTargetsPath)\\Microsoft.Data.Tools.Schema.SqlTasks.targets';
@@ -270,6 +274,7 @@ export enum DatabaseProjectItemType {
 	project = 'databaseProject.itemType.project',
 	folder = 'databaseProject.itemType.folder',
 	file = 'databaseProject.itemType.file',
+	externalStreamingJob = 'databaseProject.itemType.file.externalStreamingJob',
 	referencesRoot = 'databaseProject.itemType.referencesRoot',
 	reference = 'databaseProject.itemType.reference',
 	dataSourceRoot = 'databaseProject.itemType.dataSourceRoot',

extensions/sql-database-projects/src/controllers/mainController.ts

@@ -16,7 +16,7 @@ import { ProjectsController } from './projectController';
 import { BaseProjectTreeItem } from '../models/tree/baseTreeItem';
 import { NetCoreTool } from '../tools/netcoreTool';
 import { Project } from '../models/project';
-import { FileNode, FolderNode } from '../models/tree/fileFolderTreeItem';
+import { ExternalStreamingJobFileNode, FileNode, FolderNode } from '../models/tree/fileFolderTreeItem';
 import { IconPathHelper } from '../common/iconHelper';
 import { IProjectProvider } from 'dataworkspace';
 import { SqlDatabaseProjectProvider } from '../projectProvider/projectProvider';
@@ -70,6 +70,7 @@ export default class MainController implements vscode.Disposable {
 		vscode.commands.registerCommand('sqlDatabaseProjects.newTable', async (node: BaseProjectTreeItem) => { await this.projectsController.addItemPromptFromNode(node, templates.table); });
 		vscode.commands.registerCommand('sqlDatabaseProjects.newView', async (node: BaseProjectTreeItem) => { await this.projectsController.addItemPromptFromNode(node, templates.view); });
 		vscode.commands.registerCommand('sqlDatabaseProjects.newStoredProcedure', async (node: BaseProjectTreeItem) => { await this.projectsController.addItemPromptFromNode(node, templates.storedProcedure); });
+		vscode.commands.registerCommand('sqlDatabaseProjects.newExternalStreamingJob', async (node: BaseProjectTreeItem) => { await this.projectsController.addItemPromptFromNode(node, templates.externalStreamingJob); });
 		vscode.commands.registerCommand('sqlDatabaseProjects.newItem', async (node: BaseProjectTreeItem) => { await this.projectsController.addItemPromptFromNode(node); });
 		vscode.commands.registerCommand('sqlDatabaseProjects.newFolder', async (node: BaseProjectTreeItem) => { await this.projectsController.addFolderPrompt(node); });
 
@@ -79,6 +80,7 @@ export default class MainController implements vscode.Disposable {
 		vscode.commands.registerCommand('sqlDatabaseProjects.delete', async (node: BaseProjectTreeItem) => { await this.projectsController.delete(node); });
 		vscode.commands.registerCommand('sqlDatabaseProjects.exclude', async (node: FileNode | FolderNode) => { await this.projectsController.exclude(node); });
 		vscode.commands.registerCommand('sqlDatabaseProjects.changeTargetPlatform', async (node: BaseProjectTreeItem) => { await this.projectsController.changeTargetPlatform(node); });
+		vscode.commands.registerCommand('sqlDatabaseProjects.validateExternalStreamingJob', async (node: ExternalStreamingJobFileNode) => { await this.projectsController.validateExternalStreamingJob(node); });
 
 		IconPathHelper.setExtensionContext(this.extensionContext);
 

extensions/sql-database-projects/src/controllers/projectController.ts

@@ -19,7 +19,7 @@ import { promises as fs } from 'fs';
 import { PublishDatabaseDialog } from '../dialogs/publishDatabaseDialog';
 import { Project, reservedProjectFolders, FileProjectEntry, SqlProjectReferenceProjectEntry, IDatabaseReferenceProjectEntry } from '../models/project';
 import { SqlDatabaseProjectTreeViewProvider } from './databaseProjectTreeViewProvider';
-import { FolderNode, FileNode } from '../models/tree/fileFolderTreeItem';
+import { FolderNode, FileNode, ExternalStreamingJobFileNode } from '../models/tree/fileFolderTreeItem';
 import { IPublishSettings, IGenerateScriptSettings } from '../models/IPublishSettings';
 import { BaseProjectTreeItem } from '../models/tree/baseTreeItem';
 import { ProjectRootTreeItem } from '../models/tree/projectTreeItem';
@@ -204,7 +204,7 @@ export class ProjectsController {
 		try {
 			await this.netCoreTool.runDotnetCommand(options);
 
-			return path.join(project.projectFolderPath, 'bin', 'Debug', `${project.projectFileName}.dacpac`);
+			return project.dacpacOutputPath;
 		}
 		catch (err) {
 			vscode.window.showErrorMessage(constants.projBuildFailed(utils.getErrorMessage(err)));
@@ -577,6 +577,31 @@ export class ProjectsController {
 		}
 	}
 
+
+	public async validateExternalStreamingJob(node: ExternalStreamingJobFileNode): Promise<mssql.ValidateStreamingJobResult> {
+		const project: Project = this.getProjectFromContext(node);
+
+		let dacpacPath: string = project.dacpacOutputPath;
+
+		if (!await utils.exists(dacpacPath)) {
+			dacpacPath = await this.buildProject(node);
+		}
+
+		const streamingJobDefinition: string = (await fs.readFile(node.fileSystemUri.fsPath)).toString();
+
+		const dacFxService = await this.getDaxFxService();
+		const result: mssql.ValidateStreamingJobResult = await dacFxService.validateStreamingJob(dacpacPath, streamingJobDefinition);
+
+		if (result.success) {
+			vscode.window.showInformationMessage(constants.externalStreamingJobValidationPassed);
+		}
+		else {
+			vscode.window.showErrorMessage(result.errorMessage);
+		}
+
+		return result;
+	}
+
 	//#region Helper methods
 
 	public getPublishDialog(project: Project): PublishDatabaseDialog {

extensions/sql-database-projects/src/models/project.ts

@@ -32,6 +32,10 @@ export class Project {
 	public postDeployScripts: FileProjectEntry[] = [];
 	public noneDeployScripts: FileProjectEntry[] = [];
 
+	public get dacpacOutputPath(): string {
+		return path.join(this.projectFolderPath, 'bin', 'Debug', `${this.projectFileName}.dacpac`);
+	}
+
 	public get projectFolderPath() {
 		return Uri.file(path.dirname(this.projectFilePath)).fsPath;
 	}
@@ -71,7 +75,7 @@ export class Project {
 
 			const buildElements = itemGroup.getElementsByTagName(constants.Build);
 			for (let b = 0; b < buildElements.length; b++) {
-				this.files.push(this.createFileProjectEntry(buildElements[b].getAttribute(constants.Include), EntryType.File));
+				this.files.push(this.createFileProjectEntry(buildElements[b].getAttribute(constants.Include), EntryType.File, buildElements[b].getAttribute(constants.Type)));
 			}
 
 			const folderElements = itemGroup.getElementsByTagName(constants.Folder);
@@ -291,7 +295,13 @@ export class Project {
 				this.files.push(fileEntry);
 		}
 
-		await this.addToProjFile(fileEntry, xmlTag);
+		const attributes = new Map<string, string>();
+
+		if (itemType === templates.externalStreamingJob) {
+			attributes.set(constants.Type, constants.ExternalStreamingJob);
+		}
+
+		await this.addToProjFile(fileEntry, xmlTag, attributes);
 
 		return fileEntry;
 	}
@@ -454,9 +464,9 @@ export class Project {
 		await this.addToProjFile(sqlCmdVariableEntry);
 	}
 
-	public createFileProjectEntry(relativePath: string, entryType: EntryType): FileProjectEntry {
+	public createFileProjectEntry(relativePath: string, entryType: EntryType, sqlObjectType?: string): FileProjectEntry {
 		let platformSafeRelativePath = utils.getPlatformSafeFileEntryPath(relativePath);
-		return new FileProjectEntry(Uri.file(path.join(this.projectFolderPath, platformSafeRelativePath)), relativePath, entryType);
+		return new FileProjectEntry(Uri.file(path.join(this.projectFolderPath, platformSafeRelativePath)), relativePath, entryType, sqlObjectType);
 	}
 
 	private findOrCreateItemGroup(containedTag?: string, prePostScriptExist?: { scriptExist: boolean; }): any {
@@ -480,6 +490,7 @@ export class Project {
 		if (!outputItemGroup) {
 			outputItemGroup = this.projFileXmlDoc.createElement(constants.ItemGroup);
 			this.projFileXmlDoc.documentElement.appendChild(outputItemGroup);
+
 			if (prePostScriptExist) {
 				prePostScriptExist.scriptExist = false;
 			}
@@ -488,12 +499,13 @@ export class Project {
 		return outputItemGroup;
 	}
 
-	private addFileToProjFile(path: string, xmlTag: string): void {
+	private addFileToProjFile(path: string, xmlTag: string, attributes?: Map<string, string>): void {
 		let itemGroup;
 
 		if (xmlTag === constants.PreDeploy || xmlTag === constants.PostDeploy) {
 			let prePostScriptExist = { scriptExist: true };
 			itemGroup = this.findOrCreateItemGroup(xmlTag, prePostScriptExist);
+
 			if (prePostScriptExist.scriptExist === true) {
 				window.showInformationMessage(constants.deployScriptExists(xmlTag));
 				xmlTag = constants.None;	// Add only one pre-deploy and post-deploy script. All additional ones get added in the same item group with None tag
@@ -504,7 +516,15 @@ export class Project {
 		}
 
 		const newFileNode = this.projFileXmlDoc.createElement(xmlTag);
+
 		newFileNode.setAttribute(constants.Include, utils.convertSlashesForSqlProj(path));
+
+		if (attributes) {
+			for (const key of attributes.keys()) {
+				newFileNode.setAttribute(key, attributes.get(key));
+			}
+		}
+
 		itemGroup.appendChild(newFileNode);
 	}
 
@@ -530,12 +550,14 @@ export class Project {
 	private removeNode(includeString: string, nodes: any): boolean {
 		for (let i = 0; i < nodes.length; i++) {
 			const parent = nodes[i].parentNode;
+
 			if (nodes[i].getAttribute(constants.Include) === utils.convertSlashesForSqlProj(includeString)) {
 				parent.removeChild(nodes[i]);
 
 				// delete ItemGroup if this was the only entry
 				// only want element nodes, not text nodes
 				const otherChildren = Array.from(parent.childNodes).filter((c: any) => c.childNodes);
+
 				if (otherChildren.length === 0) {
 					parent.parentNode.removeChild(parent);
 				}
@@ -809,10 +831,10 @@ export class Project {
 		}
 	}
 
-	private async addToProjFile(entry: ProjectEntry, xmlTag?: string): Promise<void> {
+	private async addToProjFile(entry: ProjectEntry, xmlTag?: string, attributes?: Map<string, string>): Promise<void> {
 		switch (entry.type) {
 			case EntryType.File:
-				this.addFileToProjFile((<FileProjectEntry>entry).relativePath, xmlTag ? xmlTag : constants.Build);
+				this.addFileToProjFile((<FileProjectEntry>entry).relativePath, xmlTag ? xmlTag : constants.Build, attributes);
 				break;
 			case EntryType.Folder:
 				this.addFolderToProjFile((<FileProjectEntry>entry).relativePath);
@@ -901,11 +923,13 @@ export class FileProjectEntry extends ProjectEntry {
 	 */
 	fsUri: Uri;
 	relativePath: string;
+	sqlObjectType: string | undefined;
 
-	constructor(uri: Uri, relativePath: string, type: EntryType) {
-		super(type);
+	constructor(uri: Uri, relativePath: string, entryType: EntryType, sqlObjectType?: string) {
+		super(entryType);
 		this.fsUri = uri;
 		this.relativePath = relativePath;
+		this.sqlObjectType = sqlObjectType;
 	}
 
 	public toString(): string {

extensions/sql-database-projects/src/models/tree/fileFolderTreeItem.ts

@@ -71,6 +71,15 @@ export class FileNode extends BaseProjectTreeItem {
 	}
 }
 
+export class ExternalStreamingJobFileNode extends FileNode {
+	public get treeItem(): vscode.TreeItem {
+		const treeItem = super.treeItem;
+		treeItem.contextValue = DatabaseProjectItemType.externalStreamingJob;
+
+		return treeItem;
+	}
+}
+
 /**
  * Compares two folder/file tree nodes so that folders come before files, then alphabetically
  * @param a a folder or file tree node

extensions/sql-database-projects/src/models/tree/projectTreeItem.ts

@@ -11,7 +11,7 @@ import * as fileTree from './fileFolderTreeItem';
 import { Project, EntryType, FileProjectEntry } from '../project';
 import * as utils from '../../common/utils';
 import { DatabaseReferencesTreeItem } from './databaseReferencesTreeItem';
-import { DatabaseProjectItemType, RelativeOuterPath } from '../../common/constants';
+import { DatabaseProjectItemType, RelativeOuterPath, ExternalStreamingJob } from '../../common/constants';
 import { IconPathHelper } from '../../common/iconHelper';
 
 /**
@@ -76,7 +76,13 @@ export class ProjectRootTreeItem extends BaseProjectTreeItem {
 
 			switch (entry.type) {
 				case EntryType.File:
-					newNode = new fileTree.FileNode(entry.fsUri, parentNode);
+					if (entry.sqlObjectType === ExternalStreamingJob) {
+						newNode = new fileTree.ExternalStreamingJobFileNode(entry.fsUri, parentNode);
+					}
+					else {
+						newNode = new fileTree.FileNode(entry.fsUri, parentNode);
+					}
+
 					break;
 				case EntryType.Folder:
 					newNode = new fileTree.FolderNode(entry.fsUri, parentNode);

extensions/sql-database-projects/src/templates/templates.ts

@@ -15,6 +15,8 @@ export const script: string = 'script';
 export const table: string = 'table';
 export const view: string = 'view';
 export const storedProcedure: string = 'storedProcedure';
+export const externalStreamingJob: string = 'externalStreamingJob';
+
 export const folder: string = 'folder';
 export const preDeployScript: string = 'preDeployScript';
 export const postDeployScript: string = 'postDeployScript';
@@ -49,7 +51,8 @@ export async function loadTemplates(templateFolderPath: string) {
 		loadObjectTypeInfo(view, constants.viewFriendlyName, templateFolderPath, 'newTsqlViewTemplate.sql'),
 		loadObjectTypeInfo(storedProcedure, constants.storedProcedureFriendlyName, templateFolderPath, 'newTsqlStoredProcedureTemplate.sql'),
 		loadObjectTypeInfo(preDeployScript, constants.preDeployScriptFriendlyName, templateFolderPath, 'newTsqlPreDeployScriptTemplate.sql'),
-		loadObjectTypeInfo(postDeployScript, constants.postDeployScriptFriendlyName, templateFolderPath, 'newTsqlPostDeployScriptTemplate.sql')
+		loadObjectTypeInfo(postDeployScript, constants.postDeployScriptFriendlyName, templateFolderPath, 'newTsqlPostDeployScriptTemplate.sql'),
+		loadObjectTypeInfo(externalStreamingJob, constants.externalStreamingJobFriendlyName, templateFolderPath, 'newTsqlExternalStreamingJobTemplate.sql')
 	]);
 
 	for (const scriptType of scriptTypes) {

extensions/sql-database-projects/src/test/baselines/openSqlProjectBaseline.xml

@@ -74,6 +74,9 @@
     <Folder Include="Views\User" />
     <Build Include="Views\User\Profile.sql" />
   </ItemGroup>
+  <ItemGroup>
+    <Build Include="MyExternalStreamingJob.sql" Type="ExternalStreamingJob" />
+  </ItemGroup>
   <ItemGroup>
     <SqlCmdVariable Include="ProdDatabaseName">
       <DefaultValue>MyProdDatabase</DefaultValue>

extensions/sql-database-projects/src/test/baselines/openSqlProjectWithAdditionalSqlCmdVariablesBaseline.xml

@@ -74,6 +74,9 @@
     <Folder Include="Views\User"/>
     <Build Include="Views\User\Profile.sql"/>
   </ItemGroup>
+  <ItemGroup>
+    <Build Include="MyExternalStreamingJob.sql" Type="ExternalStreamingJob"/>
+  </ItemGroup>
   <ItemGroup>
     <SqlCmdVariable Include="BackupDatabaseName">
       <DefaultValue>MyBackupDatabase</DefaultValue>

extensions/sql-database-projects/src/test/project.test.ts

@@ -31,12 +31,14 @@ describe('Project: sqlproj content operations', function (): void {
 		const project: Project = await Project.openProject(projFilePath);
 
 		// Files and folders
-		should(project.files.filter(f => f.type === EntryType.File).length).equal(5);
+		should(project.files.filter(f => f.type === EntryType.File).length).equal(6);
 		should(project.files.filter(f => f.type === EntryType.Folder).length).equal(4);
 
 		should(project.files.find(f => f.type === EntryType.Folder && f.relativePath === 'Views\\User')).not.equal(undefined); // mixed ItemGroup folder
 		should(project.files.find(f => f.type === EntryType.File && f.relativePath === 'Views\\User\\Profile.sql')).not.equal(undefined); // mixed ItemGroup file
 		should(project.files.find(f => f.type === EntryType.File && f.relativePath === '..\\Test\\Test.sql')).not.equal(undefined); // mixed ItemGroup file
+		should(project.files.find(f => f.type === EntryType.File && f.relativePath === 'MyExternalStreamingJob.sql')).not.equal(undefined); // entry with custom attribute
+
 
 		// SqlCmdVariables
 		should(Object.keys(project.sqlCmdVariables).length).equal(2);
@@ -94,20 +96,26 @@ describe('Project: sqlproj content operations', function (): void {
 		const project = await Project.openProject(projFilePath);
 
 		const folderPath = 'Stored Procedures';
-		const filePath = path.join(folderPath, 'Fake Stored Proc.sql');
-		const fileContents = 'SELECT \'This is not actually a stored procedure.\'';
+		const scriptPath = path.join(folderPath, 'Fake Stored Proc.sql');
+		const scriptContents = 'SELECT \'This is not actually a stored procedure.\'';
+
+		const scriptPathTagged = path.join(folderPath, 'Fake External Streaming Job.sql');
+		const scriptContentsTagged = 'EXEC sys.sp_create_streaming_job \'job\', \'SELECT 7\'';
 
 		await project.addFolderItem(folderPath);
-		await project.addScriptItem(filePath, fileContents);
+		await project.addScriptItem(scriptPath, scriptContents);
+		await project.addScriptItem(scriptPathTagged, scriptContentsTagged, templates.externalStreamingJob);
 
 		const newProject = await Project.openProject(projFilePath);
 
 		should(newProject.files.find(f => f.type === EntryType.Folder && f.relativePath === convertSlashesForSqlProj(folderPath))).not.equal(undefined);
-		should(newProject.files.find(f => f.type === EntryType.File && f.relativePath === convertSlashesForSqlProj(filePath))).not.equal(undefined);
+		should(newProject.files.find(f => f.type === EntryType.File && f.relativePath === convertSlashesForSqlProj(scriptPath))).not.equal(undefined);
+		should(newProject.files.find(f => f.type === EntryType.File && f.relativePath === convertSlashesForSqlProj(scriptPathTagged))).not.equal(undefined);
+		should(newProject.files.find(f => f.type === EntryType.File && f.relativePath === convertSlashesForSqlProj(scriptPathTagged))?.sqlObjectType).equal(constants.ExternalStreamingJob);
 
-		const newFileContents = (await fs.readFile(path.join(newProject.projectFolderPath, filePath))).toString();
+		const newScriptContents = (await fs.readFile(path.join(newProject.projectFolderPath, scriptPath))).toString();
 
-		should(newFileContents).equal(fileContents);
+		should(newScriptContents).equal(scriptContents);
 	});
 
 	it('Should add Folder and Build entries to sqlproj with pre-existing scripts on disk', async function (): Promise<void> {

extensions/sql-database-projects/src/test/projectController.test.ts

@@ -85,7 +85,7 @@ describe('ProjectsController', function (): void {
 
 				const project = await projController.openProject(vscode.Uri.file(sqlProjPath));
 
-				should(project.files.length).equal(9); // detailed sqlproj tests in their own test file
+				should(project.files.length).equal(10); // detailed sqlproj tests in their own test file
 				should(project.dataSources.length).equal(3); // detailed datasources tests in their own test file
 			});
 

extensions/sql-database-projects/src/test/templates.test.ts

@@ -23,7 +23,7 @@ describe('Templates: loading templates from disk', function (): void {
 
 		// check expected counts
 
-		const numScriptObjectTypes = 6;
+		const numScriptObjectTypes = 7;
 
 		should(templates.projectScriptTypes().length).equal(numScriptObjectTypes);
 		should(Object.keys(templates.projectScriptTypes()).length).equal(numScriptObjectTypes);

extensions/sql-database-projects/src/test/testContext.ts

@@ -115,6 +115,7 @@ export class MockDacFxService implements mssql.IDacFxService {
 	public generateDeployScript(_: string, __: string, ___: string, ____: azdata.TaskExecutionMode, ______?: Record<string, string>): Thenable<mssql.DacFxResult> { return Promise.resolve(mockDacFxResult); }
 	public generateDeployPlan(_: string, __: string, ___: string, ____: azdata.TaskExecutionMode): Thenable<mssql.GenerateDeployPlanResult> { return Promise.resolve(mockDacFxResult); }
 	public getOptionsFromProfile(_: string): Thenable<mssql.DacFxOptionsResult> { return Promise.resolve(mockDacFxOptionsResult); }
+	public validateStreamingJob(_: string, __: string): Thenable<mssql.ValidateStreamingJobResult> { return Promise.resolve(mockDacFxResult); }
 }
 
 export function createContext(): TestContext {