Remove sql project build dependency on STS (#20447)

* download Microsoft.Build.Sql sdk and extract

* cleanup extracted folder and nuget

* add constants

* cleanup

* remove package-lock.json

* making outputChannel required and some cleanup

* only download if the files aren't already there

* Add todo

* add try catches

* addressing comments

extensions/sql-database-projects/src/common/constants.ts

@@ -615,3 +615,14 @@ export enum PublishTargetType {
 }
 
 export const CollapseProjectNodesKey = 'collapseProjectNodes';
+
+// httpClient
+export const downloadError = localize('downloadError', "Download error");
+export const downloadProgress = localize('downloadProgress', "Download progress");
+export const downloading = localize('downloading', "Downloading");
+
+// buildHelper
+export const downloadingDacFxDlls = localize('downloadingDacFxDlls', "Downloading Microsoft.Build.Sql nuget to get build DLLs");
+export const extractingDacFxDlls = localize('extractingDacFxDlls', "Extracting DacFx build DLLs");
+export function errorDownloading(url: string, error: string) { return localize('errorDownloading', "Error downloading {0}. Error: {1}", url, error); }
+export function errorExtracting(path: string, error: string) { return localize('errorExtracting', "Error extracting files from {0}. Error: {1}", path, error); }

extensions/sql-database-projects/src/common/httpClient.ts

@@ -4,7 +4,13 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as os from 'os';
+import * as fs from 'fs';
+import * as request from 'request';
+import * as vscode from 'vscode';
 import axios, { AxiosRequestConfig } from 'axios';
+import * as constants from '../common/constants';
+
+const DownloadTimeoutMs = 20000;
 
 /**
  * Class includes method for making http request
@@ -48,4 +54,53 @@ export class HttpClient {
 		}
 		return response.data;
 	}
+
+	/**
+	 * Gets a file/fileContents at the given URL. Function is copied from Machine Learning extension extensions/machine-learning/src/common/httpClient.ts
+	 * @param downloadUrl The URL to download the file from
+	 * @param targetPath The path to download the file to
+	 * @param outputChannel The output channel to output status messages to
+	 * @returns Full path to the downloaded file or the contents of the file at the given downloadUrl
+	 */
+	public download(downloadUrl: string, targetPath: string, outputChannel?: vscode.OutputChannel): Promise<void> {
+		return new Promise((resolve, reject) => {
+			let totalMegaBytes: number | undefined = undefined;
+			let receivedBytes = 0;
+			let printThreshold = 0.1;
+			let downloadRequest = request.get(downloadUrl, { timeout: DownloadTimeoutMs })
+				.on('error', downloadError => {
+					outputChannel?.appendLine(constants.downloadError);
+					reject(downloadError);
+				})
+				.on('response', (response) => {
+					if (response.statusCode !== 200) {
+						outputChannel?.appendLine(constants.downloadError);
+						return reject(response.statusMessage);
+					}
+					let contentLength = response.headers['content-length'];
+					let totalBytes = parseInt(contentLength || '0');
+					totalMegaBytes = totalBytes / (1024 * 1024);
+					outputChannel?.appendLine(`${constants.downloading} ${downloadUrl} (0 / ${totalMegaBytes.toFixed(2)} MB)`);
+				})
+				.on('data', (data) => {
+					receivedBytes += data.length;
+					if (totalMegaBytes) {
+						let receivedMegaBytes = receivedBytes / (1024 * 1024);
+						let percentage = receivedMegaBytes / totalMegaBytes;
+						if (percentage >= printThreshold) {
+							outputChannel?.appendLine(`${constants.downloadProgress} (${receivedMegaBytes.toFixed(2)} / ${totalMegaBytes.toFixed(2)} MB)`);
+							printThreshold += 0.1;
+						}
+					}
+				});
+			downloadRequest.pipe(fs.createWriteStream(targetPath))
+				.on('close', async () => {
+					resolve();
+				})
+				.on('error', (downloadError) => {
+					reject(downloadError);
+					downloadRequest.abort();
+				});
+		});
+	}
 }

extensions/sql-database-projects/src/controllers/projectController.ts

@@ -230,8 +230,10 @@ export class ProjectsController {
 			this.buildInfo.shift();		// Remove the first element to maintain the length
 		}
 
-		// Check mssql extension for project dlls (tracking issue #10273)
-		await this.buildHelper.createBuildDirFolder();
+		// get dlls and targets file needed for building for legacy style projects
+		if (!project.isSdkStyleProject) {
+			await this.buildHelper.createBuildDirFolder(this._outputChannel);
+		}
 
 		const options: ShellCommandOptions = {
 			commandTitle: 'Build',

extensions/sql-database-projects/src/test/buildHelper.test.ts

@@ -8,6 +8,7 @@ import * as os from 'os';
 import * as vscode from 'vscode';
 import * as path from 'path';
 import { BuildHelper } from '../tools/buildHelper';
+import { TestContext, createContext } from './testContext';
 
 describe('BuildHelper: Build Helper tests', function (): void {
 
@@ -38,8 +39,9 @@ describe('BuildHelper: Build Helper tests', function (): void {
 	});
 
 	it('Should get correct build folder', async function (): Promise<void> {
+		const testContext: TestContext = createContext();
 		const buildHelper = new BuildHelper();
-		await buildHelper.createBuildDirFolder();
+		await buildHelper.createBuildDirFolder(testContext.outputChannel);
 
 		// get expected path for build
 		let expectedPath = vscode.extensions.getExtension('Microsoft.sql-database-projects')?.extensionPath ?? 'EmptyPath';
@@ -48,4 +50,3 @@ describe('BuildHelper: Build Helper tests', function (): void {
 	});
 });
 
-

extensions/sql-database-projects/src/tools/buildHelper.ts

@@ -7,12 +7,17 @@ import * as vscode from 'vscode';
 import * as path from 'path';
 import { promises as fs } from 'fs';
 import * as utils from '../common/utils';
-import { errorFindingBuildFilesLocation } from '../common/constants';
-import * as mssql from 'mssql';
-import * as vscodeMssql from 'vscode-mssql';
 import * as sqldbproj from 'sqldbproj';
+import * as extractZip from 'extract-zip';
+import * as constants from '../common/constants';
+import { HttpClient } from '../common/httpClient';
 
 const buildDirectory = 'BuildDirectory';
+const sdkName = 'Microsoft.Build.Sql';
+const microsoftBuildSqlVersion = '0.1.3-preview'; // TODO: have this be configurable
+const fullSdkName = `${sdkName}.${microsoftBuildSqlVersion}`;
+const microsoftBuildSqlUrl = `https://www.nuget.org/api/v2/package/${sdkName}/${microsoftBuildSqlVersion}`;
+
 const buildFiles: string[] = [
 	'Microsoft.Data.SqlClient.dll',
 	'Microsoft.Data.Tools.Schema.Sql.dll',
@@ -39,9 +44,11 @@ export class BuildHelper {
 		this.extensionBuildDir = path.join(this.extensionDir, buildDirectory);
 	}
 
-	// create build dlls directory
-	// this should not be required. temporary solution for issue #10273
-	public async createBuildDirFolder(): Promise<void> {
+	/**
+	 * Create build dlls directory with the dlls and targets needed for building a sqlproj
+	 * @param outputChannel
+	 */
+	public async createBuildDirFolder(outputChannel: vscode.OutputChannel): Promise<void> {
 
 		if (this.initialized) {
 			return;
@@ -51,29 +58,61 @@ export class BuildHelper {
 			await fs.mkdir(this.extensionBuildDir);
 		}
 
-		const buildfilesPath = await this.getBuildDirPathFromMssqlTools();
-		buildFiles.forEach(async (fileName) => {
+		// check if this if the nuget needs to be downloaded
+		const nugetPath = path.join(this.extensionBuildDir, `${fullSdkName}.nupkg`);
+
+		if (await utils.exists(nugetPath)) {
+			// if it does exist, make sure all the necessary files are also in the BuildDirectory
+			let missingFiles = false;
+			for (const fileName of buildFiles) {
+				if (!await (utils.exists(path.join(this.extensionBuildDir, fileName)))) {
+					missingFiles = true;
+					break;
+				}
+			}
+
+			// if all the files are there, no need to continue
+			if (!missingFiles) {
+				return;
+			}
+		}
+
+		// TODO: check nuget cache locations first to avoid downloading if those exist
+		// download the Microsoft.Build.Sql sdk nuget
+		outputChannel.appendLine(constants.downloadingDacFxDlls);
+
+		try {
+			const httpClient = new HttpClient();
+			await httpClient.download(microsoftBuildSqlUrl, nugetPath, outputChannel);
+		} catch (e) {
+			void vscode.window.showErrorMessage(constants.errorDownloading(microsoftBuildSqlUrl, utils.getErrorMessage(e)));
+			return;
+		}
+
+		// extract the files from the nuget
+		outputChannel.appendLine(constants.extractingDacFxDlls);
+		const extractedFolderPath = path.join(this.extensionDir, buildDirectory, sdkName);
+
+		try {
+			await extractZip(nugetPath, { dir: extractedFolderPath });
+		} catch (e) {
+			void vscode.window.showErrorMessage(constants.errorExtracting(nugetPath, utils.getErrorMessage(e)));
+			return;
+		}
+
+		// copy the dlls and targets file to the BuildDirectory folder
+		const buildfilesPath = path.join(extractedFolderPath, 'tools', 'netstandard2.1');
+
+		for (const fileName of buildFiles) {
 			if (await (utils.exists(path.join(buildfilesPath, fileName)))) {
 				await fs.copyFile(path.join(buildfilesPath, fileName), path.join(this.extensionBuildDir, fileName));
 			}
-		});
-		this.initialized = true;
-	}
-
-	/**
-	 * Gets the path to the SQL Tools Service installation
-	 * @returns
-	 */
-	private async getBuildDirPathFromMssqlTools(): Promise<string> {
-		try {
-			if (utils.getAzdataApi()) {
-				return (vscode.extensions.getExtension(mssql.extension.name)?.exports as mssql.IExtension).sqlToolsServicePath;
-			} else {
-				return (vscode.extensions.getExtension(vscodeMssql.extension.name)?.exports as vscodeMssql.IExtension).sqlToolsServicePath;
-			}
-		} catch (err) {
-			throw new Error(errorFindingBuildFilesLocation(err));
 		}
+
+		// cleanup extracted folder
+		await fs.rm(extractedFolderPath, { recursive: true });
+
+		this.initialized = true;
 	}
 
 	public get extensionBuildDirPath(): string {