Add script to combine coverage files (#10674)

* Add script to combine coverage files

* add script

* Add core coverage file as well

* Add json reporting

* Update README

* more

.github/workflows/ci.yml

@@ -51,16 +51,14 @@ jobs:
       name: Run Unit Tests (Electron)
     - run: DISPLAY=:10 ./scripts/test-extensions-unit.sh
       name: Run Extension Unit Tests (Electron)
-    # {{SQL CARBON EDIT}} Add coveralls. We merge first to get around issue where parallel builds weren't being combined correctly
-    - run: |
-        mkdir .build/coverage-combined
-        cat .build/coverage-single/lcov.info ./extensions/admin-tool-ext-win/coverage/lcov.info ./extensions/agent/coverage/lcov.info ./extensions/azurecore/coverage/lcov.info ./extensions/cms/coverage/lcov.info ./extensions/dacpac/coverage/lcov.info ./extensions/schema-compare/coverage/lcov.info ./extensions/notebook/coverage/lcov.info ./extensions/resource-deployment/coverage/lcov.info ./extensions/machine-learning/coverage/lcov.info > .build/coverage-combined/lcov.info
-      name: Merge coverage reports
+      # {{SQL CARBON EDIT}} Add coveralls. We merge first to get around issue where parallel builds weren't being combined correctly
+    - run: node test/combineCoverage
+      name: Combine code coverage files
     - name: Upload Code Coverage
       uses: coverallsapp/github-action@v1.1.1
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
-        path-to-lcov: '.build/coverage-combined/lcov.info'
+        path-to-lcov: 'test/coverage/lcov.info'
 
     # Fails with cryptic error (e.g. https://github.com/microsoft/vscode/pull/90292/checks?check_run_id=433681926#step:13:9)
     # - run: DISPLAY=:10 yarn test-browser --browser chromium

extensions/admin-tool-ext-win/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/agent/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/azurecore/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/cms/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/dacpac/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/integration-tests/coverConfig.json

@@ -11,7 +11,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/machine-learning/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/mssql/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/notebook/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/resource-deployment/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/schema-compare/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

extensions/sql-database-projects/coverConfig.json

@@ -9,7 +9,8 @@
 	"includePid": false,
 	"reports": [
 		"cobertura",
-		"lcov"
+		"lcov",
+		"json"
 	],
 	"verbose": false,
 	"remapOptions": {

test/README.md

@@ -24,3 +24,9 @@ Code coverage is enabled by default. Reports can be found in the coverage folder
 ### Integration tests
 
 These are the group of tests that have dependencies on outside resources such as SQL instances. See the [README](../extensions/integration-tests/readme.md) for more information.
+
+## Code Coverage
+
+To generate a report combining the code coverage for extensions + core run `node ./test/combineCoverage`. Currently this will combine coverage from the Core Unit Tests, Extension Unit Tests and Extension Integration tests - see above docs for instructions on how to run those to generate coverage.
+
+Once ran the combined coverage report will be located in `./test/coverage`.

test/combineCoverage.js

@@ -0,0 +1,36 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+const iLibCoverage = require('istanbul-lib-coverage');
+const iLibReport = require('istanbul-lib-report');
+const iReports = require('istanbul-reports');
+const fs = require('fs');
+const path = require('path');
+const glob = require('glob');
+
+const coverageMap = iLibCoverage.createCoverageMap();
+const repoRoot = path.join(path.dirname(__filename), '..');
+
+// .build contain the core coverage files
+// extensions each may container their own coverage file
+const coverageFiles = glob.sync('{extensions,.build}/**/coverage-final.json',
+	{
+		cwd: repoRoot,
+		ignore: ['**/node_modules/**']
+	}
+);
+
+coverageFiles.forEach(file => {
+	console.log(`Merging coverage file ${path.join(repoRoot, file)}`);
+	coverageMap.merge(JSON.parse(fs.readFileSync(file)));
+});
+
+const context = iLibReport.createContext({
+	dir: path.join(path.dirname(__filename), 'coverage'),
+	coverageMap: coverageMap
+});
+
+const tree = context.getTree('flat');
+tree.visit(iReports.create('lcov'), context);

test/unit/coverage.js

@@ -56,6 +56,7 @@ exports.createReport = function (isSingle) {
 		let reports = [];
 		if (isSingle) {
 			reports.push(iReports.create('lcovonly'));
+			reports.push(iReports.create('json')); // {{SQL CARBON EDIT}} add json for code coverage merging
 		} else {
 			reports.push(iReports.create('json'));
 			reports.push(iReports.create('lcov'));