Add more testing in the pipelines (#10455)

* run tests in every pipeline and include integration tests

* fix test env

* fix integration test scripts

* remove windows unit tests

* remove unused tests in integrations

* add comments

build/azure-pipelines/darwin/sql-product-build-darwin.yml

@@ -104,10 +104,23 @@ steps:
 
   - script: |
       set -e
-      ./scripts/test.sh --build --coverage --reporter mocha-junit-reporter
+      ./scripts/test.sh --build --coverage --reporter mocha-junit-reporter --tfs "Unit Tests"
     displayName: Run unit tests
     condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
 
+  - script: |
+      # Figure out the full absolute path of the product we just built
+      # including the remote server and configure the integration tests
+      # to run with these builds instead of running out of sources.
+      set -e
+      APP_ROOT=$(agent.builddirectory)/azuredatastudio-darwin
+      APP_NAME="`ls $APP_ROOT | head -n 1`"
+      INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
+      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/azuredatastudio-reh-darwin" \
+      ./scripts/test-integration.sh --build --tfs "Integration Tests"
+    displayName: Run integration tests (Electron)
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+
   - script: |
       set -e
       APP_ROOT=$(agent.builddirectory)/azuredatastudio-darwin

build/azure-pipelines/linux/sql-product-build-linux.yml

@@ -94,12 +94,6 @@ steps:
     env:
       VSCODE_MIXIN_PASSWORD: $(github-distro-mixin-password)
 
-  - script: |
-      set -e
-      service xvfb start
-    displayName: Start xvfb
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
-
   - script: |
       set -e
       yarn gulp package-rebuild-extensions
@@ -107,6 +101,31 @@ steps:
       yarn gulp package-external-extensions
     displayName: Package External extensions
 
+  - script: |
+      set -e
+      service xvfb start
+    displayName: Start xvfb
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+
+  - script: |
+      set -e
+      DISPLAY=:10 ./scripts/test.sh --build --coverage --reporter mocha-junit-reporter --tfs "Unit Tests"
+    displayName: Run unit tests (Electron)
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+
+  - script: |
+      # Figure out the full absolute path of the product we just built
+      # including the remote server and configure the integration tests
+      # to run with these builds instead of running out of sources.
+      set -e
+      APP_ROOT=$(agent.builddirectory)/azuredatastudio-linux-x64
+      APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
+      INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
+      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/azuredatastudio-reh-linux-x64" \
+      DISPLAY=:10 ./scripts/test-integration.sh --build --tfs "Integration Tests"
+    displayName: Run integration tests (Electron)
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+
   - script: |
       set -e
       APP_ROOT=$(agent.builddirectory)/azuredatastudio-linux-x64

build/azure-pipelines/win32/sql-product-build-win32.yml

@@ -103,6 +103,27 @@ steps:
     env:
       VSCODE_MIXIN_PASSWORD: $(github-distro-mixin-password)
 
+  # - powershell: | @anthonydresser unit tests timeout never existing the node process
+  #     . build/azure-pipelines/win32/exec.ps1
+  #     $ErrorActionPreference = "Stop"
+  #     exec { yarn electron x64 }
+  #     exec { .\scripts\test.bat --build --coverage --reporter mocha-junit-reporter --tfs "Unit Tests" }
+  #   displayName: Run unit tests (Electron)
+  #   condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+
+  - powershell: |
+      # Figure out the full absolute path of the product we just built
+      # including the remote server and configure the integration tests
+      # to run with these builds instead of running out of sources.
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      $AppRoot = "$(agent.builddirectory)\azuredatastudio-win32-x64"
+      $AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
+      $AppNameShort = $AppProductJson.nameShort
+      exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\azuredatastudio-reh-win32-x64"; .\scripts\test-integration.bat --build --tfs "Integration Tests" }
+    displayName: Run integration tests (Electron)
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+
   - powershell: |
       . build/azure-pipelines/win32/exec.ps1
       $ErrorActionPreference = "Stop"

scripts/test-integration.bat

@@ -19,13 +19,7 @@ if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" (
 	:: Run from a built: need to compile all test extensions
 	:: because we run extension tests from their source folders
 	:: and the build bundles extensions into .build webpacked
-	call yarn gulp 	compile-extension:vscode-api-tests^
+	call yarn gulp 	compile-extension:azurecore^
-					compile-extension:vscode-colorize-tests^
-					compile-extension:markdown-language-features^
-					compile-extension:emmet^
-					compile-extension:css-language-features-server^
-					compile-extension:html-language-features-server^
-					compile-extension:json-language-features-server^
 					compile-extension:git
 
 	:: Configuration for more verbose output
@@ -68,8 +62,8 @@ call "%INTEGRATION_TEST_ELECTRON_PATH%" %GITWORKSPACE% --extensionDevelopmentPat
 if %errorlevel% neq 0 exit /b %errorlevel%
 
 :: Tests in commonJS (HTML, CSS, JSON language server tests...)
-call .\scripts\node-electron.bat .\node_modules\mocha\bin\_mocha .\extensions\*\server\out\test\**\*.test.js
+REM call .\scripts\node-electron.bat .\node_modules\mocha\bin\_mocha .\extensions\*\server\out\test\**\*.test.js
-if %errorlevel% neq 0 exit /b %errorlevel%
+REM if %errorlevel% neq 0 exit /b %errorlevel%
 
 rmdir /s /q %VSCODEUSERDATADIR%
 

scripts/test-integration.sh

@@ -25,14 +25,7 @@ else
 	# Run from a built: need to compile all test extensions
 	# because we run extension tests from their source folders
 	# and the build bundles extensions into .build webpacked
-	yarn gulp 	compile-extension:vscode-api-tests \
+	yarn gulp 	compile-extension:azurecore \
-				compile-extension:vscode-colorize-tests \
-				compile-extension:vscode-notebook-tests \
-				compile-extension:markdown-language-features \
-				compile-extension:emmet \
-				compile-extension:css-language-features-server \
-				compile-extension:html-language-features-server \
-				compile-extension:json-language-features-server \
 				compile-extension:git
 
 	# Configuration for more verbose output
@@ -54,12 +47,11 @@ fi
 # "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $ROOT/extensions/vscode-colorize-tests/test --extensionDevelopmentPath=$ROOT/extensions/vscode-colorize-tests --extensionTestsPath=$ROOT/extensions/vscode-colorize-tests/out --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR
 # "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $ROOT/extensions/markdown-language-features/out/test/test-fixtures --extensionDevelopmentPath=$ROOT/extensions/markdown-language-features --extensionTestsPath=$ROOT/extensions/markdown-language-features/out/test --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR
 # "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $ROOT/extensions/emmet/out/test/test-fixtures --extensionDevelopmentPath=$ROOT/extensions/emmet --extensionTestsPath=$ROOT/extensions/emmet/out/test --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions  --user-data-dir=$VSCODEUSERDATADIR
-# "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $(mktemp -d 2>/dev/null) --enable-proposed-api=vscode.git --extensionDevelopmentPath=$ROOT/extensions/git --extensionTestsPath=$ROOT/extensions/git/out/test --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR
+"$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $(mktemp -d 2>/dev/null) --enable-proposed-api=vscode.git --extensionDevelopmentPath=$ROOT/extensions/git --extensionTestsPath=$ROOT/extensions/git/out/test --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR
-
 "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $ROOT/extensions/azurecore/test-fixtures --extensionDevelopmentPath=$ROOT/extensions/azurecore --extensionTestsPath=$ROOT/extensions/azurecore/out/test --disable-telemetry --disable-crash-reporter --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR
 
 # Tests in commonJS
-cd $ROOT/extensions/css-language-features/server && $ROOT/scripts/node-electron.sh test/index.js
+# cd $ROOT/extensions/css-language-features/server && $ROOT/scripts/node-electron.sh test/index.js
-cd $ROOT/extensions/html-language-features/server && $ROOT/scripts/node-electron.sh test/index.js
+# cd $ROOT/extensions/html-language-features/server && $ROOT/scripts/node-electron.sh test/index.js
 
 rm -r $VSCODEUSERDATADIR