From 4763685237d27d065c5d01d407ba6bac1ef56642 Mon Sep 17 00:00:00 2001 From: Anthony Dresser Date: Mon, 18 May 2020 13:54:19 -0700 Subject: [PATCH] Add more testing in the pipelines (#10455) * run tests in every pipeline and include integration tests * fix test env * fix integration test scripts * remove windows unit tests * remove unused tests in integrations * add comments --- .../darwin/sql-product-build-darwin.yml | 15 ++++++++- .../linux/sql-product-build-linux.yml | 31 +++++++++++++++---- .../win32/sql-product-build-win32.yml | 21 +++++++++++++ scripts/test-integration.bat | 12 ++----- scripts/test-integration.sh | 16 +++------- 5 files changed, 67 insertions(+), 28 deletions(-) diff --git a/build/azure-pipelines/darwin/sql-product-build-darwin.yml b/build/azure-pipelines/darwin/sql-product-build-darwin.yml index 0044b0e206..74b452f167 100644 --- a/build/azure-pipelines/darwin/sql-product-build-darwin.yml +++ b/build/azure-pipelines/darwin/sql-product-build-darwin.yml @@ -104,10 +104,23 @@ steps: - script: | set -e - ./scripts/test.sh --build --coverage --reporter mocha-junit-reporter + ./scripts/test.sh --build --coverage --reporter mocha-junit-reporter --tfs "Unit Tests" displayName: Run unit tests condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true')) + - script: | + # Figure out the full absolute path of the product we just built + # including the remote server and configure the integration tests + # to run with these builds instead of running out of sources. + set -e + APP_ROOT=$(agent.builddirectory)/azuredatastudio-darwin + APP_NAME="`ls $APP_ROOT | head -n 1`" + INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \ + VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/azuredatastudio-reh-darwin" \ + ./scripts/test-integration.sh --build --tfs "Integration Tests" + displayName: Run integration tests (Electron) + condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true')) + - script: | set -e APP_ROOT=$(agent.builddirectory)/azuredatastudio-darwin diff --git a/build/azure-pipelines/linux/sql-product-build-linux.yml b/build/azure-pipelines/linux/sql-product-build-linux.yml index 00a8425a64..f42d50f554 100644 --- a/build/azure-pipelines/linux/sql-product-build-linux.yml +++ b/build/azure-pipelines/linux/sql-product-build-linux.yml @@ -94,12 +94,6 @@ steps: env: VSCODE_MIXIN_PASSWORD: $(github-distro-mixin-password) - - script: | - set -e - service xvfb start - displayName: Start xvfb - condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - - script: | set -e yarn gulp package-rebuild-extensions @@ -107,6 +101,31 @@ steps: yarn gulp package-external-extensions displayName: Package External extensions + - script: | + set -e + service xvfb start + displayName: Start xvfb + condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true')) + + - script: | + set -e + DISPLAY=:10 ./scripts/test.sh --build --coverage --reporter mocha-junit-reporter --tfs "Unit Tests" + displayName: Run unit tests (Electron) + condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true')) + + - script: | + # Figure out the full absolute path of the product we just built + # including the remote server and configure the integration tests + # to run with these builds instead of running out of sources. + set -e + APP_ROOT=$(agent.builddirectory)/azuredatastudio-linux-x64 + APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName") + INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \ + VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/azuredatastudio-reh-linux-x64" \ + DISPLAY=:10 ./scripts/test-integration.sh --build --tfs "Integration Tests" + displayName: Run integration tests (Electron) + condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true')) + - script: | set -e APP_ROOT=$(agent.builddirectory)/azuredatastudio-linux-x64 diff --git a/build/azure-pipelines/win32/sql-product-build-win32.yml b/build/azure-pipelines/win32/sql-product-build-win32.yml index 237b733e17..ea76541515 100644 --- a/build/azure-pipelines/win32/sql-product-build-win32.yml +++ b/build/azure-pipelines/win32/sql-product-build-win32.yml @@ -103,6 +103,27 @@ steps: env: VSCODE_MIXIN_PASSWORD: $(github-distro-mixin-password) + # - powershell: | @anthonydresser unit tests timeout never existing the node process + # . build/azure-pipelines/win32/exec.ps1 + # $ErrorActionPreference = "Stop" + # exec { yarn electron x64 } + # exec { .\scripts\test.bat --build --coverage --reporter mocha-junit-reporter --tfs "Unit Tests" } + # displayName: Run unit tests (Electron) + # condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true')) + + - powershell: | + # Figure out the full absolute path of the product we just built + # including the remote server and configure the integration tests + # to run with these builds instead of running out of sources. + . build/azure-pipelines/win32/exec.ps1 + $ErrorActionPreference = "Stop" + $AppRoot = "$(agent.builddirectory)\azuredatastudio-win32-x64" + $AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json + $AppNameShort = $AppProductJson.nameShort + exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\azuredatastudio-reh-win32-x64"; .\scripts\test-integration.bat --build --tfs "Integration Tests" } + displayName: Run integration tests (Electron) + condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true')) + - powershell: | . build/azure-pipelines/win32/exec.ps1 $ErrorActionPreference = "Stop" diff --git a/scripts/test-integration.bat b/scripts/test-integration.bat index c07b93569c..11623c2e60 100755 --- a/scripts/test-integration.bat +++ b/scripts/test-integration.bat @@ -19,13 +19,7 @@ if "%INTEGRATION_TEST_ELECTRON_PATH%"=="" ( :: Run from a built: need to compile all test extensions :: because we run extension tests from their source folders :: and the build bundles extensions into .build webpacked - call yarn gulp compile-extension:vscode-api-tests^ - compile-extension:vscode-colorize-tests^ - compile-extension:markdown-language-features^ - compile-extension:emmet^ - compile-extension:css-language-features-server^ - compile-extension:html-language-features-server^ - compile-extension:json-language-features-server^ + call yarn gulp compile-extension:azurecore^ compile-extension:git :: Configuration for more verbose output @@ -68,8 +62,8 @@ call "%INTEGRATION_TEST_ELECTRON_PATH%" %GITWORKSPACE% --extensionDevelopmentPat if %errorlevel% neq 0 exit /b %errorlevel% :: Tests in commonJS (HTML, CSS, JSON language server tests...) -call .\scripts\node-electron.bat .\node_modules\mocha\bin\_mocha .\extensions\*\server\out\test\**\*.test.js -if %errorlevel% neq 0 exit /b %errorlevel% +REM call .\scripts\node-electron.bat .\node_modules\mocha\bin\_mocha .\extensions\*\server\out\test\**\*.test.js +REM if %errorlevel% neq 0 exit /b %errorlevel% rmdir /s /q %VSCODEUSERDATADIR% diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index 20e9d20d2c..390135260d 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -25,14 +25,7 @@ else # Run from a built: need to compile all test extensions # because we run extension tests from their source folders # and the build bundles extensions into .build webpacked - yarn gulp compile-extension:vscode-api-tests \ - compile-extension:vscode-colorize-tests \ - compile-extension:vscode-notebook-tests \ - compile-extension:markdown-language-features \ - compile-extension:emmet \ - compile-extension:css-language-features-server \ - compile-extension:html-language-features-server \ - compile-extension:json-language-features-server \ + yarn gulp compile-extension:azurecore \ compile-extension:git # Configuration for more verbose output @@ -54,12 +47,11 @@ fi # "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $ROOT/extensions/vscode-colorize-tests/test --extensionDevelopmentPath=$ROOT/extensions/vscode-colorize-tests --extensionTestsPath=$ROOT/extensions/vscode-colorize-tests/out --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR # "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $ROOT/extensions/markdown-language-features/out/test/test-fixtures --extensionDevelopmentPath=$ROOT/extensions/markdown-language-features --extensionTestsPath=$ROOT/extensions/markdown-language-features/out/test --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR # "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $ROOT/extensions/emmet/out/test/test-fixtures --extensionDevelopmentPath=$ROOT/extensions/emmet --extensionTestsPath=$ROOT/extensions/emmet/out/test --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR -# "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $(mktemp -d 2>/dev/null) --enable-proposed-api=vscode.git --extensionDevelopmentPath=$ROOT/extensions/git --extensionTestsPath=$ROOT/extensions/git/out/test --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR - +"$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $(mktemp -d 2>/dev/null) --enable-proposed-api=vscode.git --extensionDevelopmentPath=$ROOT/extensions/git --extensionTestsPath=$ROOT/extensions/git/out/test --disable-telemetry --crash-reporter-directory=$VSCODECRASHDIR --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR "$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_NO_SANDBOX $ROOT/extensions/azurecore/test-fixtures --extensionDevelopmentPath=$ROOT/extensions/azurecore --extensionTestsPath=$ROOT/extensions/azurecore/out/test --disable-telemetry --disable-crash-reporter --disable-updates --disable-extensions --user-data-dir=$VSCODEUSERDATADIR # Tests in commonJS -cd $ROOT/extensions/css-language-features/server && $ROOT/scripts/node-electron.sh test/index.js -cd $ROOT/extensions/html-language-features/server && $ROOT/scripts/node-electron.sh test/index.js +# cd $ROOT/extensions/css-language-features/server && $ROOT/scripts/node-electron.sh test/index.js +# cd $ROOT/extensions/html-language-features/server && $ROOT/scripts/node-electron.sh test/index.js rm -r $VSCODEUSERDATADIR