Add var to let smoke tests fail build (#18677)

* Add var to let smoke tests fail build

* force failure

* try another

* fix

* undo failure

* update titles

build/azure-pipelines/darwin/sql-product-build-darwin.yml

@@ -128,14 +128,24 @@ steps:
       yarn gulp compile-extensions
     displayName: Compile Extensions
 
+  # Per https://developercommunity.visualstudio.com/t/variablesexpressions-dont-work-with-continueonerro/1187733 we can't use variables
+  # in continueOnError directly so instead make two copies of the task and only run one or the other based on the SMOKE_FAIL_ON_ERROR value
   - script: |
       set -e
       APP_ROOT=$(agent.builddirectory)/azuredatastudio-darwin-x64
       APP_NAME="`ls $APP_ROOT | head -n 1`"
       yarn smoketest --build "$APP_ROOT/$APP_NAME" --screenshots "$(build.artifactstagingdirectory)/smokeshots" --log "$(build.artifactstagingdirectory)/logs/darwin/smoke.log" --extensionsDir "$(build.sourcesdirectory)/extensions"
-    displayName: Run smoke tests (Electron)
+    displayName: Run smoke tests (Electron) (Continue on Error)
     continueOnError: true
-    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+    condition: and(succeeded(), and(eq(variables['RUN_TESTS'], 'true'), ne(variables['SMOKE_FAIL_ON_ERROR'], 'true')))
+
+  - script: |
+      set -e
+      APP_ROOT=$(agent.builddirectory)/azuredatastudio-darwin-x64
+      APP_NAME="`ls $APP_ROOT | head -n 1`"
+      yarn smoketest --build "$APP_ROOT/$APP_NAME" --screenshots "$(build.artifactstagingdirectory)/smokeshots" --log "$(build.artifactstagingdirectory)/logs/darwin/smoke.log" --extensionsDir "$(build.sourcesdirectory)/extensions"
+    displayName: Run smoke tests (Electron) (Fail on Error)
+    condition: and(succeeded(), and(eq(variables['RUN_TESTS'], 'true'), eq(variables['SMOKE_FAIL_ON_ERROR'], 'true')))
 
   # - script: |
   #     set -e

build/azure-pipelines/web/sql-product-build-web.yml

@@ -105,14 +105,25 @@ steps:
   displayName: Compile Extensions
 
 # disable smoke tests (karlb 3/2/2022)
+# # Per https://developercommunity.visualstudio.com/t/variablesexpressions-dont-work-with-continueonerro/1187733 we can't use variables
+# # in continueOnError directly so instead make two copies of the task and only run one or the other based on the SMOKE_FAIL_ON_ERROR value
 # - script: |
 #     set -e
 #     node ./node_modules/playwright/install.js
 #     APP_ROOT=$(Agent.BuildDirectory)/vscode-reh-web-linux-x64
 #     xvfb-run yarn smoketest --build "$(Agent.BuildDirectory)/vscode-reh-web-linux-x64" --web --headless --screenshots "$(Build.ArtifactStagingDirectory)/smokeshots" --log "$(Build.ArtifactStagingDirectory)/logs/web/smoke.log"
-#   displayName: Run smoke tests (Browser)
+#   displayName: Run smoke tests (Browser) (Continue on Error)
 #   continueOnError: true
-#   condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+#   condition: and(succeeded(), and(eq(variables['RUN_TESTS'], 'true'), ne(variables['SMOKE_FAIL_ON_ERROR'], 'true')))
+
+# disable smoke tests (karlb 3/2/2022)
+# - script: |
+#     set -e
+#     node ./node_modules/playwright/install.js
+#     APP_ROOT=$(Agent.BuildDirectory)/vscode-reh-web-linux-x64
+#     xvfb-run yarn smoketest --build "$(Agent.BuildDirectory)/vscode-reh-web-linux-x64" --web --headless --screenshots "$(Build.ArtifactStagingDirectory)/smokeshots" --log "$(Build.ArtifactStagingDirectory)/logs/web/smoke.log"
+#   displayName: Run smoke tests (Browser) (Fail on Error)
+#   condition: and(succeeded(), and(eq(variables['RUN_TESTS'], 'true'), eq(variables['SMOKE_FAIL_ON_ERROR'], 'true')))
 
 # - script: |
 #     set -e