Smoke tests in nodejs

.gitignore

@@ -1,7 +1,8 @@
 .DS_Store
 .huskyrc.json
 out
-node_modules
+log.log
+**/node_modules
 *.pyc
 *.vsix
 **/.vscode/.ropeproject/**
@@ -14,9 +15,10 @@ npm-debug.log
 **/.mypy_cache/**
 !yarn.lock
 coverage/
-.vscode-test/**
-.vscode test/**
-.vscode-smoke/**
+cucumber-report.json
+**/.vscode-test/**
+**/.vscode test/**
+**/.vscode-smoke/**
 **/.venv*/
 port.txt
 precommit.hook
@@ -25,11 +27,15 @@ pythonFiles/lib/**
 debug_coverage*/**
 languageServer/**
 languageServer.*/**
+!uitests/features/languageServer/**
+!uitests/src/languageServer/**
+!uitests/code/**/languageServer/**
 bin/**
 obj/**
 .pytest_cache
 tmp/**
 .python-version
 .vs/
 test-results.xml
+uitests/out/**
 !build/

.vscode/settings.json

@@ -2,16 +2,22 @@
 {
     "files.exclude": {
         "out": true, // set this to true to hide the "out" folder with the compiled JS files
+        "uitests/out": true,
         "**/*.pyc": true,
         ".nyc_output": true,
         "obj": true,
         "bin": true,
         "**/__pycache__": true,
+        "**/node_modules": true,
+        ".vscode-test": false,
+        ".vscode test": false,
         "**/.mypy_cache/**": true,
         "**/.ropeproject/**": true
     },
     "search.exclude": {
         "out": true, // set this to false to include "out" folder in search results
+        "uitests/out": true,
+        "**/node_modules": true,
         "coverage": true,
         "languageServer*/**": true,
         ".vscode-test": true,

.vscodeignore

@@ -18,6 +18,7 @@ CODING_STANDARDS.md
 CONTRIBUTING.md
 CONTRIBUTING - LANGUAGE SERVER.md
 coverconfig.json
+cucumber-report.json
 gulpfile.js
 package.datascience-ui.dependencies.json
 package-lock.json

build/ci/templates/jobs/build_compile.yml

@@ -14,8 +14,17 @@ jobs:
   steps:
     - template: ../steps/build.yml
 
-- job: Lint
+- job: Hygiene
   pool:
     vmImage: "macos-latest"
   steps:
-    - template: ../steps/lint.yml
+    - template: ../steps/initialization.yml
+      parameters:
+        PythonVersion: $(PythonVersion)
+        workingDirectory: $(Build.SourcesDirectory)
+        compile: 'false'
+        installVSCEorNPX: 'false'
+
+    - bash: npx tslint --project tsconfig.json
+      displayName: "Lint"
+      workingDirectory: $(Build.SourcesDirectory)

build/ci/templates/jobs/build_uitests.yml

@@ -0,0 +1,31 @@
+# Compile and build uitests
+
+jobs:
+- job: Compile
+  pool:
+    vmImage: "macos-latest"
+  steps:
+    - template: ../steps/initialization.yml
+      parameters:
+        workingDirectory: $(Build.SourcesDirectory)/uitests
+        compile: 'true'
+        installVSCEorNPX: 'false'
+
+
+- job: Hygiene
+  pool:
+    vmImage: "macos-latest"
+  steps:
+    - template: ../steps/initialization.yml
+      parameters:
+        workingDirectory: $(Build.SourcesDirectory)/uitests
+        compile: 'false'
+        installVSCEorNPX: 'false'
+
+    - bash: npx tslint --project tsconfig.json
+      displayName: "Lint"
+      workingDirectory: $(Build.SourcesDirectory)/uitests
+
+    - bash: npx prettier --check 'src/**/*.ts'
+      displayName: "Format"
+      workingDirectory: $(Build.SourcesDirectory)/uitests

build/ci/templates/jobs/merge_upload_uitest_report.yml

@@ -0,0 +1,47 @@
+# Overview:
+# Generic jobs template to compile and build extension
+
+jobs:
+- job: UI_Test_Report
+  timeoutInMinutes: 90
+  pool:
+    vmImage: "macos-latest"
+  steps:
+    - template: ../steps/initialization.yml
+      parameters:
+        workingDirectory: $(Build.SourcesDirectory)/uitests
+        compile: 'true'
+
+    - bash: mkdir -p reports
+      workingDirectory: $(Build.SourcesDirectory)/uitests
+      displayName: "Create Reports Directory"
+
+    - task: DownloadBuildArtifacts@0
+      inputs:
+          buildType: "current"
+          allowPartiallySucceededBuilds: true
+          downloadType: "Specific"
+          itemPattern: "**/.vscode test/reports/cucumber_report_*.json"
+          downloadPath: "$(Build.SourcesDirectory)/uitests/reports"
+      displayName: "Restore Cucumber Reports"
+      condition: always()
+
+    - bash: node ./out/index.js report --jsonDir=./reports --htmlOutput=./reports
+      workingDirectory: $(Build.SourcesDirectory)/uitests
+      displayName: "Merge and generate report"
+      condition: always()
+
+    - task: CopyFiles@2
+      inputs:
+          sourceFolder: $(Build.SourcesDirectory)/uitests/reports
+          contents: "**"
+          targetFolder: $(Build.ArtifactStagingDirectory)
+      displayName: "Copy Report"
+      condition: always()
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+          pathtoPublish: $(Build.ArtifactStagingDirectory)
+          artifactName: UIReport
+      displayName: "Publish Report"
+      condition: always()

build/ci/templates/jobs/uitest.yml

@@ -25,18 +25,18 @@
 #   parameters:
 #     jobs:
 #     - test: "Smoke"
-#       tags: "--tags=@smoke"
+#       tags: "@smoke"
 #     - test: "Test"
-#       tags: "--tags=@test"
+#       tags: "@test"
 #     - test: "Terminal"
-#       tags: "--tags=@terminal"
+#       tags: "@terminal"
 #   ```
 #   Based on this sample, we're running 3 tests with the names `Smoke`, `Test`, and `Terminal`.
 #   The tags inside each test contains the arguments that needs to be passd into behave.
 #   I.e. we're only testing BDD tests that contain the tag `@smoke`, `@test` & `@terminal` (as separate jobs).
 #   Please pass in just the `tags` arguments.
 #   Multiple tag values can be passed in as follows:
-#       tags: "--tags=@debug --tags=@remote"
+#       tags: "@debug and @remote"
 #   More information on --tags argument for behave can be found here:
 #       * https://behave.readthedocs.io/en/latest/tutorial.html#controlling-things-with-tags
 #       * https://behave.readthedocs.io/en/latest/tag_expressions.html
@@ -52,7 +52,7 @@
 #   vscodeChannels: ['stable']
 #   jobs:
 #   - test: "Smoke"
-#     tags: "--tags=@smoke"
+#     tags: "@smoke"
 #     ignorePythonVersions: "3.6,3.5"
 #   ```
 #   Based on this sample, we're running 1 test with the name `Smoke`.
@@ -73,7 +73,7 @@
 #   vscodeChannels: ['stable']
 #   jobs:
 #   - test: "Smoke"
-#     tags: "--tags=@smoke"
+#     tags: "@smoke"
 #     ignorePythonVersions: "3.6,3.5"
 #   ```
 #   Based on this sample, we're running 1 test with the name `Smoke`.
@@ -93,36 +93,33 @@ parameters:
   vscodeChannels: ['stable', 'insider']
   pythonVersions: [
       {
-          "version": "3.7",
+          "version": "3.7.4",
           "displayName": "37",
-          "excludeTags": "--tags=~@python3.6 --tags=~@python3.5 --tags=~@python2"
+          "excludeTags": "not @python3.6 and not @python3.5 and not @python2"
       },
       {
           "version": "3.6",
           "displayName": "36",
-          "excludeTags": "--tags=~@python3.7 --tags=~@python3.5 --tags=~@python2"
+          "excludeTags": "not @python3.7 and not @python3.5 and not @python2 and not @noNeedToTestInAllPython"
       },
       {
           "version": "3.5",
           "displayName": "35",
-          "excludeTags": "--tags=~@python3.7 --tags=~@python3.6 --tags=~@python2"
+          "excludeTags": "not @python3.7 and not @python3.6 and not @python2 and not @noNeedToTestInAllPython"
       },
       {
           "version": "2.7",
           "displayName": "27",
-          "excludeTags": "--tags=~@python3.7 --tags=~@python3.5 --tags=~@python3"
+          "excludeTags": "not @python3.7 and not @python3.5 and not @python3 and not @noNeedToTestInAllPython"
       }
     ]
 
 
 jobs:
 - job: UITest
-  dependsOn:
-    - Compile
-    - Build
   # Remember, some tests can take easily an hour (the `tests` features take just around 1 hour).
   timeoutInMinutes: 90
-  # Build our matrix (permutations of all environments & tests).
+  # Build our matrix (permutations of VS Code + Tests + Pyhton + OS).
   strategy:
     matrix:
       ${{ each channel in parameters.vscodeChannels }}:
@@ -134,24 +131,24 @@ jobs:
                     PythonVersion: ${{ py.version }}
                     VMImageName: "macos-latest"
                     VSCodeChannel: ${{ channel }}
-                    Tags: ${{ format('{0} {1} --tags=~@win --tags=~@linux', job.tags, py.excludeTags) }}
+                    Tags: ${{ format('{0} and {1} and not @win and not @linux', job.tags, py.excludeTags) }}
 
               ${{ if not(contains(coalesce(job.ignoreOperatingSystems, ''), 'win')) }}:
                 ${{ format('Win{2}{0}{1}', py.displayName, job.test, channel) }}:
                     PythonVersion: ${{ py.version }}
                     VSCodeChannel: ${{ channel }}
                     VMImageName: "vs2017-win2016"
-                    Tags: ${{ format('{0} {1} --tags=~@mac --tags=~@linux', job.tags, py.excludeTags) }}
+                    Tags: ${{ format('{0} and {1} and not @mac and not @linux', job.tags, py.excludeTags) }}
 
               ${{ if not(contains(coalesce(job.ignoreOperatingSystems, ''), 'linux')) }}:
                 ${{ format('Linux{2}{0}{1}', py.displayName, job.test, channel) }}:
                     PythonVersion: ${{ py.version }}
                     VSCodeChannel: ${{ channel }}
                     VMImageName: "ubuntu-latest"
-                    Tags: ${{ format('{0} {1} --tags=~@mac --tags=~@win', job.tags, py.excludeTags) }}
+                    Tags: ${{ format('{0} and {1} and not @mac and not @win', job.tags, py.excludeTags) }}
 
   pool:
     vmImage: $(VMImageName)
 
   steps:
-    - template: uitest_phases.yml
+    - template: ../steps/uitest.yml