master update

.editorconfig

@@ -9,6 +9,6 @@ trim_trailing_whitespace = true
 indent_style = space
 indent_size = 4
 
-[{package.json,.travis.yml}]
+[{package.json,.travis.yml,nightwatch.json}]
 indent_style = space
 indent_size = 2

.eslintrc.json

@@ -1,10 +1,12 @@
 {
+    "parser": "babel-eslint",
     "parserOptions": {
         "ecmaVersion": 9,
         "ecmaFeatures": {
             "impliedStrict": true
         },
-        "sourceType": "module"
+        "sourceType": "module",
+        "allowImportExportEverywhere": true
     },
     "env": {
         "browser": true,
@@ -87,6 +89,15 @@
         "no-var": "error",
         "prefer-const": "error"
     },
+    "overrides": [
+        {
+            "files": "tests/**/*",
+            "rules": {
+                "no-unused-expressions": "off",
+                "no-console": "off"
+            }
+        }
+    ],
     "globals": {
         "$": false,
         "jQuery": false,

.gitignore

@@ -9,4 +9,5 @@ docs/*
 src/core/config/modules/*
 src/core/config/OperationConfig.json
 src/core/operations/index.mjs
+tests/browser/output/*
 

.travis.yml

@@ -1,15 +1,19 @@
 language: node_js
 node_js:
   - node
+addons:
+  chrome: stable
 install: npm install
 before_script:
   - npm install -g grunt
+  - export NODE_OPTIONS=--max_old_space_size=2048
 script:
   - grunt lint
   - grunt test
   - grunt docs
   - grunt node
   - grunt prod --msg="$COMPILE_MSG"
+  - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui
 before_deploy:
   - grunt exec:sitemap
   - grunt copy:ghPages

CHANGELOG.md

@@ -2,6 +2,36 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+### [8.24.0] - 2019-02-08
+- 'DNS over HTTPS' operation added [@h345983745] | [#489]
+
+### [8.23.1] - 2019-01-18
+- 'Convert co-ordinate format' operation added [@j433866] | [#476]
+
+### [8.23.0] - 2019-01-18
+- 'YARA Rules' operation added [@artemisbot] | [#468]
+
+### [8.22.0] - 2019-01-10
+- 'Subsection' operation added [@j433866] | [#467]
+
+### [8.21.0] - 2019-01-10
+- 'To Case Insensitive Regex' and 'From Case Insensitive Regex' operations added [@masq] | [#461]
+
+### [8.20.0] - 2019-01-09
+- 'Generate Lorem Ipsum' operation added [@klaxon1] | [#455]
+
+### [8.19.0] - 2018-12-30
+- UI test suite added to confirm that the app loads correctly in a reasonable time and that various operations from each module can be run [@n1474335] | [#458]
+
+### [8.18.0] - 2018-12-26
+- 'Split Colour Channels' operation added [@artemisbot] | [#449]
+
+### [8.17.0] - 2018-12-25
+- 'Generate QR Code' and 'Parse QR Code' operations added [@j433866] | [#448]
+
+### [8.16.0] - 2018-12-19
+- 'Play Media' operation added [@anthony-arnold] | [#446]
+
 ### [8.15.0] - 2018-12-18
 - 'Text Encoding Brute Force' operation added [@Cynser] | [#439]
 
@@ -76,6 +106,16 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[8.24.0]: https://github.com/gchq/CyberChef/releases/tag/v8.24.0
+[8.23.1]: https://github.com/gchq/CyberChef/releases/tag/v8.23.1
+[8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0
+[8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0
+[8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0
+[8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0
+[8.19.0]: https://github.com/gchq/CyberChef/releases/tag/v8.19.0
+[8.18.0]: https://github.com/gchq/CyberChef/releases/tag/v8.18.0
+[8.17.0]: https://github.com/gchq/CyberChef/releases/tag/v8.17.0
+[8.16.0]: https://github.com/gchq/CyberChef/releases/tag/v8.16.0
 [8.15.0]: https://github.com/gchq/CyberChef/releases/tag/v8.15.0
 [8.14.0]: https://github.com/gchq/CyberChef/releases/tag/v8.14.0
 [8.13.0]: https://github.com/gchq/CyberChef/releases/tag/v8.13.0
@@ -99,7 +139,9 @@ All major and minor version changes will be documented in this file. Details of
 
 [@n1474335]: https://github.com/n1474335
 [@d98762625]: https://github.com/d98762625
+[@j433866]: https://github.com/j433866
 [@GCHQ77703]: https://github.com/GCHQ77703
+[@h345983745]: https://github.com/h345983745
 [@artemisbot]: https://github.com/artemisbot
 [@picapi]: https://github.com/picapi
 [@Dachande663]: https://github.com/Dachande663
@@ -112,6 +154,8 @@ All major and minor version changes will be documented in this file. Details of
 [@jarmovanlenthe]: https://github.com/jarmovanlenthe
 [@tcode2k16]: https://github.com/tcode2k16
 [@Cynser]: https://github.com/Cynser
+[@anthony-arnold]: https://github.com/anthony-arnold
+[@masq]: https://github.com/masq
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -138,3 +182,13 @@ All major and minor version changes will be documented in this file. Details of
 [#439]: https://github.com/gchq/CyberChef/pull/439
 [#441]: https://github.com/gchq/CyberChef/pull/441
 [#443]: https://github.com/gchq/CyberChef/pull/443
+[#446]: https://github.com/gchq/CyberChef/pull/446
+[#448]: https://github.com/gchq/CyberChef/pull/448
+[#449]: https://github.com/gchq/CyberChef/pull/449
+[#455]: https://github.com/gchq/CyberChef/pull/455
+[#458]: https://github.com/gchq/CyberChef/pull/458
+[#461]: https://github.com/gchq/CyberChef/pull/461
+[#467]: https://github.com/gchq/CyberChef/pull/467
+[#468]: https://github.com/gchq/CyberChef/pull/468
+[#476]: https://github.com/gchq/CyberChef/pull/476
+[#489]: https://github.com/gchq/CyberChef/pull/489

Gruntfile.js

@@ -2,6 +2,7 @@
 
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
 const NodeExternals = require("webpack-node-externals");
 const Inliner = require("web-resource-inliner");
 const glob = require("glob");
@@ -29,8 +30,12 @@ module.exports = function (grunt) {
         ["clean:node", "clean:config", "exec:generateConfig", "webpack:node", "chmod:build"]);
 
     grunt.registerTask("test",
-        "A task which runs all the tests in test/tests.",
-        ["exec:generateConfig", "exec:tests"]);
+        "A task which runs all the operation tests in the tests directory.",
+        ["exec:generateConfig", "exec:opTests"]);
+
+    grunt.registerTask("testui",
+        "A task which runs all the UI tests in the tests directory. The prod task must already have been run.",
+        ["connect:prod", "exec:browserTests"]);
 
     grunt.registerTask("docs",
         "Compiles documentation in the /docs directory.",
@@ -66,6 +71,7 @@ module.exports = function (grunt) {
     grunt.loadNpmTasks("grunt-exec");
     grunt.loadNpmTasks("grunt-accessibility");
     grunt.loadNpmTasks("grunt-concurrent");
+    grunt.loadNpmTasks("grunt-contrib-connect");
 
 
     // Project configuration
@@ -143,11 +149,11 @@ module.exports = function (grunt) {
             options: {
                 configFile: "./.eslintrc.json"
             },
-            configs: ["*.js"],
+            configs: ["*.{js,mjs}"],
             core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"],
             web: ["src/web/**/*.{js,mjs}"],
             node: ["src/node/**/*.{js,mjs}"],
-            tests: ["test/**/*.{js,mjs}"],
+            tests: ["tests/**/*.{js,mjs}"],
         },
         jsdoc: {
             options: {
@@ -179,37 +185,45 @@ module.exports = function (grunt) {
         },
         webpack: {
             options: webpackConfig,
-            web: {
-                mode: "production",
-                target: "web",
-                entry: Object.assign({
-                    main: "./src/web/index.js",
-                    sitemap: "./src/web/static/sitemap.js"
-                }, moduleEntryPoints),
-                output: {
-                    path: __dirname + "/build/prod"
-                },
-                resolve: {
-                    alias: {
-                        "./config/modules/OpModules": "./config/modules/Default"
-                    }
-                },
-                plugins: [
-                    new webpack.DefinePlugin(BUILD_CONSTANTS),
-                    new HtmlWebpackPlugin({
-                        filename: "index.html",
-                        template: "./src/web/html/index.html",
-                        chunks: ["main"],
-                        compileTime: compileTime,
-                        version: pkg.version,
-                        minify: {
-                            removeComments: true,
-                            collapseWhitespace: true,
-                            minifyJS: true,
-                            minifyCSS: true
+            web: () => {
+                return {
+                    mode: "production",
+                    target: "web",
+                    entry: Object.assign({
+                        main: "./src/web/index.js",
+                        sitemap: "./src/web/static/sitemap.js"
+                    }, moduleEntryPoints),
+                    output: {
+                        path: __dirname + "/build/prod",
+                        globalObject: "this"
+                    },
+                    resolve: {
+                        alias: {
+                            "./config/modules/OpModules": "./config/modules/Default"
                         }
-                    }),
-                ]
+                    },
+                    plugins: [
+                        new webpack.DefinePlugin(BUILD_CONSTANTS),
+                        new HtmlWebpackPlugin({
+                            filename: "index.html",
+                            template: "./src/web/html/index.html",
+                            chunks: ["main"],
+                            compileTime: compileTime,
+                            version: pkg.version,
+                            minify: {
+                                removeComments: true,
+                                collapseWhitespace: true,
+                                minifyJS: true,
+                                minifyCSS: true
+                            }
+                        }),
+                        new BundleAnalyzerPlugin({
+                            analyzerMode: "static",
+                            reportFilename: "BundleAnalyzerReport.html",
+                            openAnalyzer: false
+                        }),
+                    ]
+                };
             },
             webInline: {
                 mode: "production",
@@ -238,19 +252,6 @@ module.exports = function (grunt) {
                     }),
                 ]
             },
-            tests: {
-                mode: "development",
-                target: "node",
-                entry: "./test/index.mjs",
-                externals: [NodeExternals()],
-                output: {
-                    filename: "index.js",
-                    path: __dirname + "/build/test"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(BUILD_CONSTANTS)
-                ]
-            },
             node: {
                 mode: "production",
                 target: "node",
@@ -299,6 +300,9 @@ module.exports = function (grunt) {
                             "./config/modules/OpModules": "./config/modules/Default"
                         }
                     },
+                    output: {
+                        globalObject: "this",
+                    },
                     plugins: [
                         new webpack.DefinePlugin(BUILD_CONSTANTS),
                         new HtmlWebpackPlugin({
@@ -312,6 +316,14 @@ module.exports = function (grunt) {
                 }
             }
         },
+        connect: {
+            prod: {
+                options: {
+                    port: 8000,
+                    base: "build/prod/"
+                }
+            }
+        },
         copy: {
             ghPages: {
                 options: {
@@ -383,16 +395,17 @@ module.exports = function (grunt) {
             generateConfig: {
                 command: [
                     "echo '\n--- Regenerating config files. ---'",
-                    "mkdir -p src/core/config/modules",
-                    "echo 'export default {};\n' > src/core/config/modules/OpModules.mjs",
-                    "echo '[]\n' > src/core/config/OperationConfig.json",
+                    "echo [] > src/core/config/OperationConfig.json",
                     "node --experimental-modules --no-warnings --no-deprecation src/core/config/scripts/generateOpsIndex.mjs",
                     "node --experimental-modules --no-warnings --no-deprecation src/core/config/scripts/generateConfig.mjs",
                     "echo '--- Config scripts finished. ---\n'"
                 ].join(";")
             },
-            tests: {
-                command: "node --experimental-modules --no-warnings --no-deprecation test/index.mjs"
+            opTests: {
+                command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
+            },
+            browserTests: {
+                command: "./node_modules/.bin/nightwatch --env prod,inline"
             }
         },
     });

babel.config.js

@@ -15,6 +15,7 @@ module.exports = function(api) {
             }]
         ],
         "plugins": [
+            "babel-plugin-syntax-dynamic-import",
             ["babel-plugin-transform-builtin-extend", {
                 "globals": ["Error"]
             }]

nightwatch.json

@@ -0,0 +1,34 @@
+{
+  "src_folders": ["tests/browser"],
+  "output_folder": "tests/browser/output",
+
+  "test_settings": {
+
+    "default": {
+      "launch_url": "http://localhost:8080",
+      "webdriver": {
+        "start_process": true,
+        "server_path": "./node_modules/.bin/chromedriver",
+        "port": 9515,
+        "log_path": false
+      },
+      "desiredCapabilities": {
+        "browserName": "chrome"
+      }
+    },
+
+    "dev": {
+      "launch_url": "http://localhost:8080"
+    },
+
+    "prod": {
+      "launch_url": "http://localhost:8000/index.html"
+    },
+
+    "inline": {
+      "launch_url": "http://localhost:8000/cyberchef.htm"
+    }
+
+  }
+}
+