Merge branch 'opensearch-project:main' into feature/OSD_NODE_HOME

.github/workflows/build_and_test_workflow.yml

@@ -9,10 +9,12 @@ on:
     branches: [ '**', '!backport/**' ]
     paths-ignore:
       - '**/*.md'
+      - 'docs/**'
   pull_request:
     branches: [ '**' ]
     paths-ignore:
       - '**/*.md'
+      - 'docs/**'
 
 env:
   TEST_BROWSER_HEADLESS: 1
@@ -45,6 +47,18 @@ jobs:
         run: |
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
+          yarn config set network-timeout 1000000 -g
+
+      - name: Configure Yarn Cache
+        run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV
+
+      - name: Initialize Yarn Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.YARN_CACHE_LOCATION }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
 
       - name: Run bootstrap
         run: yarn osd bootstrap
@@ -103,14 +117,23 @@ jobs:
         run: |
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
+          yarn config set network-timeout 1000000 -g
 
-      # https://github.com/yarnpkg/yarn/issues/8242#issuecomment-776561223
-      # Increase network timeout for Windows, retry once if bootstrap fails
-      - name: Run bootstrap
+      - name: Configure Yarn Cache
         run: |
-          yarn cache clean
-          yarn config set network-timeout 1000000 -g
-          yarn osd bootstrap || yarn osd bootstrap
+          echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $env:GITHUB_ENV
+          echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%"
+
+      - name: Initialize Yarn Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.YARN_CACHE_LOCATION }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
+
+      - name: Run bootstrap
+        run: yarn osd bootstrap || yarn osd bootstrap
 
       - name: Run linter
         id: linter
@@ -164,6 +187,18 @@ jobs:
         run: |
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
+          yarn config set network-timeout 1000000 -g
+
+      - name: Configure Yarn Cache
+        run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV
+
+      - name: Initialize Yarn Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.YARN_CACHE_LOCATION }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
 
       - name: Setup chromedriver
         run: node scripts/upgrade_chromedriver.js
@@ -223,17 +258,26 @@ jobs:
         run: |
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
+          yarn config set network-timeout 1000000 -g
+
+      - name: Configure Yarn Cache
+        run: |
+          echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $env:GITHUB_ENV
+          echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%"
+
+      - name: Initialize Yarn Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.YARN_CACHE_LOCATION }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
 
       - name: Setup chromedriver
         run: node scripts/upgrade_chromedriver.js
 
-      # https://github.com/yarnpkg/yarn/issues/8242#issuecomment-776561223
-      # Increase network timeout for Windows, retry once if bootstrap fails
       - name: Run bootstrap
-        run: |
-          yarn cache clean
-          yarn config set network-timeout 1000000 -g
-          yarn osd bootstrap || yarn osd bootstrap
+        run: yarn osd bootstrap || yarn osd bootstrap
 
       - name: Build plugins
         run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 10
@@ -291,6 +335,18 @@ jobs:
         run: |
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
+          yarn config set network-timeout 1000000 -g
+
+      - name: Configure Yarn Cache
+        run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV
+
+      - name: Initialize Yarn Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.YARN_CACHE_LOCATION }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
 
       - name: Get package version
         run: |
@@ -347,6 +403,20 @@ jobs:
         run: |
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
+          yarn config set network-timeout 1000000 -g
+
+      - name: Configure Yarn Cache
+        run: |
+          echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $env:GITHUB_ENV
+          echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%"
+
+      - name: Initialize Yarn Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.YARN_CACHE_LOCATION }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
 
       - name: Get package version
         run: |
@@ -401,6 +471,18 @@ jobs:
         run: |
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
+          yarn config set network-timeout 1000000 -g
+
+      - name: Configure Yarn Cache
+        run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV
+
+      - name: Initialize Yarn Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.YARN_CACHE_LOCATION }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
 
       - name: Get package version
         run: |

CHANGELOG.md

@@ -15,6 +15,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Multi DataSource] Add explicit no spellcheck on password fields ([#2818](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2818))
 - [CVE-2022-25912] Bumps simple-git from 3.4.0 to 3.15.0 ([#3036](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3036))
 - [CVE-2022-35256] Bumps node version from 14.20.0 to 14.20.1 [#3166](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3166))
+- [CVE-2022-46175] Bumps json5 version from 1.0.1 and 2.2.1 to 1.0.2 and 2.2.3 ([#3201](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3201))
 
 ### 📈 Features/Enhancements
 
@@ -49,11 +50,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [I18n] Register ru, ru-RU locale ([#2817](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2817))
 - Add yarn opensearch arg to setup plugin dependencies ([#2544](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/2544))
 - [Multi DataSource] Test the connection to an external data source when creating or updating ([#2973](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2973))
-- [Doc] Add current plugin persistence implementation readme ([#3081](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3081))
 - [Table Visualization] Refactor table visualization using React and DataGrid component ([#2863](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2863))
 - [Vis Builder] Add redux store persistence ([#3088](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3088))
 - [Multi DataSource] Improve test connection ([#3110](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3110))
+- [Vis Builder] Add app filter and query persistence without using state container ([#3100](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3100))
 - [Optimizer] Increase timeout waiting for the exiting of an optimizer worker ([#3193](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3193))
+- [Data] Update `createAggConfig` so that newly created configs can be added to beginning of `aggConfig` array ([#3160](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3160))
+
 
 ### 🐛 Bug Fixes
 
@@ -69,6 +72,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Multi DataSource] Address UX comments on index pattern management stack ([#2611](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2611))
 - [Multi DataSource] Apply get indices error handling in step index pattern ([#2652](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2652))
 - [Vis Builder] Last Updated Timestamp for visbuilder savedobject is getting Generated ([#2628](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2628))
+- [Vis Builder] fixes filters for table visualisation ([#3210](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3210))
 - Removed Leftover X Pack references ([#2638](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2638))
 - Removes Add Integration button ([#2723](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2723))
 - Change geckodriver version to make consistency ([#2772](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2772))
@@ -80,11 +84,12 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Build] Fixed "Last Access Time" not being set by `scanCopy` on Windows ([#2964](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2964))
 - [Vis Builder] Add global data persistence for vis builder #2896 ([#2896](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2896))
 - Update `leaflet-vega` and fix its usage ([#3005](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3005))
-- [Table Visualization][BUG] Fix Url content display issue in table ([#2918](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2918))
+- [Table Visualization][bug] Fix Url content display issue in table ([#2918](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2918))
 - Fixes misleading embaddable plugin error message ([#3043](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3043))
 - [MD] Update dummy url in tests to follow lychee url allowlist ([#3099](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3099))
 - Adds config override to fix obsolete theme:version config value of v8 (beta) rendering issue ([#3045](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3045))
 - [CI] Update test workflow to increase network-timeout for yarn for installing dependencies ([#3118](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3118))
+- [Region Maps] Fixes bug that prevents selected join field to be used ([#3213](Fix bug that prevents selected join field to be used))
 
 ### 🚞 Infrastructure
 
@@ -94,6 +99,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add Windows CI workflows ([#2966](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2966))
 - Add automatic selection of the appropriate version of chrome driver to run functional tests ([#2990](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2990))
 - Add recording of functional test artifacts if they fail ([#3190](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3190))
+- Improve yarn's performance in workflows by caching yarn's cache folder ([#3194](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3194))
 
 ### 📝 Documentation
 
@@ -104,12 +110,14 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add `current-usage.md` and more details to `README.md` of `charts` plugin ([#2695](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2695))
 - [Doc] Add readme for global query persistence ([#3001](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3001))
 - Updates NOTICE file, adds validation to GitHub CI ([#3051](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3051))
+- [Doc] Add current plugin persistence implementation readme ([#3081](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3081))
 
 ### 🛠 Maintenance
 
 - Adding @zhongnansu as maintainer. ([#2590](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2590))
 - Removes `minimatch` manual resolution ([#3019](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3019))
 - Remove `github-checks-reporter`, an unused dependency ([#3126](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3126))
+- Upgrade `vega-lite` dependency to ^5.6.0 ([#3076](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3076))
 
 ### 🪛 Refactoring
 
@@ -127,6 +135,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Make tests covering plugin installation on cluster snapshots work across platforms ([#2994](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2994))
 - Correct the linting logic for `no-restricted-path` to ignore trailing slashes ([#3020](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3020))
 - [Tests] Bumps `chromedriver` to v107 ([#3017](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3017))
+- [Vis Builder] Adds field unit tests ([#3211](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3211))
 
 ## [2.x]
 
@@ -145,7 +154,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [CVE-2022-37601] Bump loader-utils to 2.0.3 ([#2689](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2689))
 - [CVE-2022-37599] Bump loader-utils to 2.0.4 ([#3031](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3031))
 - [CVE-2022-37603] Bump loader-utils to 2.0.4 ([#3031](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3031))
-- [WS-2021-0638][Security] bump mocha to 10.1.0 ([#2711](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2711))
+- [WS-2021-0638][security] bump mocha to 10.1.0 ([#2711](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2711))
 
 ### 📈 Features/Enhancements
 

docs/plugins/data_persistence.md

@@ -196,6 +196,8 @@ There are two types for data persistence:
 
 4. In `useEditorUpdates()`, we use the saved appState to load the visualize editor
 
+5. We can also choose to do state sync without using state container by directing hooking up the state managers with the URL data storage. For example, we implemented state management in Vis Builder with redux store, while the global state persistence implementation is the same, we implemented app state persistence without using any state containers or state syncing utils. For the actual visual data, we directly hook up redux store with `OsdUrlStateStorage` to sync the values by using `saveReduxState()` and `loadReduxState()`. For app filter and query, since they are part of the app states but not part of the redux store, we directly hook up their state managers from data plugin with `OsdUrlStateStorage` and `connectStorageToQueryState()`.
+
 # Refresh
 When we refresh the page, both app state and global state should persist:
 

package.json

@@ -456,7 +456,7 @@
     "ui-select": "0.19.8",
     "vega": "^5.17.3",
     "vega-interpreter": "npm:@amoo-miki/vega-forced-csp-compliant-interpreter@1.0.6",
-    "vega-lite": "^4.16.8",
+    "vega-lite": "^5.6.0",
     "vega-schema-url-parser": "^2.1.0",
     "vega-tooltip": "^0.24.2",
     "vinyl-fs": "^3.0.3",

packages/osd-babel-preset/common_preset.js

@@ -32,27 +32,27 @@ const plugins = [
   require.resolve('babel-plugin-add-module-exports'),
 
   // The class properties proposal was merged with the private fields proposal
-  // into the "class fields" proposal. Babel doesn't support this combined
-  // proposal yet, which includes private field, so this transform is
-  // TECHNICALLY stage 2, but for all intents and purposes it's stage 3
-  //
+  // into the "class fields" proposal which is stage 3.
   // See https://github.com/babel/proposals/issues/12 for progress
   require.resolve('@babel/plugin-proposal-class-properties'),
 
-  // Optional Chaining proposal is stage 3 (https://github.com/tc39/proposal-optional-chaining)
+  // Optional Chaining proposal is stage 4 (https://github.com/tc39/proposal-optional-chaining)
   // Need this since we are using TypeScript 3.7+
   require.resolve('@babel/plugin-proposal-optional-chaining'),
-  // Nullish coalescing proposal is stage 3 (https://github.com/tc39/proposal-nullish-coalescing)
+  // Nullish coalescing proposal is stage 4 (https://github.com/tc39/proposal-nullish-coalescing)
   // Need this since we are using TypeScript 3.7+
   require.resolve('@babel/plugin-proposal-nullish-coalescing-operator'),
 
-  // Proposal is on stage 4 (https://github.com/tc39/proposal-export-ns-from)
+  // Proposal is merged into ECMA-262 (https://github.com/tc39/proposal-export-ns-from)
   // Need this since we are using TypeScript 3.8+
   require.resolve('@babel/plugin-proposal-export-namespace-from'),
 
-  // Proposal is on stage 4 (https://github.com/tc39/proposal-export-ns-from)
+  // Proposal is merged into ECMA-262 (https://github.com/tc39/proposal-export-ns-from)
   // Need this since we are using TypeScript 3.9+
   require.resolve('@babel/plugin-proposal-private-methods'),
+
+  // Proposal is on stage 4 (https://github.com/tc39/proposal-logical-assignment)
+  require.resolve('@babel/plugin-proposal-logical-assignment-operators'),
 ];
 
 module.exports = {

packages/osd-babel-preset/package.json

@@ -9,6 +9,7 @@
   "dependencies": {
     "@babel/plugin-proposal-class-properties": "^7.16.5",
     "@babel/plugin-proposal-export-namespace-from": "^7.16.5",
+    "@babel/plugin-proposal-logical-assignment-operators": "^7.16.5",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.16.5",
     "@babel/plugin-proposal-optional-chaining": "^7.16.5",
     "@babel/plugin-proposal-private-methods": "^7.16.5",

packages/osd-optimizer/src/worker/webpack.config.ts

@@ -205,7 +205,9 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
         },
         {
           test: /\.(js|tsx?)$/,
-          exclude: /node_modules/,
+          // vega-lite and some of its dependencies don't have es5 builds
+          // so we need to build from source and transpiled for webpack v4
+          exclude: /[\/\\]node_modules[\/\\](?!vega-(lite|label|functions)[\/\\])/,
           use: {
             loader: 'babel-loader',
             options: {