[Windows] Facilitate building of OSD and plugins (#2601) (#2984)

* [Windows] Replaces `rm -rf` with `remove.js`
* [dev/build] Facilitates using zipped archives of node releases
* [div/build] Introduces Windows as a platform
* [dev/build] Corrects cleaning of platform specific build artifacts
* [dev/build] Enhances the cleanup of downloaded node binaries
* [opensearch-dashboards-plugin] Removes prohibition on installing plugins on Windows
* [@osd/utils] Adds a method to standardize path references across platforms
* [dev/build] Standardize paths in tests
* [@osd/telemetry-tools] Normalizes the collection paths
* [plugins/url-forwarding] Fixes the usage of `normalizePath` across node and browser
* [@osd/pm] Allows symlink created for tests without elevated privileges on Windows
* [@osd/opensearch] Allows usage of Windows snapshots in integration tests
* [@osd/opensearch] Employs absolute paths in tests
* [@osd/apm-config-loader] Employs absolute paths in tests
* [core/server] Employs absolute and posix references to paths
* [@osd/optimizer] Standardize paths in tests
* [@osd/tests] Employs absolute paths in tests
* [@osd/pm] Standardize paths in project trees
* [plugins/telemetry] Accommodates the inability of Windows to create unreadable files for testing
* [@osd/config-schema] Normalize paths in tests
* [@osd/plugin-helpers] Standardize paths in tests
* [@osd/plugin-generator] Standardize paths in tests
* [Windows] Update changelog

cherry-picked from ec9e0b2

Signed-off-by: Miki <amoo_miki@yahoo.com>

Signed-off-by: Miki <amoo_miki@yahoo.com>

CHANGELOG.md

@@ -20,6 +20,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### 📈 Features/Enhancements
 
+- [Windows] Facilitate building and running OSD and plugins on Windows platforms ([#2601](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2601))
+
 ### 🐛 Bug Fixes
 
 * [BUG] Fix suggestion list cutoff issue ([#2607](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2607))

package.json

@@ -64,7 +64,7 @@
     "uiFramework:createComponent": "cd packages/osd-ui-framework && yarn createComponent",
     "uiFramework:documentComponent": "cd packages/osd-ui-framework && yarn documentComponent",
     "osd:watch": "node scripts/opensearch_dashboards --dev --logging.json=false",
-    "build:types": "rm -rf ./target/types && tsc --p tsconfig.types.json",
+    "build:types": "node scripts/remove.js ./target/types && tsc --p tsconfig.types.json",
     "docs:acceptApiChanges": "node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept",
     "osd:bootstrap": "node scripts/build_ts_refs && node scripts/register_git_hook",
     "spec_to_console": "node scripts/spec_to_console"
@@ -435,6 +435,7 @@
     "mutation-observer": "^1.0.3",
     "ngreact": "^0.5.1",
     "nock": "12.0.3",
+    "node-stream-zip": "^1.15.0",
     "normalize-path": "^3.0.0",
     "nyc": "^14.1.1",
     "pixelmatch": "^5.1.0",

packages/osd-apm-config-loader/src/utils/get_config_file_paths.test.ts

@@ -42,7 +42,7 @@ describe('getConfigurationFilePaths', () => {
 
     expect(getConfigurationFilePaths(argv)).toEqual([
       resolve(cwd, join('.', 'relative-path')),
-      '/absolute-path',
+      resolve('/absolute-path'),
     ]);
   });
 

packages/osd-config-schema/src/errors/schema_error.test.ts

@@ -30,7 +30,7 @@
  * GitHub history for details.
  */
 
-import { relative } from 'path';
+import { relative, sep } from 'path';
 import { SchemaError } from '.';
 
 /**
@@ -39,7 +39,7 @@ import { SchemaError } from '.';
 export const cleanStack = (stack: string) =>
   stack
     .split('\n')
-    .filter((line) => !line.includes('node_modules/') && !line.includes('internal/'))
+    .filter((line) => !line.includes('node_modules' + sep) && !line.includes('internal/'))
     .map((line) => {
       const parts = /.*\((.*)\).?/.exec(line) || [];
 
@@ -48,7 +48,11 @@ export const cleanStack = (stack: string) =>
       }
 
       const path = parts[1];
-      return line.replace(path, relative(process.cwd(), path));
+      // Cannot use `standardize` from `@osd/utils
+      let relativePath = relative(process.cwd(), path);
+      if (process.platform === 'win32') relativePath = relativePath.replace(/\\/g, '/');
+
+      return line.replace(path, relativePath);
     })
     .join('\n');
 

packages/osd-opensearch-archiver/package.json

@@ -7,8 +7,8 @@
     "devOnly": true
   },
   "scripts": {
-    "osd:bootstrap": "rm -rf target && tsc",
-    "osd:watch": "rm -rf target && tsc --watch"
+    "osd:bootstrap": "node ../../scripts/remove.js target && tsc",
+    "osd:watch": "node ../../scripts/remove.js target && tsc --watch"
   },
   "dependencies": {
     "@osd/dev-utils": "1.0.0",
@@ -17,4 +17,4 @@
   "devDependencies": {
     "@types/elasticsearch": "^5.0.33"
   }
-}
+}

packages/osd-opensearch/src/artifact.js

@@ -182,12 +182,14 @@ async function getArtifactSpecForSnapshotFromUrl(urlVersion, log) {
   // issue: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/475
   const platform = process.platform === 'win32' ? 'windows' : process.platform;
   const arch = process.arch === 'arm64' ? 'arm64' : 'x64';
-  if (platform !== 'linux') {
-    throw createCliError(`Snapshots are only available for Linux`);
+  const extension = process.platform === 'win32' ? 'zip' : 'tar.gz';
+
+  if (platform !== 'linux' && platform !== 'windows') {
+    throw createCliError(`Snapshots are only available for Linux and Windows`);
   }
 
   const latestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}-SNAPSHOT`;
-  const latestFile = `opensearch-min-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`;
+  const latestFile = `opensearch-min-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.${extension}`;
   const completeLatestUrl = `${latestUrl}/${latestFile}`;
 
   let { abc, resp } = await verifySnapshotUrl(completeLatestUrl, log);

packages/osd-opensearch/src/artifact.test.js

@@ -164,10 +164,10 @@ describe('Artifact', () => {
         });
       });
 
-      it('should throw when on a non-Linux platform', async () => {
+      it('should throw when on a non-Linux or non-Windows platform', async () => {
         Object.defineProperties(process, {
           platform: {
-            value: 'win32',
+            value: 'darwin',
           },
           arch: {
             value: ORIGINAL_ARCHITECTURE,

packages/osd-opensearch/src/utils/decompress.test.js

@@ -54,8 +54,8 @@ beforeEach(() => {
   fs.copyFileSync(path.resolve(fixturesFolder, 'snapshot.tar.gz'), tarGzSnapshot);
 });
 
-afterEach(() => {
-  del.sync(tmpFolder, { force: true });
+afterEach(async () => {
+  await del(tmpFolder, { force: true });
 });
 
 test('zip strips root directory', async () => {

packages/osd-opensearch/src/utils/extract_config_files.test.js

@@ -38,6 +38,7 @@ jest.mock('fs', () => ({
 
 const { extractConfigFiles } = require('./extract_config_files');
 const fs = require('fs');
+const path = require('path');
 
 afterEach(() => {
   jest.clearAllMocks();
@@ -57,7 +58,7 @@ test('copies file', () => {
   extractConfigFiles(['path=/data/foo.yml'], '/opensearch');
 
   expect(fs.readFileSync.mock.calls[0][0]).toEqual('/data/foo.yml');
-  expect(fs.writeFileSync.mock.calls[0][0]).toEqual('/opensearch/config/foo.yml');
+  expect(fs.writeFileSync.mock.calls[0][0]).toEqual(path.resolve('/opensearch/config/foo.yml'));
 });
 
 test('ignores file which does not exist', () => {

packages/osd-opensearch/src/utils/find_most_recently_changed.test.js

@@ -24,6 +24,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+const path = require('path');
 
 /*
  * Modifications Copyright OpenSearch Contributors. See
@@ -59,7 +60,7 @@ const { findMostRecentlyChanged } = require('./find_most_recently_changed');
 
 test('returns newest file', () => {
   const file = findMostRecentlyChanged('/data/*.yml');
-  expect(file).toEqual('/data/newest.yml');
+  expect(file).toEqual(path.resolve('/data/newest.yml'));
 });
 
 afterAll(() => {

packages/osd-optimizer/src/common/bundle.test.ts

@@ -30,6 +30,7 @@
  * GitHub history for details.
  */
 
+import { resolve } from 'path';
 import { Bundle, BundleSpec, parseBundles } from './bundle';
 
 jest.mock('fs');
@@ -90,13 +91,16 @@ it('provides the module count from the cache', () => {
 
 it('parses bundles from JSON specs', () => {
   const bundles = parseBundles(JSON.stringify([SPEC]));
+  let expectedCachePath = resolve('/foo/bar/target/.osd-optimizer-cache');
+  // Cannot use `standardize` from `@osd/util` due to mocking of fs
+  if (process?.platform === 'win32') expectedCachePath = expectedCachePath.replace(/\\/g, '\\\\');
 
   expect(bundles).toMatchInlineSnapshot(`
     Array [
       Bundle {
         "banner": undefined,
         "cache": BundleCache {
-          "path": "/foo/bar/target/.osd-optimizer-cache",
+          "path": "${expectedCachePath}",
           "state": undefined,
         },
         "contextDir": "/foo/bar",

packages/osd-optimizer/src/optimizer/get_changes.test.ts

@@ -30,9 +30,12 @@
  * GitHub history for details.
  */
 
+import path from 'path';
+
 jest.mock('execa');
 
 import { getChanges } from './get_changes';
+import { standardize } from '@osd/dev-utils';
 
 const execa: jest.Mock = jest.requireMock('execa');
 
@@ -58,12 +61,16 @@ it('parses git ls-files output', async () => {
     };
   });
 
+  const rootPath = path.resolve('/foo/bar/x/osd-optimizer') + path.sep;
+  const srcPath = path.join(rootPath, 'src') + path.sep;
+  const commonPath = path.join(srcPath, 'common') + path.sep;
+
   await expect(getChanges('/foo/bar/x')).resolves.toMatchInlineSnapshot(`
     Map {
-      "/foo/bar/x/osd-optimizer/package.json" => "modified",
-      "/foo/bar/x/osd-optimizer/src/common/bundle.ts" => "modified",
-      "/foo/bar/x/osd-optimizer/src/common/bundles.ts" => "deleted",
-      "/foo/bar/x/osd-optimizer/src/get_bundle_definitions.test.ts" => "deleted",
+      "${standardize(rootPath, false, true)}package.json" => "modified",
+      "${standardize(commonPath, false, true)}bundle.ts" => "modified",
+      "${standardize(commonPath, false, true)}bundles.ts" => "deleted",
+      "${standardize(srcPath, false, true)}get_bundle_definitions.test.ts" => "deleted",
     }
   `);
 });

packages/osd-optimizer/src/optimizer/get_plugin_bundles.test.ts

@@ -33,10 +33,15 @@
 import { createAbsolutePathSerializer } from '@osd/dev-utils';
 
 import { getPluginBundles } from './get_plugin_bundles';
+import path from 'path';
 
 expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo', '<repoRoot>'));
 expect.addSnapshotSerializer(createAbsolutePathSerializer('/output', '<outputRoot>'));
+expect.addSnapshotSerializer(createAbsolutePathSerializer(path.resolve('/output'), '<outputRoot>'));
 expect.addSnapshotSerializer(createAbsolutePathSerializer('/outside/of/repo', '<outsideOfRepo>'));
+expect.addSnapshotSerializer(
+  createAbsolutePathSerializer(path.resolve('/outside/of/repo'), '<outsideOfRepo>')
+);
 
 it('returns a bundle for core and each plugin', () => {
   expect(

packages/osd-plugin-generator/src/integration_tests/generate_plugin.test.ts

@@ -34,13 +34,17 @@ import Path from 'path';
 
 import del from 'del';
 import execa from 'execa';
-import { REPO_ROOT } from '@osd/utils';
-import { createAbsolutePathSerializer } from '@osd/dev-utils';
+import { REPO_ROOT, standardize, createAbsolutePathSerializer } from '@osd/dev-utils';
 import globby from 'globby';
 
-const GENERATED_DIR = Path.resolve(REPO_ROOT, `plugins`);
+// Has to be a posix reference because it is used to generate glob patterns
+const GENERATED_DIR = standardize(Path.resolve(REPO_ROOT, `plugins`), true);
 
-expect.addSnapshotSerializer(createAbsolutePathSerializer());
+expect.addSnapshotSerializer(
+  createAbsolutePathSerializer(
+    process?.platform === 'win32' ? standardize(REPO_ROOT, true) : REPO_ROOT
+  )
+);
 
 beforeEach(async () => {
   await del([`${GENERATED_DIR}/**`, `!${GENERATED_DIR}`, `!${GENERATED_DIR}/.gitignore`], {

packages/osd-plugin-helpers/package.json

@@ -12,7 +12,7 @@
     "plugin-helpers": "bin/plugin-helpers.js"
   },
   "scripts": {
-    "osd:bootstrap": "rm -rf target && tsc",
+    "osd:bootstrap": "node ../../scripts/remove.js && tsc",
     "osd:watch": "tsc --watch"
   },
   "dependencies": {

packages/osd-plugin-helpers/src/integration_tests/build.test.ts

@@ -34,8 +34,12 @@ import Path from 'path';
 import Fs from 'fs';
 
 import execa from 'execa';
-import { REPO_ROOT } from '@osd/utils';
-import { createStripAnsiSerializer, createReplaceSerializer } from '@osd/dev-utils';
+import {
+  REPO_ROOT,
+  standardize,
+  createStripAnsiSerializer,
+  createReplaceSerializer,
+} from '@osd/dev-utils';
 import extract from 'extract-zip';
 import del from 'del';
 import globby from 'globby';
@@ -80,7 +84,7 @@ it('builds a generated plugin into a viable archive', async () => {
   expect(generateProc.all).toMatchInlineSnapshot(`
     " succ 🎉
 
-          Your plugin has been created in plugins/foo_test_plugin
+          Your plugin has been created in ${standardize('plugins/foo_test_plugin', false, true)}
     "
   `);
 
@@ -169,7 +173,7 @@ it('builds a non-semver generated plugin into a viable archive', async () => {
   expect(generateProc.all).toMatchInlineSnapshot(`
     " succ 🎉
 
-          Your plugin has been created in plugins/foo_test_plugin
+          Your plugin has been created in ${standardize('plugins/foo_test_plugin', false, true)}
     "
   `);
 

packages/osd-pm/src/utils/projects.test.ts

@@ -55,7 +55,8 @@ describe('#getProjects', () => {
 
     await promisify(symlink)(
       join(__dirname, '__fixtures__/symlinked-plugins/corge'),
-      join(rootPlugins, 'corge')
+      join(rootPlugins, 'corge'),
+      'junction' // This parameter would only be used on Windows
     );
   });
 

packages/osd-pm/src/utils/projects_tree.ts

@@ -33,6 +33,7 @@
 import chalk from 'chalk';
 import path from 'path';
 
+import { standardize } from '@osd/utils';
 import { Project } from './project';
 
 const projectKey = Symbol('__project');
@@ -119,7 +120,7 @@ function createTreeStructure(tree: IProjectsTree): ITree {
     // `foo/bar/baz` instead.
     if (subtree.children && subtree.children.length === 1) {
       const child = subtree.children[0];
-      const newName = chalk.dim(path.join(dir.toString(), child.name!));
+      const newName = chalk.dim(standardize(path.join(dir.toString(), child.name!), true));
 
       children.push({
         children: child.children,

packages/osd-telemetry-tools/src/tools/ts_parser.ts

@@ -33,7 +33,7 @@
 import * as ts from 'typescript';
 import { createFailError } from '@osd/dev-utils';
 import * as path from 'path';
-import { getProperty, getPropertyValue } from './utils';
+import { getProperty, getPropertyValue, normalizePath } from './utils';
 import { getDescriptor, Descriptor } from './serializer';
 
 export function* traverseNodes(maybeNodes: ts.Node | ts.Node[]): Generator<ts.Node> {
@@ -207,7 +207,7 @@ export function* parseUsageCollection(
   sourceFile: ts.SourceFile,
   program: ts.Program
 ): Generator<ParsedUsageCollection> {
-  const relativePath = path.relative(process.cwd(), sourceFile.fileName);
+  const relativePath = normalizePath(path.relative(process.cwd(), sourceFile.fileName), false);
   if (sourceHasUsageCollector(sourceFile)) {
     for (const node of traverseNodes(sourceFile)) {
       if (isMakeUsageCollectorFunction(node, sourceFile)) {

packages/osd-telemetry-tools/src/tools/utils.ts

@@ -313,6 +313,7 @@ export function difference(actual: any, expected: any) {
   return changes(actual, expected);
 }
 
-export function normalizePath(inputPath: string) {
-  return normalize(path.relative('.', inputPath));
+export function normalizePath(inputPath: string, relativeToRoot: boolean = true) {
+  if (relativeToRoot) return normalize(path.relative('.', inputPath));
+  return normalize(inputPath);
 }

packages/osd-test/src/functional_test_runner/lib/suite_tracker.test.ts

@@ -42,7 +42,7 @@ import { REPO_ROOT } from '@osd/dev-utils';
 import { Lifecycle } from './lifecycle';
 import { SuiteTracker } from './suite_tracker';
 
-const DEFAULT_TEST_METADATA_PATH = join(REPO_ROOT, 'target', 'test_metadata.json');
+const DEFAULT_TEST_METADATA_PATH = resolve(REPO_ROOT, 'target', 'test_metadata.json');
 const MOCK_CONFIG_PATH = join('test', 'config.js');
 const MOCK_TEST_PATH = join('test', 'apps', 'test.js');
 const ENVS_TO_RESET = ['TEST_METADATA_PATH'];