feat(v2): v1-v2 migration cli automate migration from v1 to v2 (#3015)

* add poc for migration tool

* fix version

* fix path

* fix some type error

* fix type errors

* fix commad

* delete package.lock.json

* ignore lib folder in eslint

* ignore lib in prettier

* fix missing version

* fix sidebar in next

* use primaey color

* fix navbar link

* fix footer

* sanatize front-matter for gray-matter

* add e2e test

* fixworkflow

* add unit test

* chore(v2): fix eslint error

* refactor(v2): use descriptive variable names

* refactor(v2): refactor createProjectStructure

* refactor(v2): refactor migrateDocusaurusProject

* refactor(v2): fix eslint errors

* fix(v2): fix mistake

* use path.join

* remove console.log

* try automate migrating md file to mdx

* fix types not found

* fix types

* remove unused fixture

* use package.json for version

* add support for pages

* add support

* sanitize fortmatter only in case of special character

* chore(v2): add color to dependencies

* feat(v2): initialize config with range of color and add logs

* chore(v2): add glob dependency

* fix(v2): fix color generation

* fix(v2): fix type issue

* feat(v2): add all unknown fields to customFields

* fix(v2): fix css mistake

* fix lerna

* fix(v2): fix github actions

* feat(v2): add option flag for migrating pages

* fix special character

* remove depulicate build

* fix test

* remove unwanted file

* fix frontmatter

* remove unused file

* rerun the test

* fix links

* feat(v2): filter out more fields from customFields

* feat(v2): filter out deprecated fields from customFields

* fix items

* fix types

* merge master

* revert docs

* fix doc

* fix broken link

* fix test

* test

* fix ci

* fix ci

* fix ci

* fix ci

* fix frontmatter

* log custom fields

Co-authored-by: teikjun <teikjunhci@gmail.com>

.eslintignore

@@ -23,6 +23,7 @@ packages/docusaurus-plugin-sitemap/lib/
 packages/docusaurus-plugin-ideal-image/lib/
 packages/docusaurus-plugin-ideal-image/copyUntypedFiles.js
 packages/docusaurus-theme-classic/lib/
+packages/docusaurus-migrate/lib/
 
 packages/docusaurus-1.x/.eslintrc.js
 packages/docusaurus-init/templates/facebook/.eslintrc.js

.github/workflows/migration-cli-e2e-test.yml

@@ -0,0 +1,35 @@
+name: MIGRATION CLI E2E TEST
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the master branch
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [10.x]
+    steps:
+      - uses: actions/checkout@v1
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Installation
+        run: yarn
+      - name: Migrate D1 website
+        run: yarn docusaurus-migrate migrate ./website-1.x ./test-migrated
+      - name: link
+        run: yarn lerna exec -- yarn link
+      - name: Build Test website
+        run: yarn build
+        working-directory: test-migrated
+        env:
+          CI: true

.gitignore

@@ -27,6 +27,7 @@ packages/docusaurus-plugin-debug/lib/
 packages/docusaurus-plugin-sitemap/lib/
 packages/docusaurus-plugin-ideal-image/lib/
 packages/docusaurus-theme-classic/lib/
+packages/docusaurus-migrate/lib/
 
 website/netlifyDeploy
 _redirects

.prettierignore

@@ -14,5 +14,6 @@ packages/docusaurus-plugin-content-pages/lib/
 packages/docusaurus-plugin-debug/lib/
 packages/docusaurus-plugin-sitemap/lib/
 packages/docusaurus-plugin-ideal-image/lib/
+packages/docusaurus-migrate/lib/
 packages/docusaurus-theme-classic/lib/
 __fixtures__

docs/getting-started-publishing.md

@@ -260,33 +260,33 @@ Now, whenever a new commit lands in `master`, Travis CI will run your suite of t
 ```yaml
 # azure-pipelines.yml
 trigger:
-- master
+  - master
 
 pool:
   vmImage: 'ubuntu-latest'
 
 steps:
-- checkout: self
-  persistCredentials: true
-
-- task: NodeTool@0
-  inputs:
-    versionSpec: '10.x'
-  displayName: 'Install Node.js'
-
-- script: |
-    git config --global user.name "${GH_NAME}"
-    git config --global user.email "${GH_EMAIL}"
-    git checkout -b master
-    echo "machine github.com login ${GH_NAME} password ${GH_TOKEN}" > ~/.netrc
-    cd website
-    yarn install
-    GIT_USER="${GH_NAME}" CURRENT_BRANCH=master yarn run publish-gh-pages
-  env:
-    GH_NAME: $(GH_NAME)
-    GH_EMAIL: $(GH_EMAIL)
-    GH_TOKEN: $(GH_TOKEN)
-  displayName: 'yarn install and build'
+  - checkout: self
+    persistCredentials: true
+
+  - task: NodeTool@0
+    inputs:
+      versionSpec: '10.x'
+    displayName: 'Install Node.js'
+
+  - script: |
+      git config --global user.name "${GH_NAME}"
+      git config --global user.email "${GH_EMAIL}"
+      git checkout -b master
+      echo "machine github.com login ${GH_NAME} password ${GH_TOKEN}" > ~/.netrc
+      cd website
+      yarn install
+      GIT_USER="${GH_NAME}" CURRENT_BRANCH=master yarn run publish-gh-pages
+    env:
+      GH_NAME: $(GH_NAME)
+      GH_EMAIL: $(GH_EMAIL)
+      GH_TOKEN: $(GH_TOKEN)
+    displayName: 'yarn install and build'
 ```
 
 ### Hosting on ZEIT Now

jest.config.js

@@ -15,6 +15,7 @@ const ignorePatterns = [
   '/packages/docusaurus-plugin-content-blog/lib',
   '/packages/docusaurus-plugin-content-docs/lib',
   '/packages/docusaurus-plugin-content-pages/lib',
+  '/packages/docusaurus-migrate/lib',
 ];
 
 module.exports = {

packages/docusaurus-migrate/bin/index.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const chalk = require('chalk');
+const semver = require('semver');
+const cli = require('commander');
+const path = require('path');
+
+const requiredVersion = require('../package.json').engines.node;
+
+const {migrateDocusaurusProject, migrateMDToMDX} = require('../lib');
+
+function wrapCommand(fn) {
+  return (...args) =>
+    fn(...args).catch((err) => {
+      console.error(chalk.red(err.stack));
+      process.exitCode = 1;
+    });
+}
+
+if (!semver.satisfies(process.version, requiredVersion)) {
+  console.log(
+    chalk.red(`\nMinimum Node version not met :(`) +
+      chalk.yellow(
+        `\n\nYou are using Node ${process.version}. We require Node ${requiredVersion} or up!\n`,
+      ),
+  );
+  process.exit(1);
+}
+
+cli
+  .command('migrate [siteDir] [newDir]')
+  .option('--mdx', 'Try to migrate MD to MDX too')
+  .option('--page', 'Try to migrate pages too')
+  .description('Migrate between versions of docusaurus website')
+  .action((siteDir = '.', newDir = '.', {mdx, page}) => {
+    const sitePath = path.resolve(siteDir);
+    const newSitePath = path.resolve(newDir);
+    wrapCommand(migrateDocusaurusProject)(sitePath, newSitePath, mdx, page);
+  });
+
+cli
+  .command('mdx [siteDir] [newDir]')
+  .description('Migrate markdown files to MDX')
+  .action((siteDir = '.', newDir = '.') => {
+    const sitePath = path.resolve(siteDir);
+    const newSitePath = path.resolve(newDir);
+    wrapCommand(migrateMDToMDX)(sitePath, newSitePath);
+  });
+cli.parse(process.argv);
+
+if (!process.argv.slice(2).length) {
+  cli.outputHelp();
+}

packages/docusaurus-migrate/external_types/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+declare module '@mapbox/hast-util-to-jsx';
+
+declare module 'hast-util-to-string';
+
+declare module 'color';