Merge branch 'master' into master

.circleci/config.yml

@@ -130,58 +130,6 @@ commands:
             - slack/status:
               channel: eng-react-integration-status
 
-  run-benchmark:
-    parameters:
-      working_directory:
-        type: string
-      NUM_PAGES:
-        type: string
-      BENCHMARK_CONTENT_SOURCE:
-        type: string
-      BENCHMARK_REPO_NAME:
-        type: string
-        default: gatsbyjs/gatsby
-      BENCHMARK_SITE_TYPE:
-        type: string
-        default: BLOG
-      BENCHMARK_BRANCH:
-        type: string
-        default: master
-    steps:
-      - checkout
-      - run:
-          command: npm install
-          working_directory: << parameters.working_directory >>
-          environment:
-            NUM_PAGES: << parameters.NUM_PAGES >>
-      - run:
-          command: npm run build
-          working_directory: << parameters.working_directory >>
-          environment:
-            BENCHMARK_BUILD_TYPE: COLD_START
-            NUM_PAGES: << parameters.NUM_PAGES >>
-            BENCHMARK_CONTENT_SOURCE: << parameters.BENCHMARK_CONTENT_SOURCE >>
-            BENCHMARK_REPO_NAME: << parameters.BENCHMARK_REPO_NAME >>
-            BENCHMARK_SITE_TYPE: << parameters.BENCHMARK_SITE_TYPE >>
-            BENCHMARK_BRANCH: << parameters.BENCHMARK_BRANCH >>
-            CI_NAME: circleci
-      - run:
-          command: npm install
-          working_directory: << parameters.working_directory >>
-          environment:
-            NUM_PAGES: << parameters.NUM_PAGES >>
-      - run:
-          command: npm run build
-          working_directory: << parameters.working_directory >>
-          environment:
-            BENCHMARK_BUILD_TYPE: WARM_START
-            NUM_PAGES: << parameters.NUM_PAGES >>
-            BENCHMARK_CONTENT_SOURCE: << parameters.BENCHMARK_CONTENT_SOURCE >>
-            BENCHMARK_REPO_NAME: << parameters.BENCHMARK_REPO_NAME >>
-            BENCHMARK_SITE_TYPE: << parameters.BENCHMARK_SITE_TYPE >>
-            BENCHMARK_BRANCH: << parameters.BENCHMARK_BRANCH >>
-            CI_NAME: circleci
-
   e2e-test:
     parameters:
       skip_file_change_test:
@@ -399,6 +347,19 @@ jobs:
           command: yarn run update-source
           working_directory: ~/project/scripts/i18n
 
+  sync_translation_repo:
+    executor: node
+    steps:
+      - checkout
+      - run: git config --global user.name "GatsbyJS Bot"
+      - run: git config --global user.email "core-team@gatsbyjs.com"
+      - run:
+          command: yarn
+          working_directory: ~/project/scripts/i18n
+      - run:
+          command: yarn run-all sync
+          working_directory: ~/project/scripts/i18n
+
   windows_unit_tests:
     executor:
       name: win/vs2019
@@ -439,162 +400,19 @@ jobs:
           paths:
             - "*"
 
-  benchmark_markdown_id_512:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_id
-          NUM_PAGES: "512"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_id
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_id_4096:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_id
-          NUM_PAGES: "4096"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_id
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_id_8192:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_id
-          NUM_PAGES: "8192"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_id
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_id_32768:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_id
-          NUM_PAGES: "32768"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_id
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_slug_512:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_slug
-          NUM_PAGES: "512"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_slug
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_slug_4096:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_slug
-          NUM_PAGES: "4096"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_slug
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_slug_8192:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_slug
-          NUM_PAGES: "8192"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_slug
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_slug_32768:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_slug
-          NUM_PAGES: "32768"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_slug
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_table_512:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_table
-          NUM_PAGES: "512"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_table
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_table_4096:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_table
-          NUM_PAGES: "4096"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_table
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_table_8192:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_table
-          NUM_PAGES: "8192"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_table
-          BENCHMARK_SITE_TYPE: BLOG
-
-  benchmark_markdown_table_32768:
-    docker:
-      - image: "circleci/node:12"
-    steps:
-      - run-benchmark:
-          working_directory: benchmarks/markdown_table/benchmarks/markdown_table
-          NUM_PAGES: "32768"
-          BENCHMARK_CONTENT_SOURCE: MDX
-          BENCHMARK_REPO_NAME: gatsbyjs/gatsby/benchmarks/markdown_table
-          BENCHMARK_SITE_TYPE: BLOG
-
 workflows:
   version: 2
 
-  benchmark:
-    jobs:
-      - benchmark_markdown_id_512
-      - benchmark_markdown_id_4096
-      - benchmark_markdown_id_8192
-      - benchmark_markdown_id_32768
-      - benchmark_markdown_slug_512
-      - benchmark_markdown_slug_4096
-      - benchmark_markdown_slug_8192
-      - benchmark_markdown_slug_32768
-      - benchmark_markdown_table_512
-      - benchmark_markdown_table_4096
-      - benchmark_markdown_table_8192
-      - benchmark_markdown_table_32768
+  weekly-i18n-sync:
     triggers:
       - schedule:
-          cron: 22 16 * * *
+          cron: "0 1 * * 6"
           filters:
             branches:
               only:
                 - master
+    jobs:
+      - sync_translation_repo
 
   nightly-react-next:
     triggers:

benchmarks/README.md

@@ -10,7 +10,7 @@ The standard interface for running a benchmark is:
 cd {benchmark directory}
 export NUM_PAGES={n}
 npm install
-gatsby build
+npm run build / gatsby build
 ```
 
 If a specific benchmark needs to perform some code generation (e.g. `markdown_id`),

benchmarks/mdx/.nvmrc

@@ -0,0 +1 @@
+12

benchmarks/mdx/README.md

@@ -1,25 +1,13 @@
 # MDX Benchmark
 
-This Gatsby site generates MDX files and downloads sample images on postinstall and places them in `src/articles`. In order for that to work, the env var `BENCHMARK_LEVEL` needs to be set to a number when you call `yarn` to install, so:
+Benchmark for MDX pages.
+Mock data is generated during the dependency installation step.
+The number of pages generated is read from `NUM_PAGES=` (defaults to `512`).
 
 ```
-BENCHMARK_LEVEL=1 yarn
-
-gatsby build
+$ export NUM_PAGES=1024
+$ yarn install    # or npm install
+$ gatsby build    # or npm run build
 ```
 
-If you see "Error: Expected an integer but received: " then you did not set the level env var.
-
 This will not use an external source for data while building, but will download some remote images while installing.
-
-# Levels
-
-The `BENCHMARK_LEVEL` is the amount of times to multiply 512 by 2. The level is used to determine how many articles to generate.
-
-Level 1 = 512
-
-Level 2 = 1024
-
-Level 3 = 2048
-
-etc.

benchmarks/mdx/package.json

@@ -9,14 +9,15 @@
     "build:send": "cross-env BENCHMARK_REPORTING_URL=true gatsby build",
     "develop": "gatsby develop",
     "format": "prettier --write \"**/*.{js,jsx,json,md}\"",
-    "postinstall": "npx willit --type=\"mdx\" --level=${BENCHMARK_LEVEL}",
+    "postinstall": "del-cli src/articles && gatsby clean && willit --type=mdx --num-pages=${NUM_PAGES:-512}",
     "start": "npm run develop",
     "serve": "gatsby serve"
   },
   "resolutions": {
     "sharp": "0.25.1"
   },
   "dependencies": {
+    "del-cli": "^3.0.0",
     "@mdx-js/mdx": "^1.5.7",
     "@mdx-js/react": "^1.5.7",
     "dotenv": "^8.2.0",
@@ -34,11 +35,12 @@
   "devDependencies": {
     "cross-env": "^7.0.0",
     "gatsby-plugin-benchmark-reporting": "*",
-    "prettier": "^1.19.1"
+    "prettier": "^1.19.1",
+    "willit": "*"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/gatsbyjs/benchmark/mdx"
+    "url": "https://github.com/gatsbyjs/gatsby/tree/master/benchmarks/mdx"
   },
   "bugs": {
     "url": "https://github.com/gatsbyjs/gatsby/issues"

benchmarks/source-contentful/.env.example

@@ -6,8 +6,8 @@ BENCHMARK_DATASOURCE_LOCAL_PATH='/path/to/benchmark/_dataset
 BENCHMARK_SITE_ID='site1'
 
 # Required for `yarn setup` script only
-CONTENTFUL_MANAGEMENT_TOKEN=''
+BENCHMARK_CONTENTFUL_MANAGEMENT_TOKEN=''
 
 # Required for `yarn build`
-CONTENTFUL_SPACE_ID=''
-CONTENTFUL_ACCESS_TOKEN=''
+BENCHMARK_CONTENTFUL_SPACE_ID=''
+BENCHMARK_CONTENTFUL_ACCESS_TOKEN=''

benchmarks/source-contentful/.nvmrc

@@ -0,0 +1 @@
+12

benchmarks/source-contentful/README.md

@@ -7,16 +7,28 @@ Those individual article pages and the homepage share a common "Layout" componen
 
 ## Setup Contentful benchmark site
 
-1. TODO <Setup data source>
+1. Setup will-it-build data source
 2. Copy `.env.example` to `.env.development` and make sure all variables are set
 3. Run `yarn setup`
 
 Note that the script is idempotent, so you can re-run it on failures.
 
 Also use `yarn setup --skip [N:number]` to skip first `N` articles
-(for example articles created during a previous run)
+(for example articles created during a previous run which failed)
+
+### Fixing broken images
+
+Sometimes Contentful silently fails to process images which causes builds to fail.
+Use following approach to fix those:
+
+1. Run `yarn site find-broken-images`
+2. Change image URLs in will-it-build dataset for this site to some other images
+   (or just use one of the larger sites and set `BENCHMARK_SITE_ID` appropriately)
+3. Run `yarn site fix-broken-images imageid1 imageid2 imageid3`
+   This command updates broken images with images from the `BENCHMARK_SITE_ID` dataset
 
 ## Build a site
 
-1. Copy `.env.example` to `.env.production` and make sure all variables are set
-2. Run `yarn build`
+1. Copy `.env.example` to `.env.production`
+2. Set `BENCHMARK_CONTENTFUL_SPACE_ID` and `BENCHMARK_CONTENTFUL_ACCESS_TOKEN` variables
+3. Run `yarn build`