Skip to content

Commit eb0fb38

Browse files
authored
Build stable and experimental with same command (#20573)
The goal is to simplify our CI pipeline so that all configurations are built and tested in a single workflow. As a first step, this adds a new build script entry point that builds both the experimental and stable release channels into a single artifacts directory. The script works by wrapping the existing build script (which only builds a single release channel at a time), then post-processing the results to match the desired filesystem layout. A future version of the build script would output the files directly without post-processing. Because many parts of our infra depend on the existing layout of the build artifacts directory, I have left the old workflows untouched. We can incremental migrate to the new layout, then delete the old workflows after we've finished.
1 parent e8eff11 commit eb0fb38

File tree

4 files changed

+175
-0
lines changed

4 files changed

+175
-0
lines changed

Diff for: .circleci/config.yml

+50
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,45 @@ jobs:
293293
- dist
294294
- sizes/*.json
295295

296+
yarn_build_combined:
297+
docker: *docker
298+
environment: *environment
299+
parallelism: 40
300+
steps:
301+
- checkout
302+
- run: yarn workspaces info | head -n -1 > workspace_info.txt
303+
- *restore_node_modules
304+
- run:
305+
command: |
306+
./scripts/circleci/add_build_info_json.sh
307+
./scripts/circleci/update_package_versions.sh
308+
yarn build-combined
309+
- persist_to_workspace:
310+
root: build2
311+
paths:
312+
- facebook-www
313+
- facebook-react-native
314+
- facebook-relay
315+
- oss-stable
316+
- oss-experimental
317+
- react-native
318+
- dist
319+
- sizes/*.json
320+
321+
process_artifacts_combined:
322+
docker: *docker
323+
environment: *environment
324+
steps:
325+
- checkout
326+
- attach_workspace:
327+
at: build2
328+
- run: yarn workspaces info | head -n -1 > workspace_info.txt
329+
- *restore_node_modules
330+
# Compress build directory into a single tarball for easy download
331+
- run: tar -zcvf ./build2.tgz ./build2
332+
- store_artifacts:
333+
path: ./build2.tgz
334+
296335
build_devtools_and_process_artifacts:
297336
docker: *docker
298337
environment: *environment
@@ -611,6 +650,17 @@ workflows:
611650
only:
612651
- master
613652

653+
# New workflow that will replace "stable" and "experimental"
654+
combined:
655+
jobs:
656+
- setup
657+
- yarn_build_combined:
658+
requires:
659+
- setup
660+
- process_artifacts_combined:
661+
requires:
662+
- yarn_build_combined
663+
614664
fuzz_tests:
615665
triggers:
616666
- schedule:

Diff for: .gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ scripts/flow/*/.flowconfig
88
_SpecRunner.html
99
__benchmarks__
1010
build/
11+
build2/
1112
remote-repo/
1213
coverage/
1314
.module-cache

Diff for: package.json

+1
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@
106106
},
107107
"scripts": {
108108
"build": "node ./scripts/rollup/build.js",
109+
"build-combined": "node ./scripts/rollup/build-all-release-channels.js",
109110
"build-for-devtools": "cross-env RELEASE_CHANNEL=experimental yarn build react/index,react-dom,react-is,react-debug-tools,scheduler,react-test-renderer,react-refresh",
110111
"build-for-devtools-dev": "yarn build-for-devtools --type=NODE_DEV",
111112
"build-for-devtools-prod": "yarn build-for-devtools --type=NODE_PROD",

Diff for: scripts/rollup/build-all-release-channels.js

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
'use strict';
2+
3+
/* eslint-disable no-for-of-loops/no-for-of-loops */
4+
5+
const fs = require('fs');
6+
const {spawnSync} = require('child_process');
7+
const tmp = require('tmp');
8+
9+
// Runs the build script for both stable and experimental release channels,
10+
// by configuring an environment variable.
11+
12+
if (process.env.CIRCLE_NODE_TOTAL) {
13+
// In CI, we use multiple concurrent processes. Allocate half the processes to
14+
// build the stable channel, and the other half for experimental. Override
15+
// the environment variables to "trick" the underlying build script.
16+
const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
17+
const halfTotal = Math.floor(total / 2);
18+
const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
19+
if (index < halfTotal) {
20+
const nodeTotal = halfTotal;
21+
const nodeIndex = index;
22+
buildForChannel('stable', nodeTotal, nodeIndex);
23+
processStable('./build');
24+
} else {
25+
const nodeTotal = total - halfTotal;
26+
const nodeIndex = index - halfTotal;
27+
buildForChannel('experimental', nodeTotal, nodeIndex);
28+
processExperimental('./build');
29+
}
30+
31+
// TODO: Currently storing artifacts as `./build2` so that it doesn't conflict
32+
// with old build job. Remove once we migrate rest of build/test pipeline.
33+
fs.renameSync('./build', './build2');
34+
} else {
35+
// Running locally, no concurrency. Move each channel's build artifacts into
36+
// a temporary directory so that they don't conflict.
37+
buildForChannel('stable', '', '');
38+
const stableDir = tmp.dirSync().name;
39+
fs.renameSync('./build', stableDir);
40+
processStable(stableDir);
41+
42+
buildForChannel('experimental', '', '');
43+
const experimentalDir = tmp.dirSync().name;
44+
fs.renameSync('./build', experimentalDir);
45+
processExperimental(experimentalDir);
46+
47+
// Then merge the experimental folder into the stable one. processExperimental
48+
// will have already removed conflicting files.
49+
//
50+
// In CI, merging is handled automatically by CircleCI's workspace feature.
51+
spawnSync('rsync', ['-ar', experimentalDir + '/', stableDir + '/']);
52+
53+
// Now restore the combined directory back to its original name
54+
// TODO: Currently storing artifacts as `./build2` so that it doesn't conflict
55+
// with old build job. Remove once we migrate rest of build/test pipeline.
56+
fs.renameSync(stableDir, './build2');
57+
}
58+
59+
function buildForChannel(channel, nodeTotal, nodeIndex) {
60+
spawnSync('node', ['./scripts/rollup/build.js', ...process.argv.slice(2)], {
61+
stdio: ['pipe', process.stdout, process.stderr],
62+
env: {
63+
...process.env,
64+
RELEASE_CHANNEL: channel,
65+
CIRCLE_NODE_TOTAL: nodeTotal,
66+
CIRCLE_NODE_INDEX: nodeIndex,
67+
},
68+
});
69+
}
70+
71+
function processStable(buildDir) {
72+
if (fs.existsSync(buildDir + '/node_modules')) {
73+
fs.renameSync(buildDir + '/node_modules', buildDir + '/oss-stable');
74+
}
75+
76+
if (fs.existsSync(buildDir + '/facebook-www')) {
77+
for (const fileName of fs.readdirSync(buildDir + '/facebook-www')) {
78+
const filePath = buildDir + '/facebook-www/' + fileName;
79+
const stats = fs.statSync(filePath);
80+
if (!stats.isDirectory()) {
81+
fs.renameSync(filePath, filePath.replace('.js', '.classic.js'));
82+
}
83+
}
84+
}
85+
86+
if (fs.existsSync(buildDir + '/sizes')) {
87+
fs.renameSync(buildDir + '/sizes', buildDir + '/sizes-stable');
88+
}
89+
}
90+
91+
function processExperimental(buildDir) {
92+
if (fs.existsSync(buildDir + '/node_modules')) {
93+
fs.renameSync(buildDir + '/node_modules', buildDir + '/oss-experimental');
94+
}
95+
96+
if (fs.existsSync(buildDir + '/facebook-www')) {
97+
for (const fileName of fs.readdirSync(buildDir + '/facebook-www')) {
98+
const filePath = buildDir + '/facebook-www/' + fileName;
99+
const stats = fs.statSync(filePath);
100+
if (!stats.isDirectory()) {
101+
fs.renameSync(filePath, filePath.replace('.js', '.modern.js'));
102+
}
103+
}
104+
}
105+
106+
if (fs.existsSync(buildDir + '/sizes')) {
107+
fs.renameSync(buildDir + '/sizes', buildDir + '/sizes-experimental');
108+
}
109+
110+
// Delete all other artifacts that weren't handled above. We assume they are
111+
// duplicates of the corresponding artifacts in the stable channel. Ideally,
112+
// the underlying build script should not have produced these files in the
113+
// first place.
114+
for (const pathName of fs.readdirSync(buildDir)) {
115+
if (
116+
pathName !== 'oss-experimental' &&
117+
pathName !== 'facebook-www' &&
118+
pathName !== 'sizes-experimental'
119+
) {
120+
spawnSync('rm', ['-rm', buildDir + '/' + pathName]);
121+
}
122+
}
123+
}

0 commit comments

Comments
 (0)