Parallelize the build script

Uses CircleCI's `parallelism` config option to spin up multiple build processes.
facebook · May 29, 2019 · 84e7c0c · 84e7c0c
1 parent 245595c
commit 84e7c0c
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 25 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -118,15 +118,14 @@ jobs:
   build:
     docker: *docker
     environment: *environment
+    parallelism: 20
     steps:
       - checkout
       - *restore_yarn_cache
       - *run_yarn
       - run: ./scripts/circleci/add_build_info_json.sh
       - run: ./scripts/circleci/update_package_versions.sh
       - run: yarn build
-      - store_artifacts:
-          path: ./scripts/error-codes/codes.json
       - persist_to_workspace:
           root: build
           paths:
@@ -153,6 +152,10 @@ jobs:
           path: ./build.tgz
       - store_artifacts:
           path: ./build/bundle-sizes.json
+      - store_artifacts:
+          # TODO: Update release script to use local file instead of pulling
+          # from artifacts.
+          path: ./scripts/error-codes/codes.json
 
   lint_build:
     docker: *docker
@@ -223,9 +226,6 @@ workflows:
       - test_source_fire:
           requires:
             - setup
-      - test_coverage:
-          requires:
-            - setup
       - build:
           requires:
             - setup
@@ -237,10 +237,10 @@ workflows:
             - build
       - test_build:
           requires:
-            - process_artifacts
+            - build
       - test_build_prod:
           requires:
-            - process_artifacts
+            - build
   hourly:
     triggers:
       - schedule:
@@ -254,3 +254,6 @@ workflows:
       - test_fuzz:
           requires:
             - setup
+      - test_coverage:
+          requires:
+            - setup
diff --git a/scripts/rollup/build.js b/scripts/rollup/build.js
@@ -21,7 +21,7 @@ const useForks = require('./plugins/use-forks-plugin');
 const stripUnusedImports = require('./plugins/strip-unused-imports');
 const extractErrorCodes = require('../error-codes/extract-errors');
 const Packaging = require('./packaging');
-const {asyncCopyTo, asyncRimRaf} = require('./utils');
+const {asyncCopyTo} = require('./utils');
 const codeFrame = require('babel-code-frame');
 const Wrappers = require('./wrappers');
 
@@ -634,27 +634,41 @@ function handleRollupError(error) {
 }
 
 async function buildEverything() {
-  await asyncRimRaf('build');
-
   // Run them serially for better console output
   // and to avoid any potential race conditions.
+
+  let bundles = [];
   // eslint-disable-next-line no-for-of-loops/no-for-of-loops
   for (const bundle of Bundles.bundles) {
-    await createBundle(bundle, UMD_DEV);
-    await createBundle(bundle, UMD_PROD);
-    await createBundle(bundle, UMD_PROFILING);
-    await createBundle(bundle, NODE_DEV);
-    await createBundle(bundle, NODE_PROD);
-    await createBundle(bundle, NODE_PROFILING);
-    await createBundle(bundle, FB_WWW_DEV);
-    await createBundle(bundle, FB_WWW_PROD);
-    await createBundle(bundle, FB_WWW_PROFILING);
-    await createBundle(bundle, RN_OSS_DEV);
-    await createBundle(bundle, RN_OSS_PROD);
-    await createBundle(bundle, RN_OSS_PROFILING);
-    await createBundle(bundle, RN_FB_DEV);
-    await createBundle(bundle, RN_FB_PROD);
-    await createBundle(bundle, RN_FB_PROFILING);
+    bundles.push(
+      [bundle, UMD_DEV],
+      [bundle, UMD_PROD],
+      [bundle, UMD_PROFILING],
+      [bundle, NODE_DEV],
+      [bundle, NODE_PROD],
+      [bundle, NODE_PROFILING],
+      [bundle, FB_WWW_DEV],
+      [bundle, FB_WWW_PROD],
+      [bundle, FB_WWW_PROFILING],
+      [bundle, RN_OSS_DEV],
+      [bundle, RN_OSS_PROD],
+      [bundle, RN_OSS_PROFILING],
+      [bundle, RN_FB_DEV],
+      [bundle, RN_FB_PROD],
+      [bundle, RN_FB_PROFILING]
+    );
+  }
+
+  if (!shouldExtractErrors && process.env.CIRCLE_NODE_TOTAL) {
+    // In CI, parallelize bundles across multiple tasks.
+    const nodeTotal = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
+    const nodeIndex = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
+    bundles = bundles.filter((_, i) => i % nodeTotal === nodeIndex);
+  }
+
+  // eslint-disable-next-line no-for-of-loops/no-for-of-loops
+  for (const [bundle, bundleType] of bundles) {
+    await createBundle(bundle, bundleType);
   }
 
   await Packaging.copyAllShims();