refactor: merge turbopack crates into next.js (#68127)

Moves all turbopack crates (and a few js packages) into the next.js
repo.

https://vercel.com/blog/turbopack-moving-homes

`git-filter-repo` https://github.com/newren/git-filter-repo (better than
filter-branch)
and
`git subtree`

See: https://github.com/vercel/next-turbo-merge

Short version:
```bash
git clone git@github.com:vercel/next.js.git --filter=blob:none --no-tags --single-branch --branch canary "$NEXT_DIR"

git --git-dir="$NEXT_DIR/.git" remote add --fetch --track main --no-tags turbo git@github.com:vercel/turbo

git switch -C hrmny/turbopack-staging turbo/main --no-track

git filter-repo --force --refs hrmny/turbopack-staging \
  --path crates/ \
  --path xtask/ \
  --path packages/devlow-bench/ \
  --path packages/node-file-trace/ \
  --path packages/node-module-trace/ \
  --path packages/turbo-pack/ \
  --path packages/turbo-tasks/ \
  --path packages/turbo-tasks-examples/ \
  --path packages/turbo-tasks-fs/ \
  --path packages/turbo-tasks-macros/ \
  --path packages/turbo-tracing-next-plugin/ \
  --path packages/webpack-nmt/

git filter-repo --force --refs hrmny/turbopack-staging \
  --path-glob 'crates/turborepo*' --invert-paths

git filter-repo --refs hrmny/turbopack-staging --commit-callback '
  commit.message = re.sub(b"(^|\\(| )(#\\d\\d\\d\\d)(\\)| |\\r|$)", b"\\1vercel/turbo\\2\\3", commit.message, flags=re.MULTILINE)
'

git gc

git switch -C hrmny/merge-turbopack origin/canary --no-track

git subtree add -P turbopack hrmny/turbopack-staging
```

Closes PACK-3163

.cargo/config.toml

@@ -1,5 +1,6 @@
 [env]
 CARGO_WORKSPACE_DIR = { value = "", relative = true }
+TURBO_PNPM_WORKSPACE_DIR = { value = "", relative = true }
 
 [build]
 rustflags = [

.config/ast-grep/rule-tests/__snapshots__/no-context-snapshot.yml

@@ -0,0 +1,42 @@
+id: no-context
+snapshots:
+  'fn foo(context: ChunkingContext) -> u32 { 5 };':
+    labels:
+      - source: context
+        style: primary
+        start: 7
+        end: 14
+      - source: 'context: ChunkingContext'
+        style: secondary
+        start: 7
+        end: 31
+  foo(|context| context):
+    labels:
+      - source: context
+        style: primary
+        start: 5
+        end: 12
+      - source: '|context|'
+        style: secondary
+        start: 4
+        end: 13
+  let context = ChunkingContext::new();:
+    labels:
+      - source: context
+        style: primary
+        start: 4
+        end: 11
+      - source: let context = ChunkingContext::new();
+        style: secondary
+        start: 0
+        end: 37
+  'struct Foo { context: Context };':
+    labels:
+      - source: context
+        style: primary
+        start: 13
+        end: 20
+      - source: 'context: Context'
+        style: secondary
+        start: 13
+        end: 29

.config/ast-grep/rule-tests/no-context-test.yml

@@ -0,0 +1,11 @@
+id: no-context
+valid:
+  - 'let chunking_context = ChunkingContext::new();'
+  - 'struct Foo { chunking_context: Context };'
+  - 'foo(|chunking_context| context)'
+  - 'fn foo(chunking_context: ChunkingContext) -> u32 { 5 };'
+invalid:
+  - 'let context = ChunkingContext::new();'
+  - 'struct Foo { context: Context };'
+  - 'foo(|context| context)'
+  - 'fn foo(context: ChunkingContext) -> u32 { 5 };'

.config/ast-grep/rules/no-context.yml

@@ -0,0 +1,35 @@
+id: no-context
+message: Don't name variables `context`.
+note: Use a more specific name, such as chunking_context, asset_context, etc.
+severity: error
+language: Rust
+rule:
+  regex: \bcontext\b
+  any:
+    - all:
+        - inside:
+            any:
+              - kind: closure_parameters
+              - kind: parameter
+              - kind: function_item
+              - kind: let_declaration
+        - kind: identifier
+    - all:
+        - kind: field_identifier
+        - inside:
+            kind: field_declaration
+ignores:
+  - './crates/turbopack-css/**'
+  - './crates/turbopack-dev-server/**'
+  - './crates/turbopack-browser/**'
+  - './crates/turbopack-ecmascript-hmr-protocol/**'
+  - './crates/turbopack-ecmascript-plugins/**'
+  - './crates/turbopack-ecmascript-runtime/**'
+  - './crates/turbopack-json/**'
+  - './crates/turbopack-mdx/**'
+  - './crates/turbopack-node/**'
+  - './crates/turbopack-static/**'
+  - './crates/turbopack-tests/**'
+  - './crates/turbopack/**'
+  - './crates/turborepo-cache/**'
+  - './crates/turborepo-scm/**'

.config/nextest.toml

@@ -0,0 +1,17 @@
+[profile.tp-test-linux.junit]
+path = "junit.xml"
+report-name = "Turbopack tests (Linux)"
+store-success-output = true
+store-failure-output = true
+
+[profile.tp-test-mac.junit]
+path = "junit.xml"
+report-name = "Turbopack tests (Mac)"
+store-success-output = true
+store-failure-output = true
+
+[profile.tp-test-win.junit]
+path = "junit.xml"
+report-name = "Turbopack tests (Windows)"
+store-success-output = true
+store-failure-output = true

.eslintignore

@@ -4,6 +4,7 @@ node_modules
 **/.vscode/**
 **/dist/**
 e2e-tests/**
+examples/cms-sanity/sanity.types.ts
 examples/with-eslint/**
 examples/with-typescript-eslint-jest/**
 examples/with-kea/**
@@ -26,18 +27,25 @@ packages/next-codemod/**/*.js
 packages/next-codemod/**/*.d.ts
 packages/next-env/**/*.d.ts
 packages/create-next-app/templates/**
-test/integration/eslint/**
-test/integration/script-loader/**/*
-test/development/basic/legacy-decorators/**/*
+test/integration/eslint/**/*.js
+test/integration/script-loader/**/*.js
+test/development/basic/legacy-decorators/**/*.js
 test/production/emit-decorator-metadata/**/*.js
+!test/**/*.test.*
 test/e2e/app-dir/rsc-errors/app/swc/use-client/page.js
 test-timings.json
 crates/**
 bench/nested-deps/**
 bench/nested-deps-app-router/**
+bench/heavy-npm-deps/**
 packages/next-bundle-analyzer/index.d.ts
 examples/with-typescript-graphql/lib/gql/
 test/development/basic/hmr/components/parse-error.js
 packages/next-swc/docs/assets/**/*
 test/lib/amp-validator-wasm.js
 test/production/pages-dir/production/fixture/amp-validator-wasm.js
+test/e2e/async-modules/amp-validator-wasm.js
+
+# turbopack crates
+turbopack/crates/*/tests/**
+turbopack/crates/*/js/src/compiled

.eslintrc.cli.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://json.schemastore.org/eslintrc",
+  "extends": [".eslintrc.json"],
+  "overrides": [
+    {
+      // This override adds type-checked rules.
+      "files": ["**/*.ts", "**/*.tsx"],
+      // Linting with type-checked rules is very slow and needs a lot of memory,
+      // so we exclude non-essential files.
+      "excludedFiles": [
+        "examples/**/*",
+        "test/**/*",
+        "**/*.d.ts",
+        "turbopack/**/*"
+      ],
+      "parserOptions": {
+        "project": true
+      },
+      "rules": {
+        // TODO: enable in follow-up PR
+        "@typescript-eslint/no-floating-promises": "off"
+      }
+    }
+  ]
+}

.eslintrc.json

@@ -7,7 +7,8 @@
     "commonjs": true,
     "es6": true,
     "node": true,
-    "jest": true
+    "jest": true,
+    "es2020": true
   },
   "parserOptions": {
     "requireConfigFile": false,
@@ -96,7 +97,11 @@
           "warn",
           {
             "args": "none",
-            "ignoreRestSiblings": true
+            "ignoreRestSiblings": true,
+            "argsIgnorePattern": "^_",
+            "caughtErrorsIgnorePattern": "^_",
+            "destructuredArrayIgnorePattern": "^_",
+            "varsIgnorePattern": "^_"
           }
         ],
         "no-use-before-define": "off",

.gitattributes

@@ -3,3 +3,7 @@ packages/next/compiled/** -text linguist-vendored
 
 # Make next/src/build folder indexable for github search
 build/** linguist-generated=false
+
+turbopack/crates/turbo-tasks-macros-tests/tests/**/*.stderr linguist-generated=true
+turbopack/crates/turbopack-ecmascript/tests/tree-shaker/analyzer/**/output.md linguist-generated=true
+turbopack/crates/turbopack-tests/tests/snapshot/**/output/** linguist-generated=true

.github/workflows/build_and_test.yml

@@ -117,13 +117,21 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       needsRust: 'yes'
-      skipInstallBuild: 'yes'
+      needsNextest: 'yes'
       skipNativeBuild: 'yes'
       afterBuild: turbo run test-cargo-unit
       mold: 'yes'
       stepName: 'test-cargo-unit'
     secrets: inherit
 
+  test-bench:
+    name: test cargo benches
+    needs: ['changes', 'build-next']
+    if: ${{ needs.changes.outputs.docs-only == 'false' }}
+
+    uses: ./.github/workflows/test-turbopack-rust-bench-test.yml
+    secrets: inherit
+
   rust-check:
     name: rust check
     needs: ['changes', 'build-next']

.github/workflows/build_reusable.yml

@@ -157,6 +157,11 @@ jobs:
       - run: pnpm install
         if: ${{ inputs.skipInstallBuild != 'yes' }}
 
+      - name: Install node-file-trace test dependencies
+        if: ${{ inputs.needsNextest == 'yes' }}
+        working-directory: turbopack/crates/turbopack/tests/node-file-trace
+        run: pnpm install -r --side-effects-cache false
+
       - run: ANALYZE=1 pnpm build
         if: ${{ inputs.skipInstallBuild != 'yes' }}
 

.github/workflows/test-turbopack-rust-bench-test.yml

@@ -0,0 +1,65 @@
+name: Turbopack Rust testing benchmarks
+on:
+  workflow_call:
+    inputs:
+      runner:
+        type: string
+        default: '["self-hosted", "linux", "x64", "metal"]'
+      os:
+        type: string
+        default: 'linux'
+      all:
+        type: boolean
+        default: false
+
+env:
+  TURBOPACK_BENCH_COUNTS: '100'
+  TURBOPACK_BENCH_PROGRESS: '1'
+
+  NODE_LTS_VERSION: 20
+
+jobs:
+  test:
+    name: Test
+    runs-on: ${{ fromJSON(inputs.runner) }}
+    steps:
+      - name: Set git to use LF
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+        if: inputs.os == 'windows'
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_LTS_VERSION }}
+          check-latest: true
+      - run: corepack enable
+
+      - name: Build benchmarks for tests
+        timeout-minutes: 120
+        run: |
+          cargo test --benches --workspace --release --no-fail-fast --exclude turbopack-bench --exclude next-swc-napi --no-run
+
+      - name: Run cargo test on benchmarks
+        timeout-minutes: 120
+        run: |
+          cargo test --benches --workspace --release --no-fail-fast --exclude turbopack-bench --exclude next-swc-napi
+
+      - name: Build benchmarks for tests for other bundlers
+        if: inputs.all
+        timeout-minutes: 120
+        run: |
+          cargo test --benches --release -p turbopack-bench --no-run
+
+      - name: Run cargo test on benchmarks for other bundlers
+        if: inputs.all
+        timeout-minutes: 120
+        run: |
+          cargo test --benches --release -p turbopack-bench

.prettierignore

@@ -46,3 +46,18 @@ bench/nested-deps/components/**/*
 **/.tina/__generated__/**
 test/lib/amp-validator-wasm.js
 test/production/pages-dir/production/fixture/amp-validator-wasm.js
+test/e2e/async-modules/amp-validator-wasm.js
+
+# turbopack crates
+turbopack/crates/*/js/src/compiled
+turbopack/crates/turbopack/bench.json
+turbopack/crates/turbopack/tests
+turbopack/crates/turbopack-ecmascript/tests/analyzer/graph
+turbopack/crates/turbopack-ecmascript/tests/tree-shaker
+turbopack/crates/next-transform-strip-page-exports/tests
+turbopack/crates/next-transform-dynamic/tests
+turbopack/crates/turbopack-tests/tests/execution/turbopack/basic/error/input/broken.js
+turbopack/crates/turbopack-tests/tests/**/output*
+
+# temporarily disable prettier for the turbopack directory
+turbopack/