refactor: merge turbopack crates into next.js

.cargo/config.toml

@@ -1,5 +1,6 @@
 [env]
 CARGO_WORKSPACE_DIR = { value = "", relative = true }
+TURBO_PNPM_WORKSPACE_DIR = { value = "", relative = true }
 
 [build]
 rustflags = [

.config/ast-grep/rule-tests/__snapshots__/no-context-snapshot.yml

@@ -0,0 +1,42 @@
+id: no-context
+snapshots:
+  'fn foo(context: ChunkingContext) -> u32 { 5 };':
+    labels:
+      - source: context
+        style: primary
+        start: 7
+        end: 14
+      - source: 'context: ChunkingContext'
+        style: secondary
+        start: 7
+        end: 31
+  foo(|context| context):
+    labels:
+      - source: context
+        style: primary
+        start: 5
+        end: 12
+      - source: '|context|'
+        style: secondary
+        start: 4
+        end: 13
+  let context = ChunkingContext::new();:
+    labels:
+      - source: context
+        style: primary
+        start: 4
+        end: 11
+      - source: let context = ChunkingContext::new();
+        style: secondary
+        start: 0
+        end: 37
+  'struct Foo { context: Context };':
+    labels:
+      - source: context
+        style: primary
+        start: 13
+        end: 20
+      - source: 'context: Context'
+        style: secondary
+        start: 13
+        end: 29

.config/ast-grep/rule-tests/no-context-test.yml

@@ -0,0 +1,11 @@
+id: no-context
+valid:
+  - 'let chunking_context = ChunkingContext::new();'
+  - 'struct Foo { chunking_context: Context };'
+  - 'foo(|chunking_context| context)'
+  - 'fn foo(chunking_context: ChunkingContext) -> u32 { 5 };'
+invalid:
+  - 'let context = ChunkingContext::new();'
+  - 'struct Foo { context: Context };'
+  - 'foo(|context| context)'
+  - 'fn foo(context: ChunkingContext) -> u32 { 5 };'

.config/ast-grep/rules/no-context.yml

@@ -0,0 +1,35 @@
+id: no-context
+message: Don't name variables `context`.
+note: Use a more specific name, such as chunking_context, asset_context, etc.
+severity: error
+language: Rust
+rule:
+  regex: \bcontext\b
+  any:
+    - all:
+        - inside:
+            any:
+              - kind: closure_parameters
+              - kind: parameter
+              - kind: function_item
+              - kind: let_declaration
+        - kind: identifier
+    - all:
+        - kind: field_identifier
+        - inside:
+            kind: field_declaration
+ignores:
+  - './crates/turbopack-css/**'
+  - './crates/turbopack-dev-server/**'
+  - './crates/turbopack-browser/**'
+  - './crates/turbopack-ecmascript-hmr-protocol/**'
+  - './crates/turbopack-ecmascript-plugins/**'
+  - './crates/turbopack-ecmascript-runtime/**'
+  - './crates/turbopack-json/**'
+  - './crates/turbopack-mdx/**'
+  - './crates/turbopack-node/**'
+  - './crates/turbopack-static/**'
+  - './crates/turbopack-tests/**'
+  - './crates/turbopack/**'
+  - './crates/turborepo-cache/**'
+  - './crates/turborepo-scm/**'

.config/nextest.toml

@@ -0,0 +1,17 @@
+[profile.tp-test-linux.junit]
+path = "junit.xml"
+report-name = "Turbopack tests (Linux)"
+store-success-output = true
+store-failure-output = true
+
+[profile.tp-test-mac.junit]
+path = "junit.xml"
+report-name = "Turbopack tests (Mac)"
+store-success-output = true
+store-failure-output = true
+
+[profile.tp-test-win.junit]
+path = "junit.xml"
+report-name = "Turbopack tests (Windows)"
+store-success-output = true
+store-failure-output = true

.eslintignore

@@ -45,3 +45,7 @@ packages/next-swc/docs/assets/**/*
 test/lib/amp-validator-wasm.js
 test/production/pages-dir/production/fixture/amp-validator-wasm.js
 test/e2e/async-modules/amp-validator-wasm.js
+
+# turbopack crates
+turbopack/crates/*/tests/**
+turbopack/crates/*/js/src/compiled

.eslintrc.cli.json

@@ -7,7 +7,12 @@
       "files": ["**/*.ts", "**/*.tsx"],
       // Linting with type-checked rules is very slow and needs a lot of memory,
       // so we exclude non-essential files.
-      "excludedFiles": ["examples/**/*", "test/**/*", "**/*.d.ts"],
+      "excludedFiles": [
+        "examples/**/*",
+        "test/**/*",
+        "**/*.d.ts",
+        "turbopack/**/*"
+      ],
       "parserOptions": {
         "project": true
       },

.eslintrc.json

@@ -12,7 +12,8 @@
     "commonjs": true,
     "es6": true,
     "node": true,
-    "jest": true
+    "jest": true,
+    "es2020": true
   },
   "parserOptions": {
     "requireConfigFile": false,
@@ -101,7 +102,11 @@
           "error",
           {
             "args": "none",
-            "ignoreRestSiblings": true
+            "ignoreRestSiblings": true,
+            "argsIgnorePattern": "^_",
+            "caughtErrorsIgnorePattern": "^_",
+            "destructuredArrayIgnorePattern": "^_",
+            "varsIgnorePattern": "^_"
           }
         ],
         "no-use-before-define": "off",

.gitattributes

@@ -3,3 +3,7 @@ packages/next/compiled/** -text linguist-vendored
 
 # Make next/src/build folder indexable for github search
 build/** linguist-generated=false
+
+turbopack/crates/turbo-tasks-macros-tests/tests/**/*.stderr linguist-generated=true
+turbopack/crates/turbopack-ecmascript/tests/tree-shaker/analyzer/**/output.md linguist-generated=true
+turbopack/crates/turbopack-tests/tests/snapshot/**/output/** linguist-generated=true

.github/workflows/build_and_test.yml

@@ -116,13 +116,21 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       needsRust: 'yes'
-      skipInstallBuild: 'yes'
+      needsNextest: 'yes'
       skipNativeBuild: 'yes'
       afterBuild: turbo run test-cargo-unit
       mold: 'yes'
       stepName: 'test-cargo-unit'
     secrets: inherit
 
+  test-bench:
+    name: test cargo benches
+    needs: ['changes', 'build-next']
+    if: ${{ needs.changes.outputs.docs-only == 'false' }}
+
+    uses: ./.github/workflows/test-turbopack-rust-bench-test.yml
+    secrets: inherit
+
   rust-check:
     name: rust check
     needs: ['changes', 'build-next']

.github/workflows/build_reusable.yml

@@ -176,6 +176,11 @@ jobs:
       - run: pnpm install
         if: ${{ inputs.skipInstallBuild != 'yes' }}
 
+      - name: Install node-file-trace test dependencies
+        if: ${{ inputs.needsNextest == 'yes' }}
+        working-directory: turbopack/crates/turbopack/tests/node-file-trace
+        run: pnpm install -r --side-effects-cache false
+
       - run: ANALYZE=1 pnpm build
         if: ${{ inputs.skipInstallBuild != 'yes' }}
 

.github/workflows/test-turbopack-rust-bench-test.yml

@@ -0,0 +1,65 @@
+name: Turbopack Rust testing benchmarks
+on:
+  workflow_call:
+    inputs:
+      runner:
+        type: string
+        default: '["self-hosted", "linux", "x64", "metal"]'
+      os:
+        type: string
+        default: 'linux'
+      all:
+        type: boolean
+        default: false
+
+env:
+  TURBOPACK_BENCH_COUNTS: '100'
+  TURBOPACK_BENCH_PROGRESS: '1'
+
+  NODE_LTS_VERSION: 20
+
+jobs:
+  test:
+    name: Test
+    runs-on: ${{ fromJSON(inputs.runner) }}
+    steps:
+      - name: Set git to use LF
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+        if: inputs.os == 'windows'
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_LTS_VERSION }}
+          check-latest: true
+      - run: corepack enable
+
+      - name: Build benchmarks for tests
+        timeout-minutes: 120
+        run: |
+          cargo test --benches --workspace --release --no-fail-fast --exclude turbopack-bench --exclude next-swc-napi --no-run
+
+      - name: Run cargo test on benchmarks
+        timeout-minutes: 120
+        run: |
+          cargo test --benches --workspace --release --no-fail-fast --exclude turbopack-bench --exclude next-swc-napi
+
+      - name: Build benchmarks for tests for other bundlers
+        if: inputs.all
+        timeout-minutes: 120
+        run: |
+          cargo test --benches --release -p turbopack-bench --no-run
+
+      - name: Run cargo test on benchmarks for other bundlers
+        if: inputs.all
+        timeout-minutes: 120
+        run: |
+          cargo test --benches --release -p turbopack-bench

.prettierignore

@@ -47,3 +47,17 @@ bench/nested-deps/components/**/*
 test/lib/amp-validator-wasm.js
 test/production/pages-dir/production/fixture/amp-validator-wasm.js
 test/e2e/async-modules/amp-validator-wasm.js
+
+# turbopack crates
+turbopack/crates/*/js/src/compiled
+turbopack/crates/turbopack/bench.json
+turbopack/crates/turbopack/tests
+turbopack/crates/turbopack-ecmascript/tests/analyzer/graph
+turbopack/crates/turbopack-ecmascript/tests/tree-shaker
+turbopack/crates/next-transform-strip-page-exports/tests
+turbopack/crates/next-transform-dynamic/tests
+turbopack/crates/turbopack-tests/tests/execution/turbopack/basic/error/input/broken.js
+turbopack/crates/turbopack-tests/tests/**/output*
+
+# temporarily disable prettier for the turbopack directory
+turbopack/