[ENG-10412] Upload projectMetadata file

CHANGELOG.md

@@ -8,6 +8,8 @@ This is the log of notable changes to EAS CLI and related packages.
 
 ### 🎉 New features
 
+- Generate metadata file for project archive ([#2149](https://github.com/expo/eas-cli/pull/2149) by [@khamilowicz](https://github.com/khamilowicz))
+
 ### 🐛 Bug fixes
 
 ### 🧹 Chores

packages/eas-cli/package.json

@@ -13,7 +13,7 @@
     "@expo/config": "8.5.4",
     "@expo/config-plugins": "7.8.4",
     "@expo/config-types": "50.0.0",
-    "@expo/eas-build-job": "1.0.59",
+    "@expo/eas-build-job": "1.0.61",
     "@expo/eas-json": "7.1.2",
     "@expo/json-file": "8.2.37",
     "@expo/multipart-body-parser": "1.1.0",

packages/eas-cli/src/build/build.ts

@@ -7,26 +7,6 @@ import fs from 'fs-extra';
 import { GraphQLError } from 'graphql/error';
 import nullthrows from 'nullthrows';
 
-import { BuildContext } from './context';
-import {
-  EasBuildDownForMaintenanceError,
-  EasBuildFreeTierDisabledAndroidError,
-  EasBuildFreeTierDisabledError,
-  EasBuildFreeTierDisabledIOSError,
-  EasBuildFreeTierIosLimitExceededError,
-  EasBuildFreeTierLimitExceededError,
-  EasBuildLegacyResourceClassNotAvailableError,
-  EasBuildProjectArchiveUploadError,
-  EasBuildResourceClassNotAvailableInFreeTierError,
-  EasBuildTooManyPendingBuildsError,
-  RequestValidationError,
-  TurtleDeprecatedJobFormatError,
-} from './errors';
-import { transformMetadata } from './graphql';
-import { LocalBuildMode, runLocalBuildAsync } from './local';
-import { collectMetadataAsync } from './metadata';
-import { printDeprecationWarnings } from './utils/printBuildInfo';
-import { makeProjectTarballAsync, reviewAndCommitChangesAsync } from './utils/repository';
 import { BuildEvent } from '../analytics/AnalyticsManager';
 import { withAnalyticsAsync } from '../analytics/common';
 import { getExpoWebsiteBaseUrl } from '../api';
@@ -54,6 +34,30 @@ import { formatBytes } from '../utils/files';
 import { printJsonOnlyOutput } from '../utils/json';
 import { createProgressTracker } from '../utils/progress';
 import { sleepAsync } from '../utils/promise';
+import { BuildContext } from './context';
+import {
+  EasBuildDownForMaintenanceError,
+  EasBuildFreeTierDisabledAndroidError,
+  EasBuildFreeTierDisabledError,
+  EasBuildFreeTierDisabledIOSError,
+  EasBuildFreeTierIosLimitExceededError,
+  EasBuildFreeTierLimitExceededError,
+  EasBuildLegacyResourceClassNotAvailableError,
+  EasBuildProjectArchiveUploadError,
+  EasBuildResourceClassNotAvailableInFreeTierError,
+  EasBuildTooManyPendingBuildsError,
+  RequestValidationError,
+  TurtleDeprecatedJobFormatError,
+} from './errors';
+import { transformMetadata } from './graphql';
+import { LocalBuildMode, runLocalBuildAsync } from './local';
+import { collectMetadataAsync } from './metadata';
+import { printDeprecationWarnings } from './utils/printBuildInfo';
+import {
+  makeProjectMetadataFileAsync,
+  makeProjectTarballAsync,
+  reviewAndCommitChangesAsync,
+} from './utils/repository';
 
 export interface CredentialsResult<Credentials> {
   source: CredentialsSource.LOCAL | CredentialsSource.REMOTE;
@@ -97,7 +101,7 @@ function resolveBuildParamsInput<T extends Platform>(
 export async function prepareBuildRequestForPlatformAsync<
   TPlatform extends Platform,
   Credentials,
-  TJob extends Job,
+  TJob extends Job
 >(builder: Builder<TPlatform, Credentials, TJob>): Promise<BuildRequestSender> {
   const { ctx } = builder;
   const credentialsResult = await withAnalyticsAsync(
@@ -133,9 +137,10 @@ export async function prepareBuildRequestForPlatformAsync<
 
   let projectArchive: ArchiveSource | undefined;
   if (ctx.localBuildOptions.localBuildMode === LocalBuildMode.LOCAL_BUILD_PLUGIN) {
+    const projectPath = (await makeProjectTarballAsync(ctx.vcsClient)).path;
     projectArchive = {
       type: ArchiveSourceType.PATH,
-      path: (await makeProjectTarballAsync(ctx.vcsClient)).path,
+      path: projectPath,
     };
   } else if (ctx.localBuildOptions.localBuildMode === LocalBuildMode.INTERNAL) {
     projectArchive = {
@@ -145,7 +150,7 @@ export async function prepareBuildRequestForPlatformAsync<
   } else if (!ctx.localBuildOptions.localBuildMode) {
     projectArchive = {
       type: ArchiveSourceType.GCS,
-      bucketKey: await uploadProjectAsync(ctx),
+      ...(await uploadProjectAsync({ ctx, generateMetadataFile: true })),
     };
   }
   assert(projectArchive);
@@ -229,9 +234,16 @@ export function handleBuildRequestError(error: any, platform: Platform): never {
   throw error;
 }
 
-async function uploadProjectAsync<TPlatform extends Platform>(
-  ctx: BuildContext<TPlatform>
-): Promise<string> {
+async function uploadProjectAsync<TPlatform extends Platform>({
+  ctx,
+  generateMetadataFile = true,
+}: {
+  ctx: BuildContext<TPlatform>;
+  generateMetadataFile: boolean;
+}): Promise<{
+  bucketKey: string;
+  metadataLocation?: string;
+}> {
   let projectTarballPath;
   try {
     return await withAnalyticsAsync(
@@ -260,7 +272,6 @@ async function uploadProjectAsync<TPlatform extends Platform>(
         }
 
         projectTarballPath = projectTarball.path;
-
         const bucketKey = await uploadFileAtPathToGCSAsync(
           ctx.graphqlClient,
           UploadSessionType.EasBuildGcsProjectSources,
@@ -274,7 +285,16 @@ async function uploadProjectAsync<TPlatform extends Platform>(
             completedMessage: (duration: string) => `Uploaded to EAS ${chalk.dim(duration)}`,
           })
         );
-        return bucketKey;
+        if (generateMetadataFile) {
+          const { metadataLocation } = await uploadMetadataFileAsync<TPlatform>(
+            projectTarball,
+            ctx
+          );
+          if (metadataLocation) {
+            return { bucketKey, metadataLocation };
+          }
+        }
+        return { bucketKey };
       },
       {
         attemptEvent: BuildEvent.PROJECT_UPLOAD_ATTEMPT,
@@ -285,9 +305,11 @@ async function uploadProjectAsync<TPlatform extends Platform>(
     );
   } catch (err: any) {
     let errMessage = 'Failed to upload the project tarball to EAS Build';
+
     if (err.message) {
       errMessage += `\n\nReason: ${err.message}`;
     }
+
     throw new EasBuildProjectArchiveUploadError(errMessage);
   } finally {
     if (projectTarballPath) {
@@ -296,6 +318,44 @@ async function uploadProjectAsync<TPlatform extends Platform>(
   }
 }
 
+async function uploadMetadataFileAsync<TPlatform extends Platform>(
+  projectTarball: { path: string; size: number },
+  ctx: BuildContext<TPlatform>
+): Promise<{ metadataLocation: string | null }> {
+  let projectMetadataFile: any;
+  try {
+    projectMetadataFile = await makeProjectMetadataFileAsync(projectTarball.path);
+
+    const metadataLocation = await uploadFileAtPathToGCSAsync(
+      ctx.graphqlClient,
+      UploadSessionType.EasBuildGcsProjectSources,
+      projectMetadataFile.path,
+      createProgressTracker({
+        total: projectMetadataFile.size,
+        message: ratio =>
+          `Uploading metadata to EAS Build (${formatBytes(
+            projectMetadataFile.size * ratio
+          )} / ${formatBytes(projectMetadataFile.size)})`,
+        completedMessage: (duration: string) => `Uploaded to EAS ${chalk.dim(duration)}`,
+      })
+    );
+    return { metadataLocation };
+  } catch (err: any) {
+    let errMessage = 'Failed to upload metadata to EAS Build';
+
+    if (err.message) {
+      errMessage += `\n\nReason: ${err.message}`;
+    }
+
+    Log.warn(errMessage);
+    return { metadataLocation: null };
+  } finally {
+    if (projectMetadataFile) {
+      await fs.remove(projectMetadataFile);
+    }
+  }
+}
+
 async function sendBuildRequestAsync<TPlatform extends Platform, Credentials, TJob extends Job>(
   builder: Builder<TPlatform, Credentials, TJob>,
   job: TJob,

packages/eas-cli/src/build/graphql.ts

@@ -25,6 +25,7 @@ export function transformProjectArchive(archiveSource: ArchiveSource): ProjectAr
     return {
       type: ProjectArchiveSourceType.Gcs,
       bucketKey: archiveSource.bucketKey,
+      metadataLocation: archiveSource.metadataLocation,
     };
   } else if (archiveSource.type === ArchiveSourceType.URL) {
     return {

packages/eas-cli/src/build/utils/repository.ts

@@ -83,6 +83,50 @@ export async function commitPromptAsync(
   });
 }
 
+export async function makeProjectMetadataFileAsync(
+  archivePath: string
+): Promise<{ path: string; size: number }> {
+  const spinner = ora('Creating project metadata file');
+  const timerLabel = 'makeProjectMetadataFileAsync';
+  const timer = setTimeout(
+    () => {
+      spinner.start();
+    },
+    Log.isDebug ? 1 : 1000
+  );
+  startTimer(timerLabel);
+
+  const metadataLocation = path.join(getTmpDirectory(), `${uuidv4()}-eas-build-metadata.json`);
+  const archiveContent: string[] = [];
+
+  try {
+    await tar.list({
+      file: archivePath,
+      onentry: (entry: tar.ReadEntry) => {
+        if (entry.type === 'File' && !entry.path.includes('.git')) {
+          archiveContent.push(entry.path);
+        }
+      },
+    });
+
+    await fs.writeJSON(metadataLocation, {
+      archiveContent,
+    });
+  } catch (e) {
+    clearTimeout(timer);
+    if (spinner.isSpinning) {
+      spinner.fail();
+    }
+    throw e;
+  }
+
+  const duration = endTimer(timerLabel);
+  const prettyTime = formatMilliseconds(duration);
+  spinner.succeed(`Created project metadata file ${chalk.dim(prettyTime)}`);
+
+  return { path: metadataLocation, size: await fs.stat(metadataLocation).then(stat => stat.size) };
+}
+
 export async function makeProjectTarballAsync(
   vcsClient: Client
 ): Promise<{ path: string; size: number }> {