refactor(artifact_proxy): use per-revision artifacts manifest (#340)

packages/artifact_proxy/bin/server.dart

@@ -3,14 +3,14 @@
 import 'dart:io';
 
 import 'package:artifact_proxy/artifact_proxy.dart';
-import 'package:artifact_proxy/config.dart';
 import 'package:shelf/shelf.dart';
 import 'package:shelf/shelf_io.dart' as shelf_io;
 import 'package:shelf_hotreload/shelf_hotreload.dart';
 
 Future<void> main() async {
   final isDev = Platform.environment['DEV'] == 'true';
-  final handler = artifactProxyHandler(config: config);
+  final client = ArtifactManifestClient();
+  final handler = artifactProxyHandler(client: client);
   final ip = InternetAddress.anyIPv6;
   final port = int.parse(Platform.environment['PORT'] ?? '8080');
 

packages/artifact_proxy/build.yaml

@@ -0,0 +1,16 @@
+targets:
+  $default:
+    builders:
+      source_gen|combining_builder:
+        options:
+          ignore_for_file:
+            - implicit_dynamic_parameter
+            - require_trailing_commas
+            - cast_nullable_to_non_nullable
+            - lines_longer_than_80_chars
+            - strict_raw_type
+      json_serializable:
+        options:
+          field_rename: snake
+          checked: true
+          explicit_to_json: true

packages/artifact_proxy/lib/artifact_proxy.dart

@@ -1,2 +1,3 @@
+export 'src/artifact_manifest_client.dart';
 export 'src/artifact_proxy.dart';
 export 'src/models/models.dart';

packages/artifact_proxy/lib/config.dart

@@ -1,78 +1,94 @@
-import 'package:artifact_proxy/artifact_proxy.dart';
-
-/// The proxy configuration used by the server.
-const config = ProxyConfig(
-  engineMappings: {
-    ...flutter_3_7_12,
-    ...flutter_3_7_10,
-    ...flutter_3_7_8,
-  },
-);
-
-// On the assumption artifact layouts don't change very often for Flutter.
-// ignore: camel_case_types
-class _EngineMapping3_7 extends EngineMapping {
-  const _EngineMapping3_7({
-    required super.flutterEngineRevision,
-  }) : super(
-          shorebirdStorageBucket: 'download.shorebird.dev',
-          shorebirdArtifactOverrides: const {
-            // artifacts.zip
-            r'flutter_infra_release/flutter/$engine/android-arm-64-release/artifacts.zip',
-            r'flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip',
-            r'flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip',
-
-            // embedding release
-            r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom',
-            r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar',
-
-            // arm64_v8a release
-            r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom',
-            r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar',
-
-            // armeabi_v7a release
-            r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom',
-            r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar',
-
-            // x86_64 release
-            r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom',
-            r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar',
-          },
-        );
-}
-
-const flutter_3_7_12 = {
-  // Shorebird v0.0.8
-  'd470ae25d21f583abe128f7b838476afd5e45bde': _EngineMapping3_7(
-    flutterEngineRevision: '1a65d409c7a1438a34d21b60bf30a6fd5db59314',
-  ),
-};
-
-/// Flutter 3.7.10
-const flutter_3_7_10 = {
-  // Shorebird v0.0.6
-  '8b89f8bd9fc6982aa9c4557fd0e5e89db1ff9986': _EngineMapping3_7(
-    flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
-  ),
-  // Shorebird v0.0.5
-  'e6a2a5a43973430d9f038cd81cb1779b6b404909': _EngineMapping3_7(
-    flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
-  ),
-  // Attempt to fix https://github.com/shorebirdtech/shorebird/issues/235
-  'adb70a20d4718b5ce60cdd99ad81d8de54afcb35': _EngineMapping3_7(
-    flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
-  ),
-  '978a56f2d97f9ce24a2b6bc22c9bbceaaba0343c': _EngineMapping3_7(
-    flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
-  ),
-  '7aa5c44764e10722d188ece75819f7d10f5269a3': _EngineMapping3_7(
-    flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
-  ),
+/// Patterns for all artifacts paths that Flutter
+/// requests which contain an engine revision.
+final engineArtifactPatterns = {
+  r'flutter_infra_release\/flutter\/(.*)\/windows-x64\/windows-x64-flutter\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/windows-x64\/font-subset.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/windows-x64\/flutter-cpp-client-wrapper\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/windows-x64\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/windows-x64-release\/windows-x64-flutter\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/windows-x64-profile\/windows-x64-flutter\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/sky_engine\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/linux-arm64\/linux-arm64-flutter-gtk\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/linux-arm64\/font-subset\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/linux-arm64\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/linux-arm64-release\/linux-arm64-flutter-gtk\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/linux-arm64-profile\/linux-arm64-flutter-gtk\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/ios\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/ios-release\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/ios-profile\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/flutter-web-sdk-darwin-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/flutter_patched_sdk\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/flutter_patched_sdk_product\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-x64\/gen_snapshot\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-x64\/FlutterMacOS.framework\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-release\/gen_snapshot\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-release\/FlutterMacOS.framework\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-release\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-profile\/gen_snapshot\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-profile\/FlutterMacOS.framework\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-profile\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-arm64\/font-subset\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/darwin-arm64\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/dart-sdk-windows-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/dart-sdk-linux-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/dart-sdk-darwin-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/dart-sdk-darwin-arm64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x86\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x86-jit-release\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64-release\/windows-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64-release\/linux-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64-release\/darwin-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64-release\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64-profile\/windows-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64-profile\/linux-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64-profile\/darwin-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-x64-profile\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64-release\/windows-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64-release\/linux-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64-release\/darwin-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64-release\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64-profile\/windows-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64-profile\/linux-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64-profile\/darwin-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm64-profile\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-release\/windows-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-release\/linux-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-release\/darwin-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-release\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-profile\/windows-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-profile\/linux-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-profile\/darwin-x64\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-profile\/artifacts\.zip',
+  r'flutter_infra_release\/flutter\/(.*)\/android-arm-64-release\/artifacts\.zip',
+  r'download.flutter.io\/io\/flutter\/x86_64_release\/1.0.0-(.*)\/x86_64_release-1.0.0-(.*)\.pom\.sha1',
+  r'download.flutter.io\/io\/flutter\/x86_64_release\/1.0.0-(.*)\/x86_64_release-1.0.0-(.*)\.pom',
+  r'download.flutter.io\/io\/flutter\/x86_64_release\/1.0.0-(.*)\/x86_64_release-1.0.0-(.*)\.jar\.sha1',
+  r'download.flutter.io\/io\/flutter\/x86_64_release\/1.0.0-(.*)\/x86_64_release-1.0.0-(.*)\.jar',
+  r'download.flutter.io\/io\/flutter\/flutter_embedding_release\/1.0.0-(.*)\/flutter_embedding_release-1.0.0-(.*)\.pom\.sha1',
+  r'download.flutter.io\/io\/flutter\/flutter_embedding_release\/1.0.0-(.*)\/flutter_embedding_release-1.0.0-(.*)\.pom',
+  r'download.flutter.io\/io\/flutter\/flutter_embedding_release\/1.0.0-(.*)\/flutter_embedding_release-1.0.0-(.*)\.jar\.sha1',
+  r'download.flutter.io\/io\/flutter\/flutter_embedding_release\/1.0.0-(.*)\/flutter_embedding_release-1.0.0-(.*)\.jar',
+  r'download.flutter.io\/io\/flutter\/armeabi_v7a_release\/1.0.0-(.*)\/armeabi_v7a_release-1.0.0-(.*)\.pom\.sha1',
+  r'download.flutter.io\/io\/flutter\/armeabi_v7a_release\/1.0.0-(.*)\/armeabi_v7a_release-1.0.0-(.*)\.pom',
+  r'download.flutter.io\/io\/flutter\/armeabi_v7a_release\/1.0.0-(.*)\/armeabi_v7a_release-1.0.0-(.*)\.jar\.sha1',
+  r'download.flutter.io\/io\/flutter\/armeabi_v7a_release\/1.0.0-(.*)\/armeabi_v7a_release-1.0.0-(.*)\.jar',
+  r'download.flutter.io\/io\/flutter\/arm64_v8a_release\/1.0.0-(.*)\/arm64_v8a_release-1.0.0-(.*)\.pom\.sha1',
+  r'download.flutter.io\/io\/flutter\/arm64_v8a_release\/1.0.0-(.*)\/arm64_v8a_release-1.0.0-(.*)\.pom',
+  r'download.flutter.io\/io\/flutter\/arm64_v8a_release\/1.0.0-(.*)\/arm64_v8a_release-1.0.0-(.*)\.jar\.sha1',
+  r'download.flutter.io\/io\/flutter\/arm64_v8a_release\/1.0.0-(.*)\/arm64_v8a_release-1.0.0-(.*)\.jar',
 };
 
-/// Flutter 3.7.8
-const flutter_3_7_8 = {
-  '79f4c5321a581f580a9bda01ec372cbf4a53aa53': _EngineMapping3_7(
-    flutterEngineRevision: '9aa7816315095c86410527932918c718cb35e7d6',
-  ),
+/// Patterns for Flutter artifacts which don't depend on an engine revision.
+final flutterArtifactPatterns = {
+  r'flutter_infra_release\/flutter\/fonts\/(.*)\/fonts\.zip',
+  r'flutter_infra_release\/gradle-wrapper\/(.*)\/gradle-wrapper\.tgz',
+  r'flutter_infra_release\/ios-usb-dependencies\/libimobiledevice\/(.*)\/libimobiledevice\.zip',
+  r'flutter_infra_release\/ios-usb-dependencies\/usbmuxd\/(.*)\/usbmuxd\.zip',
+  r'flutter_infra_release\/ios-usb-dependencies\/libplist\/(.*)\/libplist\.zip',
+  r'flutter_infra_release\/ios-usb-dependencies\/openssl\/(.*)\/openssl\.zip',
+  r'flutter_infra_release\/ios-usb-dependencies\/ios-deploy\/(.*)\/ios-deploy\.zip',
+  r'flutter_infra_release\/cipd\/flutter\/web\/canvaskit_bundle\/\+\/(.*)',
 };

packages/artifact_proxy/lib/src/artifact_manifest_client.dart

@@ -0,0 +1,49 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:artifact_proxy/artifact_proxy.dart';
+import 'package:checked_yaml/checked_yaml.dart';
+import 'package:http/http.dart' as http;
+import 'package:quiver/collection.dart';
+
+/// {@template artifact_manifest_client}
+/// A client that fetches [ArtifactsManifest]s from the shorebird storage bucket
+/// and caches them in memory.
+/// {@endtemplate}
+class ArtifactManifestClient {
+  /// {@macro artifact_manifest_client}
+  ArtifactManifestClient({http.Client? httpClient})
+      : _httpClient = httpClient ?? http.Client();
+
+  final http.Client _httpClient;
+
+  final _cache = LruMap<String, ArtifactsManifest>(maximumSize: 1000);
+
+  /// Fetches the [ArtifactsManifest] for the provided [revision] from the
+  /// shorebird storage bucket.
+  Future<ArtifactsManifest> getManifest(String revision) async {
+    if (_cache.containsKey(revision)) return _cache[revision]!;
+    final manifest = await _fetchManifest(revision);
+    _cache[revision] = manifest;
+    return manifest;
+  }
+
+  Future<ArtifactsManifest> _fetchManifest(String revision) async {
+    final url = Uri.parse(
+      'https://storage.googleapis.com/download.shorebird.dev/shorebird/$revision/artifacts_manifest.yaml',
+    );
+    final response = await _httpClient.get(url);
+    if (response.statusCode != HttpStatus.ok) {
+      throw Exception(
+        '''
+Failed to fetch artifacts manifest for revision $revision.
+${response.statusCode} ${response.reasonPhrase}''',
+      );
+    }
+
+    return checkedYamlDecode(
+      response.body,
+      (m) => ArtifactsManifest.fromJson(m!),
+    );
+  }
+}

packages/artifact_proxy/lib/src/artifact_proxy.dart

@@ -1,51 +1,70 @@
 // ignore_for_file: avoid_print
 
 import 'package:artifact_proxy/artifact_proxy.dart';
-import 'package:collection/collection.dart';
+import 'package:artifact_proxy/config.dart';
 import 'package:shelf/shelf.dart';
 
 /// A [Handler] that proxies artifact requests to the correct location.
-/// This is determined based on the [config].
-Handler artifactProxyHandler({required ProxyConfig config}) {
-  final engineMappings = config.engineMappings;
-  final shorebirdEngineRevisions = engineMappings.keys.cast<String>();
-
-  return (Request request) {
+Handler artifactProxyHandler({required ArtifactManifestClient client}) {
+  return (Request request) async {
     final path = request.url.path;
-    final shorebirdEngineRevision = shorebirdEngineRevisions.firstWhereOrNull(
-      path.contains,
-    );
-
-    final normalizedPath = shorebirdEngineRevision != null
-        ? path.replaceAll(shorebirdEngineRevision, r'$engine')
-        : path;
 
-    if (shorebirdEngineRevision == null) {
-      final location = getFlutterArtifactLocation(artifactPath: normalizedPath);
-      print('No engine revision detected, forwarding to: $location');
-      return Response.found(location);
+    RegExpMatch? engineArtifactMatch;
+    for (final pattern in engineArtifactPatterns) {
+      final match = RegExp(pattern).firstMatch(path);
+      if (match != null && match.group(1) != null) {
+        engineArtifactMatch = match;
+        break;
+      }
     }
 
-    final engineMapping = engineMappings[shorebirdEngineRevision]!;
-    final shorebirdOverrides = engineMapping.shorebirdArtifactOverrides;
-    final flutterEngineRevision = engineMapping.flutterEngineRevision;
-    final shorebirdStorageBucket = engineMapping.shorebirdStorageBucket;
-    final shouldOverride = shorebirdOverrides.contains(normalizedPath);
+    if (engineArtifactMatch != null) {
+      final shorebirdEngineRevision = engineArtifactMatch.group(1)!;
+      final ArtifactsManifest manifest;
+      try {
+        manifest = await client.getManifest(shorebirdEngineRevision);
+      } catch (error) {
+        return Response.notFound(
+          'Failed to fetch manifest for $shorebirdEngineRevision\n$error',
+        );
+      }
+
+      final normalizedPath = path.replaceAll(
+        shorebirdEngineRevision,
+        r'$engine',
+      );
+
+      final shouldOverride = manifest.artifactOverrides.contains(
+        normalizedPath,
+      );
+
+      if (shouldOverride) {
+        final location = getShorebirdArtifactLocation(
+          artifactPath: normalizedPath,
+          engine: shorebirdEngineRevision,
+          bucket: manifest.storageBucket,
+        );
+        print('Shorebird engine artifact detected, forwarding to: $location');
+        return Response.found(location);
+      }
 
-    if (shouldOverride) {
-      final location = getShorebirdArtifactLocation(
+      final location = getFlutterArtifactLocation(
         artifactPath: normalizedPath,
-        engine: shorebirdEngineRevision,
-        bucket: shorebirdStorageBucket,
+        engine: manifest.flutterEngineRevision,
       );
-      print('Shorebird artifact detected, forwarding to: $location');
+      print('Flutter artifact detected, forwarding to: $location');
       return Response.found(location);
     }
 
-    final location = getFlutterArtifactLocation(
-      artifactPath: normalizedPath,
-      engine: flutterEngineRevision,
+    final isRecognizedFlutterArtifact = flutterArtifactPatterns.any(
+      (pattern) => RegExp(pattern).hasMatch(path),
     );
+
+    if (!isRecognizedFlutterArtifact) {
+      return Response.notFound('Unrecognized artifact path: $path');
+    }
+
+    final location = getFlutterArtifactLocation(artifactPath: path);
     print('Flutter artifact detected, forwarding to: $location');
     return Response.found(location);
   };