Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(artifact_proxy): use per-revision artifacts manifest #340

Merged
merged 5 commits into from
Apr 21, 2023
Merged

refactor(artifact_proxy): use per-revision artifacts manifest #340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
92ce32f
refactor(artifact_proxy): use per-revision artifacts manifest
felangel Apr 20, 2023
addd0e2
update artifacts
felangel Apr 21, 2023
4108051
Merge branch 'main' into refactor/artifact-proxy-to-use-per-revision-…
felangel Apr 21, 2023
ae25218
tests
felangel Apr 21, 2023
40f192f
feedback
felangel Apr 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/artifact_proxy/bin/server.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
import 'dart:io';

import 'package:artifact_proxy/artifact_proxy.dart';
import 'package:artifact_proxy/config.dart';
import 'package:shelf/shelf.dart';
import 'package:shelf/shelf_io.dart' as shelf_io;
import 'package:shelf_hotreload/shelf_hotreload.dart';

Future<void> main() async {
final isDev = Platform.environment['DEV'] == 'true';
final handler = artifactProxyHandler(config: config);
final client = ArtifactManifestClient();
final handler = artifactProxyHandler(client: client);
final ip = InternetAddress.anyIPv6;
final port = int.parse(Platform.environment['PORT'] ?? '8080');

Expand Down
16 changes: 16 additions & 0 deletions packages/artifact_proxy/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
targets:
$default:
builders:
source_gen|combining_builder:
options:
ignore_for_file:
- implicit_dynamic_parameter
- require_trailing_commas
- cast_nullable_to_non_nullable
- lines_longer_than_80_chars
- strict_raw_type
json_serializable:
options:
field_rename: snake
checked: true
explicit_to_json: true
1 change: 1 addition & 0 deletions packages/artifact_proxy/lib/artifact_proxy.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
export 'src/artifact_manifest_client.dart';
export 'src/artifact_proxy.dart';
export 'src/models/models.dart';
166 changes: 91 additions & 75 deletions packages/artifact_proxy/lib/config.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,78 +1,94 @@
import 'package:artifact_proxy/artifact_proxy.dart';

/// The proxy configuration used by the server.
const config = ProxyConfig(
engineMappings: {
...flutter_3_7_12,
...flutter_3_7_10,
...flutter_3_7_8,
},
);

// On the assumption artifact layouts don't change very often for Flutter.
// ignore: camel_case_types
class _EngineMapping3_7 extends EngineMapping {
const _EngineMapping3_7({
required super.flutterEngineRevision,
}) : super(
shorebirdStorageBucket: 'download.shorebird.dev',
shorebirdArtifactOverrides: const {
// artifacts.zip
r'flutter_infra_release/flutter/$engine/android-arm-64-release/artifacts.zip',
r'flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip',
r'flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip',

// embedding release
r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom',
r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar',

// arm64_v8a release
r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom',
r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar',

// armeabi_v7a release
r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom',
r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar',

// x86_64 release
r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom',
r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar',
},
);
}

const flutter_3_7_12 = {
// Shorebird v0.0.8
'd470ae25d21f583abe128f7b838476afd5e45bde': _EngineMapping3_7(
flutterEngineRevision: '1a65d409c7a1438a34d21b60bf30a6fd5db59314',
),
};

/// Flutter 3.7.10
const flutter_3_7_10 = {
// Shorebird v0.0.6
'8b89f8bd9fc6982aa9c4557fd0e5e89db1ff9986': _EngineMapping3_7(
flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
),
// Shorebird v0.0.5
'e6a2a5a43973430d9f038cd81cb1779b6b404909': _EngineMapping3_7(
flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
),
// Attempt to fix https://github.com/shorebirdtech/shorebird/issues/235
'adb70a20d4718b5ce60cdd99ad81d8de54afcb35': _EngineMapping3_7(
flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
),
'978a56f2d97f9ce24a2b6bc22c9bbceaaba0343c': _EngineMapping3_7(
flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
),
'7aa5c44764e10722d188ece75819f7d10f5269a3': _EngineMapping3_7(
flutterEngineRevision: 'ec975089acb540fc60752606a3d3ba809dd1528b',
),
/// Patterns for all artifacts paths that Flutter
/// requests which contain an engine revision.
final engineArtifactPatterns = {
r'flutter_infra_release\/flutter\/(.*)\/windows-x64\/windows-x64-flutter\.zip',
r'flutter_infra_release\/flutter\/(.*)\/windows-x64\/font-subset.zip',
r'flutter_infra_release\/flutter\/(.*)\/windows-x64\/flutter-cpp-client-wrapper\.zip',
r'flutter_infra_release\/flutter\/(.*)\/windows-x64\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/windows-x64-release\/windows-x64-flutter\.zip',
r'flutter_infra_release\/flutter\/(.*)\/windows-x64-profile\/windows-x64-flutter\.zip',
r'flutter_infra_release\/flutter\/(.*)\/sky_engine\.zip',
r'flutter_infra_release\/flutter\/(.*)\/linux-arm64\/linux-arm64-flutter-gtk\.zip',
r'flutter_infra_release\/flutter\/(.*)\/linux-arm64\/font-subset\.zip',
r'flutter_infra_release\/flutter\/(.*)\/linux-arm64\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/linux-arm64-release\/linux-arm64-flutter-gtk\.zip',
r'flutter_infra_release\/flutter\/(.*)\/linux-arm64-profile\/linux-arm64-flutter-gtk\.zip',
r'flutter_infra_release\/flutter\/(.*)\/ios\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/ios-release\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/ios-profile\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/flutter-web-sdk-darwin-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/flutter_patched_sdk\.zip',
r'flutter_infra_release\/flutter\/(.*)\/flutter_patched_sdk_product\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-x64\/gen_snapshot\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-x64\/FlutterMacOS.framework\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-release\/gen_snapshot\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-release\/FlutterMacOS.framework\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-release\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-profile\/gen_snapshot\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-profile\/FlutterMacOS.framework\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-x64-profile\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-arm64\/font-subset\.zip',
r'flutter_infra_release\/flutter\/(.*)\/darwin-arm64\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/dart-sdk-windows-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/dart-sdk-linux-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/dart-sdk-darwin-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/dart-sdk-darwin-arm64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x86\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x86-jit-release\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64-release\/windows-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64-release\/linux-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64-release\/darwin-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64-release\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64-profile\/windows-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64-profile\/linux-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64-profile\/darwin-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-x64-profile\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64-release\/windows-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64-release\/linux-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64-release\/darwin-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64-release\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64-profile\/windows-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64-profile\/linux-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64-profile\/darwin-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm64-profile\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-release\/windows-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-release\/linux-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-release\/darwin-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-release\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-profile\/windows-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-profile\/linux-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-profile\/darwin-x64\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-profile\/artifacts\.zip',
r'flutter_infra_release\/flutter\/(.*)\/android-arm-64-release\/artifacts\.zip',
r'download.flutter.io\/io\/flutter\/x86_64_release\/1.0.0-(.*)\/x86_64_release-1.0.0-(.*)\.pom\.sha1',
r'download.flutter.io\/io\/flutter\/x86_64_release\/1.0.0-(.*)\/x86_64_release-1.0.0-(.*)\.pom',
r'download.flutter.io\/io\/flutter\/x86_64_release\/1.0.0-(.*)\/x86_64_release-1.0.0-(.*)\.jar\.sha1',
r'download.flutter.io\/io\/flutter\/x86_64_release\/1.0.0-(.*)\/x86_64_release-1.0.0-(.*)\.jar',
r'download.flutter.io\/io\/flutter\/flutter_embedding_release\/1.0.0-(.*)\/flutter_embedding_release-1.0.0-(.*)\.pom\.sha1',
r'download.flutter.io\/io\/flutter\/flutter_embedding_release\/1.0.0-(.*)\/flutter_embedding_release-1.0.0-(.*)\.pom',
r'download.flutter.io\/io\/flutter\/flutter_embedding_release\/1.0.0-(.*)\/flutter_embedding_release-1.0.0-(.*)\.jar\.sha1',
r'download.flutter.io\/io\/flutter\/flutter_embedding_release\/1.0.0-(.*)\/flutter_embedding_release-1.0.0-(.*)\.jar',
r'download.flutter.io\/io\/flutter\/armeabi_v7a_release\/1.0.0-(.*)\/armeabi_v7a_release-1.0.0-(.*)\.pom\.sha1',
r'download.flutter.io\/io\/flutter\/armeabi_v7a_release\/1.0.0-(.*)\/armeabi_v7a_release-1.0.0-(.*)\.pom',
r'download.flutter.io\/io\/flutter\/armeabi_v7a_release\/1.0.0-(.*)\/armeabi_v7a_release-1.0.0-(.*)\.jar\.sha1',
r'download.flutter.io\/io\/flutter\/armeabi_v7a_release\/1.0.0-(.*)\/armeabi_v7a_release-1.0.0-(.*)\.jar',
r'download.flutter.io\/io\/flutter\/arm64_v8a_release\/1.0.0-(.*)\/arm64_v8a_release-1.0.0-(.*)\.pom\.sha1',
r'download.flutter.io\/io\/flutter\/arm64_v8a_release\/1.0.0-(.*)\/arm64_v8a_release-1.0.0-(.*)\.pom',
r'download.flutter.io\/io\/flutter\/arm64_v8a_release\/1.0.0-(.*)\/arm64_v8a_release-1.0.0-(.*)\.jar\.sha1',
r'download.flutter.io\/io\/flutter\/arm64_v8a_release\/1.0.0-(.*)\/arm64_v8a_release-1.0.0-(.*)\.jar',
};

/// Flutter 3.7.8
const flutter_3_7_8 = {
'79f4c5321a581f580a9bda01ec372cbf4a53aa53': _EngineMapping3_7(
flutterEngineRevision: '9aa7816315095c86410527932918c718cb35e7d6',
),
/// Patterns for Flutter artifacts which don't depend on an engine revision.
final flutterArtifactPatterns = {
r'flutter_infra_release\/flutter\/fonts\/(.*)\/fonts\.zip',
r'flutter_infra_release\/gradle-wrapper\/(.*)\/gradle-wrapper\.tgz',
r'flutter_infra_release\/ios-usb-dependencies\/libimobiledevice\/(.*)\/libimobiledevice\.zip',
r'flutter_infra_release\/ios-usb-dependencies\/usbmuxd\/(.*)\/usbmuxd\.zip',
r'flutter_infra_release\/ios-usb-dependencies\/libplist\/(.*)\/libplist\.zip',
r'flutter_infra_release\/ios-usb-dependencies\/openssl\/(.*)\/openssl\.zip',
r'flutter_infra_release\/ios-usb-dependencies\/ios-deploy\/(.*)\/ios-deploy\.zip',
r'flutter_infra_release\/cipd\/flutter\/web\/canvaskit_bundle\/\+\/(.*)',
};
49 changes: 49 additions & 0 deletions packages/artifact_proxy/lib/src/artifact_manifest_client.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import 'dart:convert';
import 'dart:io';

import 'package:artifact_proxy/artifact_proxy.dart';
import 'package:checked_yaml/checked_yaml.dart';
import 'package:http/http.dart' as http;
import 'package:quiver/collection.dart';

/// {@template artifact_manifest_client}
/// A client that fetches [ArtifactsManifest]s from the shorebird storage bucket
/// and caches them in memory.
/// {@endtemplate}
class ArtifactManifestClient {
/// {@macro artifact_manifest_client}
ArtifactManifestClient({http.Client? httpClient})
: _httpClient = httpClient ?? http.Client();

final http.Client _httpClient;

final _cache = LruMap<String, ArtifactsManifest>(maximumSize: 1000);

/// Fetches the [ArtifactsManifest] for the provided [revision] from the
/// shorebird storage bucket.
Future<ArtifactsManifest> getManifest(String revision) async {
if (_cache.containsKey(revision)) return _cache[revision]!;
final manifest = await _fetchManifest(revision);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you will want to cache negative results with a TTL? I guess as an MVP doesn't matter.

_cache[revision] = manifest;
return manifest;
}

Future<ArtifactsManifest> _fetchManifest(String revision) async {
final url = Uri.parse(
'https://storage.googleapis.com/download.shorebird.dev/shorebird/$revision/artifacts_manifest.yaml',
);
final response = await _httpClient.get(url);
if (response.statusCode != HttpStatus.ok) {
throw Exception(
'''
Failed to fetch artifacts manifest for revision $revision.
${response.statusCode} ${response.reasonPhrase}''',
);
}

return checkedYamlDecode(
response.body,
(m) => ArtifactsManifest.fromJson(m!),
);
}
}
81 changes: 50 additions & 31 deletions packages/artifact_proxy/lib/src/artifact_proxy.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,51 +1,70 @@
// ignore_for_file: avoid_print

import 'package:artifact_proxy/artifact_proxy.dart';
import 'package:collection/collection.dart';
import 'package:artifact_proxy/config.dart';
import 'package:shelf/shelf.dart';

/// A [Handler] that proxies artifact requests to the correct location.
/// This is determined based on the [config].
Handler artifactProxyHandler({required ProxyConfig config}) {
final engineMappings = config.engineMappings;
final shorebirdEngineRevisions = engineMappings.keys.cast<String>();

return (Request request) {
Handler artifactProxyHandler({required ArtifactManifestClient client}) {
return (Request request) async {
final path = request.url.path;
final shorebirdEngineRevision = shorebirdEngineRevisions.firstWhereOrNull(
path.contains,
);

final normalizedPath = shorebirdEngineRevision != null
? path.replaceAll(shorebirdEngineRevision, r'$engine')
: path;

if (shorebirdEngineRevision == null) {
final location = getFlutterArtifactLocation(artifactPath: normalizedPath);
print('No engine revision detected, forwarding to: $location');
return Response.found(location);
RegExpMatch? engineArtifactMatch;
for (final pattern in engineArtifactPatterns) {
final match = RegExp(pattern).firstMatch(path);
if (match != null && match.group(1) != null) {
engineArtifactMatch = match;
break;
}
}

final engineMapping = engineMappings[shorebirdEngineRevision]!;
final shorebirdOverrides = engineMapping.shorebirdArtifactOverrides;
final flutterEngineRevision = engineMapping.flutterEngineRevision;
final shorebirdStorageBucket = engineMapping.shorebirdStorageBucket;
final shouldOverride = shorebirdOverrides.contains(normalizedPath);
if (engineArtifactMatch != null) {
final shorebirdEngineRevision = engineArtifactMatch.group(1)!;
final ArtifactsManifest manifest;
try {
manifest = await client.getManifest(shorebirdEngineRevision);
} catch (error) {
return Response.notFound(
'Failed to fetch manifest for $shorebirdEngineRevision\n$error',
);
}

final normalizedPath = path.replaceAll(
shorebirdEngineRevision,
r'$engine',
);

final shouldOverride = manifest.artifactOverrides.contains(
normalizedPath,
);

if (shouldOverride) {
final location = getShorebirdArtifactLocation(
artifactPath: normalizedPath,
engine: shorebirdEngineRevision,
bucket: manifest.storageBucket,
);
print('Shorebird engine artifact detected, forwarding to: $location');
return Response.found(location);
}

if (shouldOverride) {
final location = getShorebirdArtifactLocation(
final location = getFlutterArtifactLocation(
artifactPath: normalizedPath,
engine: shorebirdEngineRevision,
bucket: shorebirdStorageBucket,
engine: manifest.flutterEngineRevision,
);
print('Shorebird artifact detected, forwarding to: $location');
print('Flutter artifact detected, forwarding to: $location');
return Response.found(location);
}

final location = getFlutterArtifactLocation(
artifactPath: normalizedPath,
engine: flutterEngineRevision,
final isRecognizedFlutterArtifact = flutterArtifactPatterns.any(
(pattern) => RegExp(pattern).hasMatch(path),
);

if (!isRecognizedFlutterArtifact) {
return Response.notFound('Unrecognized artifact path: $path');
}

final location = getFlutterArtifactLocation(artifactPath: path);
print('Flutter artifact detected, forwarding to: $location');
return Response.found(location);
};
Expand Down
Loading