From 8d1d7624a9c1ec3b9117003e5f0d1be4c2d61d95 Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 18:36:32 -0400 Subject: [PATCH 1/9] First, alter game_log_archives.archive_type to integer type --- ...alter_game_log_archives_type_field_enum.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 ark_nova_stats/api/migrations/versions/fef90e3dfd97_alter_game_log_archives_type_field_enum.py diff --git a/ark_nova_stats/api/migrations/versions/fef90e3dfd97_alter_game_log_archives_type_field_enum.py b/ark_nova_stats/api/migrations/versions/fef90e3dfd97_alter_game_log_archives_type_field_enum.py new file mode 100644 index 00000000..4bb1d147 --- /dev/null +++ b/ark_nova_stats/api/migrations/versions/fef90e3dfd97_alter_game_log_archives_type_field_enum.py @@ -0,0 +1,39 @@ +"""alter-game-log-archives-type-field-enum + +Revision ID: fef90e3dfd97 +Revises: 89684eb58df8 +Create Date: 2024-08-03 18:31:10.362460 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "fef90e3dfd97" +down_revision = "89684eb58df8" +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + "game_log_archives", + "archive_type", + type_=sa.Integer, + server_default=0, + nullable=False, + existing_type=sa.UnicodeText, + existing_nullable=False, + ) + + +def downgrade(): + op.alter_column( + "game_log_archives", + "archive_type", + type_=sa.UnicodeText, + nullable=False, + existing_type=sa.Integer, + existing_nullable=False, + ) From bd6b50dbc0d0efef6c7f1a8bcf2b0c936cb9fbd2 Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 19:09:15 -0400 Subject: [PATCH 2/9] In worker, set enum type --- ark_nova_stats/api/gql/schema.py | 3 +- ark_nova_stats/api/gql/types/game_log.py | 46 ++++++++++++++++++++++++ ark_nova_stats/models.py | 10 +++++- ark_nova_stats/worker/app.py | 8 ++--- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/ark_nova_stats/api/gql/schema.py b/ark_nova_stats/api/gql/schema.py index b085cc7e..b0e0afe4 100644 --- a/ark_nova_stats/api/gql/schema.py +++ b/ark_nova_stats/api/gql/schema.py @@ -1,3 +1,4 @@ +import flask from graphql import GraphQLObjectType, GraphQLSchema from ark_nova_stats.api.gql.types.game_log import ( @@ -10,7 +11,7 @@ from ark_nova_stats.models import GameLog, User -def Schema(app): +def Schema(app: flask.Flask): return GraphQLSchema( query=GraphQLObjectType( name="Query", diff --git a/ark_nova_stats/api/gql/types/game_log.py b/ark_nova_stats/api/gql/types/game_log.py index f7bade3d..79ece449 100644 --- a/ark_nova_stats/api/gql/types/game_log.py +++ b/ark_nova_stats/api/gql/types/game_log.py @@ -15,6 +15,7 @@ from ark_nova_stats.bga_log_parser.game_log import GameLog as ParsedGameLog from ark_nova_stats.config import app, db from ark_nova_stats.models import GameLog as GameLogModel +from ark_nova_stats.models import GameLogArchiveType from ark_nova_stats.models import User as UserModel @@ -238,3 +239,48 @@ def stats_field( args={}, resolve=lambda root, info, **args: fetch_stats(game_log_model, user_model), ) + + +def game_log_archive_fields() -> dict[str, GraphQLField]: + archive_types = set(t.name for t in GameLogArchiveType) + return { + "id": GraphQLField( + GraphQLNonNull(GraphQLInt), + description="ID of game log archive.", + ), + "archiveType": GraphQLField( + GraphQLNonNull(GraphQLString), + description=f"Type of game logs. Possible values are: {', '.join(archive_types)}.", + ), + "url": GraphQLField( + GraphQLNonNull(GraphQLString), + description=f"URL of archive.", + ), + "sizeBytes": GraphQLField( + GraphQLNonNull(GraphQLInt), + description=f"Size of archive, in bytes.", + ), + "numGameLogs": GraphQLField( + GraphQLNonNull(GraphQLInt), + description=f"Number of game logs in this archive.", + ), + "numUsers": GraphQLField( + GraphQLNonNull(GraphQLInt), + description=f"Number of users in this archive.", + ), + "maxGameLog": GraphQLField( + GraphQLNonNull(GraphQLInt), + description=f"The game log with the highest BGA table ID.", + ), + "createdAt": GraphQLField( + GraphQLInt, + description="UNIX timestamp for when this archive was created.", + ), + } + + +game_log_archive_type = GraphQLObjectType( + "GameLogArchive", + description="An archive of game logs.", + fields=game_log_archive_fields, +) diff --git a/ark_nova_stats/models.py b/ark_nova_stats/models.py index 26028240..2abe6552 100644 --- a/ark_nova_stats/models.py +++ b/ark_nova_stats/models.py @@ -1,4 +1,5 @@ import datetime +import enum from sqlalchemy import ForeignKey from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -92,9 +93,16 @@ class User(db.Model): # type: ignore ) +class GameLogArchiveType(enum.Enum): + GAME_LOG_ARCHIVE_TYPE_UNKNOWN = 0 + RAW_BGA_JSONL = 1 + + class GameLogArchive(db.Model): # type: ignore id: Mapped[int] = mapped_column(primary_key=True) - archive_type: Mapped[str] + archive_type: Mapped[GameLogArchiveType] = mapped_column( + default=GameLogArchiveType.GAME_LOG_ARCHIVE_TYPE_UNKNOWN + ) url: Mapped[str] size_bytes: Mapped[int] num_game_logs: Mapped[int] diff --git a/ark_nova_stats/worker/app.py b/ark_nova_stats/worker/app.py index e3e71b87..dde6c54f 100644 --- a/ark_nova_stats/worker/app.py +++ b/ark_nova_stats/worker/app.py @@ -10,7 +10,7 @@ from sqlalchemy import desc from ark_nova_stats.config import app, db -from ark_nova_stats.models import GameLog, GameLogArchive, User +from ark_nova_stats.models import GameLog, GameLogArchive, GameLogArchiveType, User max_delay = 10 @@ -41,7 +41,7 @@ def archive_logs_to_tigris( users: set[str] = set() last_game_log: Optional[GameLog] = None log_list = [] - archive_type = "raw_bga_jsonl" + archive_type = GameLogArchiveType.RAW_BGA_JSONL # Assemble a list of the game logs and compress them using gzip. for game_log in all_logs: @@ -55,7 +55,7 @@ def archive_logs_to_tigris( compressed_jsonl = gzip.compress("\n".join(log_list).encode("utf-8")) size_bytes = len(compressed_jsonl) filename = ( - archive_type + archive_type.name + "_" + datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y_%M_%d") + ".gz" @@ -65,7 +65,7 @@ def archive_logs_to_tigris( tigris_client.upload_fileobj( io.BytesIO(compressed_jsonl), os.getenv("BUCKET_NAME"), - archive_type + "/" + filename, + archive_type.name + "/" + filename, ) url = f"{os.getenv('TIGRIS_CUSTOM_DOMAIN_HOST')}/{filename}" logger.info(f"Uploaded game log archive at: {url} with size: {size_bytes}") From 43beabe65439c2879ed16d04d5f450db0b87404c Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 19:12:36 -0400 Subject: [PATCH 3/9] Run gazelle --- ark_nova_stats/api/gql/BUILD.bazel | 1 + ark_nova_stats/api/gql/types/BUILD.bazel | 1 + ark_nova_stats/worker/BUILD.bazel | 2 ++ ark_nova_stats/worker/fly.toml | 2 +- 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ark_nova_stats/api/gql/BUILD.bazel b/ark_nova_stats/api/gql/BUILD.bazel index 4c603369..ab982d13 100644 --- a/ark_nova_stats/api/gql/BUILD.bazel +++ b/ark_nova_stats/api/gql/BUILD.bazel @@ -7,6 +7,7 @@ py_library( deps = [ "//ark_nova_stats:models_py", "//ark_nova_stats/api/gql/types:game_log", + "@py_deps//flask", "@py_deps//graphql_core", ], ) diff --git a/ark_nova_stats/api/gql/types/BUILD.bazel b/ark_nova_stats/api/gql/types/BUILD.bazel index ad1d724f..c9f68684 100644 --- a/ark_nova_stats/api/gql/types/BUILD.bazel +++ b/ark_nova_stats/api/gql/types/BUILD.bazel @@ -9,5 +9,6 @@ py_library( "//ark_nova_stats:models_py", "//ark_nova_stats/bga_log_parser:game_log", "@py_deps//graphql_core", + "@py_deps//sqlalchemy", ], ) diff --git a/ark_nova_stats/worker/BUILD.bazel b/ark_nova_stats/worker/BUILD.bazel index 85b02126..23e872cc 100644 --- a/ark_nova_stats/worker/BUILD.bazel +++ b/ark_nova_stats/worker/BUILD.bazel @@ -9,7 +9,9 @@ py_binary( visibility = ["//:__subpackages__"], deps = [ "//ark_nova_stats:config_py", + "//ark_nova_stats:models_py", "@py_deps//boto3", + "@py_deps//sqlalchemy", ], ) diff --git a/ark_nova_stats/worker/fly.toml b/ark_nova_stats/worker/fly.toml index 07ccd699..036234c0 100644 --- a/ark_nova_stats/worker/fly.toml +++ b/ark_nova_stats/worker/fly.toml @@ -1,4 +1,4 @@ -# fly.toml app configuration file generated for worker-red-log-3531 on 2024-07-31T21:33:15-04:00 +# fly.toml app configuration file generated for ark-nova-stats-worker on 2024-08-03T19:09:45-04:00 # # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # From e31fee51a448266d39d086471c4db522f9308605 Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 19:17:24 -0400 Subject: [PATCH 4/9] Rename table --- ark_nova_stats/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ark_nova_stats/models.py b/ark_nova_stats/models.py index 2abe6552..8b5c49e8 100644 --- a/ark_nova_stats/models.py +++ b/ark_nova_stats/models.py @@ -99,6 +99,8 @@ class GameLogArchiveType(enum.Enum): class GameLogArchive(db.Model): # type: ignore + __tablename__ = "game_log_archives" + id: Mapped[int] = mapped_column(primary_key=True) archive_type: Mapped[GameLogArchiveType] = mapped_column( default=GameLogArchiveType.GAME_LOG_ARCHIVE_TYPE_UNKNOWN From 3fdee64eb715edc7f13b8216f10be260f3a0fb7d Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 19:32:47 -0400 Subject: [PATCH 5/9] Abandon enum type in postgres --- ...alter_game_log_archives_type_field_enum.py | 27 ++++++++++--------- ark_nova_stats/models.py | 4 +-- ark_nova_stats/worker/app.py | 15 ++++++----- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/ark_nova_stats/api/migrations/versions/fef90e3dfd97_alter_game_log_archives_type_field_enum.py b/ark_nova_stats/api/migrations/versions/fef90e3dfd97_alter_game_log_archives_type_field_enum.py index 4bb1d147..9f0c6baf 100644 --- a/ark_nova_stats/api/migrations/versions/fef90e3dfd97_alter_game_log_archives_type_field_enum.py +++ b/ark_nova_stats/api/migrations/versions/fef90e3dfd97_alter_game_log_archives_type_field_enum.py @@ -17,23 +17,24 @@ def upgrade(): - op.alter_column( + op.drop_column("game_log_archives", "archive_type") + op.add_column( "game_log_archives", - "archive_type", - type_=sa.Integer, - server_default=0, - nullable=False, - existing_type=sa.UnicodeText, - existing_nullable=False, + sa.Column( + "archive_type", + sa.Integer, + nullable=False, + ), ) def downgrade(): - op.alter_column( + op.drop_column("game_log_archives", "archive_type") + op.add_column( "game_log_archives", - "archive_type", - type_=sa.UnicodeText, - nullable=False, - existing_type=sa.Integer, - existing_nullable=False, + sa.Column( + "archive_type", + sa.UnicodeText, + nullable=False, + ), ) diff --git a/ark_nova_stats/models.py b/ark_nova_stats/models.py index 8b5c49e8..f2cff7ff 100644 --- a/ark_nova_stats/models.py +++ b/ark_nova_stats/models.py @@ -102,9 +102,7 @@ class GameLogArchive(db.Model): # type: ignore __tablename__ = "game_log_archives" id: Mapped[int] = mapped_column(primary_key=True) - archive_type: Mapped[GameLogArchiveType] = mapped_column( - default=GameLogArchiveType.GAME_LOG_ARCHIVE_TYPE_UNKNOWN - ) + archive_type: Mapped[int] url: Mapped[str] size_bytes: Mapped[int] num_game_logs: Mapped[int] diff --git a/ark_nova_stats/worker/app.py b/ark_nova_stats/worker/app.py index dde6c54f..6212cfcd 100644 --- a/ark_nova_stats/worker/app.py +++ b/ark_nova_stats/worker/app.py @@ -29,12 +29,13 @@ def archive_logs_to_tigris( last_archive: GameLogArchive = GameLogArchive.query.order_by( desc(GameLogArchive.created_at) ).first() - time_since_last_archive = datetime.datetime.now() - last_archive.created_at - if last_archive is not None and time_since_last_archive < min_interval: - logger.debug( - f"Last archive was uploaded at {last_archive.created_at}, which was {time_since_last_archive} ago; skipping." - ) - return None + if last_archive is not None: + time_since_last_archive = datetime.datetime.now() - last_archive.created_at + if time_since_last_archive < min_interval: + logger.debug( + f"Last archive was uploaded at {last_archive.created_at}, which was {time_since_last_archive} ago; skipping." + ) + return None # Retrieve all the game logs so we can serialize them. all_logs: list[GameLog] = GameLog.query.all() @@ -78,7 +79,7 @@ def archive_logs_to_tigris( new_archive = GameLogArchive( url=url, - archive_type=archive_type, + archive_type=archive_type.value, size_bytes=size_bytes, num_game_logs=len(all_logs), num_users=len(users), From 4cb3c7453ec26fc9e30aec520b8989fb5bfe3960 Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 19:33:50 -0400 Subject: [PATCH 6/9] Check much less often --- ark_nova_stats/worker/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ark_nova_stats/worker/app.py b/ark_nova_stats/worker/app.py index 6212cfcd..8565665c 100644 --- a/ark_nova_stats/worker/app.py +++ b/ark_nova_stats/worker/app.py @@ -12,7 +12,7 @@ from ark_nova_stats.config import app, db from ark_nova_stats.models import GameLog, GameLogArchive, GameLogArchiveType, User -max_delay = 10 +max_delay = 12 * 60 * 60 logging.basicConfig( format="[%(asctime)s][%(levelname)s] %(message)s", level=logging.WARNING From 347beef3e1efab96195e1b929dc37e9a445d0720 Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 19:34:19 -0400 Subject: [PATCH 7/9] Log more --- ark_nova_stats/worker/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ark_nova_stats/worker/app.py b/ark_nova_stats/worker/app.py index 8565665c..eb680dce 100644 --- a/ark_nova_stats/worker/app.py +++ b/ark_nova_stats/worker/app.py @@ -15,7 +15,7 @@ max_delay = 12 * 60 * 60 logging.basicConfig( - format="[%(asctime)s][%(levelname)s] %(message)s", level=logging.WARNING + format="[%(asctime)s][%(levelname)s] %(message)s", level=logging.INFO ) logger = logging.getLogger(__name__) From 8c76429e5dd76cbcbd6e19625b43b2f8134db63c Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 19:35:23 -0400 Subject: [PATCH 8/9] Fix timestamp type --- ark_nova_stats/worker/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ark_nova_stats/worker/app.py b/ark_nova_stats/worker/app.py index eb680dce..9ed9bc2b 100644 --- a/ark_nova_stats/worker/app.py +++ b/ark_nova_stats/worker/app.py @@ -58,7 +58,7 @@ def archive_logs_to_tigris( filename = ( archive_type.name + "_" - + datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y_%M_%d") + + datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y_%m_%d") + ".gz" ) From 61ee0909d964784e8da7cc5bfbd076a531b6aa9c Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Sat, 3 Aug 2024 19:39:28 -0400 Subject: [PATCH 9/9] Push & deploy worker --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2c62f0b7..b6beaf96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,6 +52,7 @@ jobs: run: | bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //ark_nova_stats/api:api_image_image_dockerhub bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //ark_nova_stats/frontend:production_cross_platform_image_dockerhub + bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //ark_nova_stats/worker:worker_image_dockerhub deploy_ark_nova_stats: needs: push_ark_nova_stats if: ${{ github.ref == 'refs/heads/main' }} @@ -68,6 +69,8 @@ jobs: working-directory: ark_nova_stats/api - run: flyctl deploy working-directory: ark_nova_stats/frontend + - run: flyctl deploy + working-directory: ark_nova_stats/worker push_fitbit_challenges: needs: build if: ${{ github.ref == 'refs/heads/main' }}