Merge branch 'main' into deploy

api/Dockerfile.dev

@@ -0,0 +1,52 @@
+ARG PYTHON_VERSION_CODE=3.10
+ARG ENVIRONMENT="dev"
+# ENVIRONMENT: dev or prod, refer to project.optional-dependencies in pyproject.toml
+
+FROM python:${PYTHON_VERSION_CODE}-bookworm as builder
+ARG PYTHON_VERSION_CODE
+ARG ENVIRONMENT
+
+WORKDIR /app
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+COPY api/pyproject.toml api/README.md ./
+COPY api/birdxplorer_api/__init__.py ./birdxplorer_api/
+
+RUN if [ "${ENVIRONMENT}" = "prod" ]; then \
+    apt-get update && apt-get install -y --no-install-recommends \
+    postgresql-client-15 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*; \
+    fi
+
+RUN python -m pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -e ".[${ENVIRONMENT}]"
+
+COPY ../common ./common
+RUN if [ "${ENVIRONMENT}" = "dev" ]; then \
+    pip install -e ./common; \
+    fi
+
+FROM python:${PYTHON_VERSION_CODE}-slim-bookworm as runner
+ARG PYTHON_VERSION_CODE
+ARG ENVIRONMENT
+
+WORKDIR /app
+
+RUN if [ "${ENVIRONMENT}" = "prod" ]; then \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libpq5 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*; \
+    fi
+
+RUN groupadd -r app && useradd -r -g app app
+RUN chown -R app:app /app
+USER app
+
+COPY --from=builder /usr/local/lib/python${PYTHON_VERSION_CODE}/site-packages /usr/local/lib/python${PYTHON_VERSION_CODE}/site-packages
+COPY --chown=app:app api ./
+COPY ../common ./common
+
+ENTRYPOINT ["python", "-m", "uvicorn", "birdxplorer_api.main:app", "--host", "0.0.0.0"]

api/birdxplorer_api/app.py

@@ -4,6 +4,7 @@
 from urllib.parse import urlencode as encode_query_string
 
 from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
 from pydantic.alias_generators import to_snake
 from starlette.types import ASGIApp, Receive, Scope, Send
 
@@ -41,6 +42,7 @@ def gen_app(settings: GlobalSettings) -> FastAPI:
     _ = get_logger(level=settings.logger_settings.level)
     storage = gen_storage(settings=settings)
     app = FastAPI()
+    app.add_middleware(CORSMiddleware, **settings.cors_settings.model_dump())
     app.add_middleware(QueryStringFlatteningMiddleware)
     app.include_router(gen_system_router(), prefix="/api/v1/system")
     app.include_router(gen_data_router(storage=storage), prefix="/api/v1/data")

api/birdxplorer_api/routers/data.py

@@ -14,7 +14,6 @@
     PostId,
     Topic,
     TopicId,
-    TweetId,
     TwitterTimestamp,
     UserEnrollment,
 )
@@ -73,7 +72,7 @@ def get_notes(
         created_at_from: Union[None, TwitterTimestamp] = Query(default=None),
         created_at_to: Union[None, TwitterTimestamp] = Query(default=None),
         topic_ids: Union[List[TopicId], None] = Query(default=None),
-        post_ids: Union[List[TweetId], None] = Query(default=None),
+        post_ids: Union[List[PostId], None] = Query(default=None),
         language: Union[LanguageIdentifier, None] = Query(default=None),
     ) -> NoteListResponse:
         return NoteListResponse(
@@ -92,11 +91,14 @@ def get_notes(
     @router.get("/posts", response_model=PostListResponse)
     def get_posts(
         post_id: Union[List[PostId], None] = Query(default=None),
+        note_id: Union[List[NoteId], None] = Query(default=None),
         created_at_start: Union[None, TwitterTimestamp, str] = Query(default=None),
         created_at_end: Union[None, TwitterTimestamp, str] = Query(default=None),
     ) -> PostListResponse:
         if post_id is not None:
             return PostListResponse(data=list(storage.get_posts_by_ids(post_ids=post_id)))
+        if note_id is not None:
+            return PostListResponse(data=list(storage.get_posts_by_note_ids(note_ids=note_id)))
         if created_at_start is not None:
             if created_at_end is not None:
                 return PostListResponse(

api/pyproject.toml

@@ -25,7 +25,6 @@ classifiers = [
 ]
 
 dependencies = [
-    "birdxplorer_common @ git+https://github.com/codeforjapan/BirdXplorer.git@main#subdirectory=common",
     "fastapi",
     "python-dateutil",
     "pydantic",
@@ -37,10 +36,10 @@ dependencies = [
 Source = "https://github.com/codeforjapan/BirdXplorer"
 
 [tool.setuptools]
-packages=["birdxplorer"]
+packages=["birdxplorer_api"]
 
 [tool.setuptools.package-data]
-birdxplorer = ["py.typed"]
+birdxplorer_api = ["py.typed"]
 
 [project.optional-dependencies]
 dev=[
@@ -62,6 +61,7 @@ dev=[
     "httpx",
 ]
 prod=[
+    "birdxplorer_common @ git+https://github.com/codeforjapan/BirdXplorer.git@main#subdirectory=common",
     "psycopg2",
     "gunicorn",
 ]
@@ -106,6 +106,7 @@ legacy_tox_ini = """
         VIRTUALENV_PIP = 24.0
     deps =
         -e .[dev]
+        -e ../common
     commands =
         black birdxplorer_api tests
         isort birdxplorer_api tests

api/tests/conftest.py

@@ -21,12 +21,15 @@
     PostId,
     Topic,
     TopicId,
-    TweetId,
     TwitterTimestamp,
     UserEnrollment,
     XUser,
 )
-from birdxplorer_common.settings import GlobalSettings, PostgresStorageSettings
+from birdxplorer_common.settings import (
+    CORSSettings,
+    GlobalSettings,
+    PostgresStorageSettings,
+)
 from birdxplorer_common.storage import Storage
 
 
@@ -223,7 +226,7 @@ def _get_notes(
         created_at_from: Union[None, TwitterTimestamp] = None,
         created_at_to: Union[None, TwitterTimestamp] = None,
         topic_ids: Union[List[TopicId], None] = None,
-        post_ids: Union[List[TweetId], None] = None,
+        post_ids: Union[List[PostId], None] = None,
         language: Union[LanguageIdentifier, None] = None,
     ) -> Generator[Note, None, None]:
         for note in note_samples:
@@ -258,6 +261,15 @@ def _get_posts_by_ids(post_ids: List[PostId]) -> Generator[Post, None, None]:
 
     mock.get_posts_by_ids.side_effect = _get_posts_by_ids
 
+    def _get_posts_by_note_ids(note_ids: List[NoteId]) -> Generator[Post, None, None]:
+        for post in post_samples:
+            for note in note_samples:
+                if note.note_id in note_ids and post.post_id == note.post_id:
+                    yield post
+                    break
+
+    mock.get_posts_by_note_ids.side_effect = _get_posts_by_note_ids
+
     def _get_posts_by_created_at_range(start: TwitterTimestamp, end: TwitterTimestamp) -> Generator[Post, None, None]:
         for post in post_samples:
             if start <= post.created_at < end:
@@ -294,6 +306,19 @@ def load_dotenv_fixture() -> None:
     load_dotenv()
 
 
+@fixture
+def cors_settings_factory(load_dotenv_fixture: None) -> Type[ModelFactory[CORSSettings]]:
+    class CORSSettingsFactory(ModelFactory[CORSSettings]):
+        __model__ = CORSSettings
+
+        allow_credentials = True
+        allow_methods = ["*"]
+        allow_headers = ["*"]
+        allow_origins = ["*"]
+
+    return CORSSettingsFactory
+
+
 @fixture
 def postgres_storage_settings_factory(
     load_dotenv_fixture: None,
@@ -312,11 +337,13 @@ class PostgresStorageSettingsFactory(ModelFactory[PostgresStorageSettings]):
 
 @fixture
 def global_settings_factory(
+    cors_settings_factory: Type[ModelFactory[CORSSettings]],
     postgres_storage_settings_factory: Type[ModelFactory[PostgresStorageSettings]],
 ) -> Type[ModelFactory[GlobalSettings]]:
     class GlobalSettingsFactory(ModelFactory[GlobalSettings]):
         __model__ = GlobalSettings
 
+        cors_settings = cors_settings_factory.build()
         storage_settings = postgres_storage_settings_factory.build()
 
     return GlobalSettingsFactory
@@ -325,10 +352,12 @@ class GlobalSettingsFactory(ModelFactory[GlobalSettings]):
 @fixture
 def settings_for_test(
     global_settings_factory: Type[ModelFactory[GlobalSettings]],
+    cors_settings_factory: Type[ModelFactory[CORSSettings]],
     postgres_storage_settings_factory: Type[ModelFactory[PostgresStorageSettings]],
 ) -> Generator[GlobalSettings, None, None]:
     yield global_settings_factory.build(
-        storage_settings=postgres_storage_settings_factory.build(database=TEST_DATABASE_NAME)
+        cors_settings=cors_settings_factory.build(allow_origins=["http://allowed.example.com"]),
+        storage_settings=postgres_storage_settings_factory.build(database=TEST_DATABASE_NAME),
     )
 
 

api/tests/routers/test_data.py

@@ -39,6 +39,13 @@ def test_posts_get_has_post_id_filter(client: TestClient, post_samples: List[Pos
     }
 
 
+def test_posts_get_has_note_id_filter(client: TestClient, post_samples: List[Post], note_samples: List[Note]) -> None:
+    response = client.get(f"/api/v1/data/posts/?noteId={','.join([n.note_id for n in note_samples])}")
+    assert response.status_code == 200
+    res_json = response.json()
+    assert res_json == {"data": [json.loads(post_samples[0].model_dump_json())]}
+
+
 def test_posts_get_has_created_at_filter_start_and_end(client: TestClient, post_samples: List[Post]) -> None:
     response = client.get("/api/v1/data/posts/?createdAtStart=2006-7-25 00:00:00&createdAtEnd=2006-7-30 23:59:59")
     assert response.status_code == 200

api/tests/routers/test_system.py

@@ -1,7 +1,30 @@
+from fastapi import status
 from fastapi.testclient import TestClient
 
 
 def test_ping(client: TestClient) -> None:
     response = client.get("/api/v1/system/ping")
     assert response.status_code == 200
     assert response.json() == {"message": "pong"}
+
+
+def test_allowed_cors(client: TestClient) -> None:
+    headers = {
+        "Access-Control-Request-Method": "GET",
+        "Origin": "http://allowed.example.com",
+    }
+
+    response = client.options("/api/v1/system/ping", headers=headers)
+    assert response.status_code == status.HTTP_200_OK
+    assert response.headers["access-control-allow-origin"] == headers["Origin"]
+
+
+def test_disallowed_cors(client: TestClient) -> None:
+    headers = {
+        "Origin": "http://disallowed.example.com",
+        "Access-Control-Request-Method": "GET",
+    }
+
+    response = client.options("/api/v1/system/ping", headers=headers)
+    assert response.status_code == status.HTTP_400_BAD_REQUEST
+    assert "access-control-allow-origin" not in response.headers

common/birdxplorer_common/models.py

@@ -565,7 +565,7 @@ class NotesValidationDifficulty(str, Enum):
     empty = ""
 
 
-class TweetId(UpToNineteenDigitsDecimalString): ...
+class PostId(UpToNineteenDigitsDecimalString): ...
 
 
 class NoteData(BaseModel):
@@ -576,7 +576,7 @@ class NoteData(BaseModel):
     note_id: NoteId
     note_author_participant_id: ParticipantId
     created_at_millis: TwitterTimestamp
-    tweet_id: TweetId
+    tweet_id: PostId
     believable: NotesBelievable
     misleading_other: BinaryBool
     misleading_factual_error: BinaryBool
@@ -629,7 +629,7 @@ class SummaryString(NonEmptyTrimmedString): ...
 
 class Note(BaseModel):
     note_id: NoteId
-    post_id: TweetId
+    post_id: PostId
     language: LanguageIdentifier
     topics: List[Topic]
     summary: SummaryString
@@ -650,9 +650,6 @@ class XUser(BaseModel):
     following_count: NonNegativeInt
 
 
-class PostId(UpToNineteenDigitsDecimalString): ...
-
-
 MediaDetails: TypeAlias = List[HttpUrl] | None
 
 

common/birdxplorer_common/settings.py

@@ -18,7 +18,7 @@ class PostgresStorageSettings(BaseSettings):
     port: int = 5432
     database: str = "postgres"
 
-    @computed_field  # type: ignore[misc]
+    @computed_field  # type: ignore[prop-decorator]
     @property
     def sqlalchemy_database_url(self) -> str:
         return PostgresDsn(
@@ -27,7 +27,16 @@ def sqlalchemy_database_url(self) -> str:
         ).unicode_string()
 
 
+class CORSSettings(BaseSettings):
+    allow_credentials: bool = True
+    allow_methods: list[str] = ["GET"]
+    allow_headers: list[str] = ["*"]
+
+    allow_origins: list[str] = []
+
+
 class GlobalSettings(BaseSettings):
+    cors_settings: CORSSettings = Field(default_factory=CORSSettings)
     model_config = SettingsConfigDict(env_file=".env")
     logger_settings: LoggerSettings = Field(default_factory=LoggerSettings)
     storage_settings: PostgresStorageSettings