From 15d068bfad7d3e20692adbf260fcaa311a2eef01 Mon Sep 17 00:00:00 2001 From: 4l3j0Ok Date: Thu, 28 Mar 2024 16:23:13 -0300 Subject: [PATCH 1/5] Moved files into src --- {backend => src}/Dockerfile | 6 +++--- {backend/src => src}/db/client.py | 0 {backend/src => src}/main.py | 4 ++-- {backend/src => src}/models/bank.py | 0 {backend/src => src}/models/response.py | 0 {backend/src => src}/modules/bank.py | 0 {backend/src => src}/modules/config.py | 5 +++-- {backend/src => src}/modules/logger.py | 0 {backend/src => src}/modules/messages.py | 0 {backend/src => src}/modules/soup.py | 0 {backend/src => src}/modules/utils.py | 0 {backend => src}/requirements.txt | 0 {backend/src => src}/routers/bank.py | 0 stack/dev.compose.yml | 19 +++++++++++++++++++ .../{docker-compose.yml => prod.compose.yml} | 9 ++------- 15 files changed, 29 insertions(+), 14 deletions(-) rename {backend => src}/Dockerfile (69%) rename {backend/src => src}/db/client.py (100%) rename {backend/src => src}/main.py (100%) rename {backend/src => src}/models/bank.py (100%) rename {backend/src => src}/models/response.py (100%) rename {backend/src => src}/modules/bank.py (100%) rename {backend/src => src}/modules/config.py (89%) rename {backend/src => src}/modules/logger.py (100%) rename {backend/src => src}/modules/messages.py (100%) rename {backend/src => src}/modules/soup.py (100%) rename {backend/src => src}/modules/utils.py (100%) rename {backend => src}/requirements.txt (100%) rename {backend/src => src}/routers/bank.py (100%) create mode 100644 stack/dev.compose.yml rename stack/{docker-compose.yml => prod.compose.yml} (59%) diff --git a/backend/Dockerfile b/src/Dockerfile similarity index 69% rename from backend/Dockerfile rename to src/Dockerfile index ee7afe6..8be10c8 100755 --- a/backend/Dockerfile +++ b/src/Dockerfile @@ -1,12 +1,12 @@ -FROM python:3.11.4-alpine as dependencies +FROM python:3.12.2-alpine as dependencies WORKDIR /app COPY ./requirements.txt . RUN pip install --user -r ./requirements.txt -FROM python:3.11.4-alpine as builder +FROM python:3.12.2-alpine as builder WORKDIR /app -COPY ./src /app/src +COPY . /app/src COPY --from=dependencies /root/.local /root/.local ENV PATH=/root/.local/bin:$PATH EXPOSE 8080 diff --git a/backend/src/db/client.py b/src/db/client.py similarity index 100% rename from backend/src/db/client.py rename to src/db/client.py diff --git a/backend/src/main.py b/src/main.py similarity index 100% rename from backend/src/main.py rename to src/main.py index 9ba5d20..32feb53 100755 --- a/backend/src/main.py +++ b/src/main.py @@ -1,6 +1,6 @@ -from fastapi import FastAPI, responses -import uvicorn import importlib +import uvicorn +from fastapi import FastAPI, responses from modules import config, utils from modules.logger import logger diff --git a/backend/src/models/bank.py b/src/models/bank.py similarity index 100% rename from backend/src/models/bank.py rename to src/models/bank.py diff --git a/backend/src/models/response.py b/src/models/response.py similarity index 100% rename from backend/src/models/response.py rename to src/models/response.py diff --git a/backend/src/modules/bank.py b/src/modules/bank.py similarity index 100% rename from backend/src/modules/bank.py rename to src/modules/bank.py diff --git a/backend/src/modules/config.py b/src/modules/config.py similarity index 89% rename from backend/src/modules/config.py rename to src/modules/config.py index 515a355..7cecd68 100755 --- a/backend/src/modules/config.py +++ b/src/modules/config.py @@ -3,7 +3,7 @@ APP_HOST = os.getenv("APP_HOST", "localhost") APP_PORT = int(os.getenv("APP_PORT", "8080")) -MONGO_URI = os.getenv("MONGO_URI") +MONGO_URI = os.getenv("MONGO_URI") if not MONGO_URI: MONGO_HOST = os.getenv("MONGO_HOST") MONGO_PORT = int(os.getenv("MONGO_PORT")) @@ -12,7 +12,8 @@ BCRA_URL = "https://www.bcra.gob.ar/SistemasFinancierosYdePagos/Sistema_financiero_nomina_de_entidades.asp?bco=AAA00&tipo=1" DOCKER_ORGANIZATION = os.getenv("DOCKER_ORGANIZATION", "alejoide") DOCKER_REPOSITORY = os.getenv("DOCKER_REPOSITORY", "bcra-scraper-api") -DOCKER_API_URL = f"https://registry.hub.docker.com/v2/repositories/{DOCKER_ORGANIZATION}/{DOCKER_REPOSITORY}/tags" +DOCKER_API_URL = f"https://registry.hub.docker.com/v2/repositories/{ + DOCKER_ORGANIZATION}/{DOCKER_REPOSITORY}/tags" ROUTERS = [ "bank" ] diff --git a/backend/src/modules/logger.py b/src/modules/logger.py similarity index 100% rename from backend/src/modules/logger.py rename to src/modules/logger.py diff --git a/backend/src/modules/messages.py b/src/modules/messages.py similarity index 100% rename from backend/src/modules/messages.py rename to src/modules/messages.py diff --git a/backend/src/modules/soup.py b/src/modules/soup.py similarity index 100% rename from backend/src/modules/soup.py rename to src/modules/soup.py diff --git a/backend/src/modules/utils.py b/src/modules/utils.py similarity index 100% rename from backend/src/modules/utils.py rename to src/modules/utils.py diff --git a/backend/requirements.txt b/src/requirements.txt similarity index 100% rename from backend/requirements.txt rename to src/requirements.txt diff --git a/backend/src/routers/bank.py b/src/routers/bank.py similarity index 100% rename from backend/src/routers/bank.py rename to src/routers/bank.py diff --git a/stack/dev.compose.yml b/stack/dev.compose.yml new file mode 100644 index 0000000..97c90a5 --- /dev/null +++ b/stack/dev.compose.yml @@ -0,0 +1,19 @@ +version: "3.9" +services: + BCRA-Scraper-API: + image: alejoide/bcra-scraper-api:latest + container_name: bcra-scraper-api + build: + context: ../src/ + dockerfile: ../src/Dockerfile + restart: unless-stopped + ports: + - "8080:8080" + env_file: + - .env + networks: + - mongo + +networks: + mongo: + external: true diff --git a/stack/docker-compose.yml b/stack/prod.compose.yml similarity index 59% rename from stack/docker-compose.yml rename to stack/prod.compose.yml index a18b44a..5d667c8 100644 --- a/stack/docker-compose.yml +++ b/stack/prod.compose.yml @@ -1,18 +1,13 @@ version: "3.9" services: - - backend: + BCRA-Scraper-API: image: alejoide/bcra-scraper-api:latest restart: unless-stopped env_file: - .env networks: - mongo - - nginx-proxy -# Only for production environments when using jwilder/nginx-proxy. networks: - nginx-proxy: - external: true mongo: - external: true \ No newline at end of file + external: true From ace0acb697c21b1f0659c0b5bb2ff70a7f8e8bc4 Mon Sep 17 00:00:00 2001 From: 4l3j0Ok Date: Thu, 28 Mar 2024 16:25:00 -0300 Subject: [PATCH 2/5] updated compose file reference --- .github/workflows/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 4894d7e..a627818 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -24,7 +24,7 @@ jobs: name: Setting up Stack run: | mkdir -p $PROJECT_PATH - cp -f ./stack/docker-compose.yml $PROJECT_PATH/docker-compose.yml + cp -f ./stack/prod.compose.yml $PROJECT_PATH/docker-compose.yml echo "$ENV_CONTENT" > $PROJECT_PATH/.env env: PROJECT_PATH: ${{ secrets.PROJECT_PATH }} From d5339d7fe4cc3399547f298c65a34c1482273a90 Mon Sep 17 00:00:00 2001 From: 4l3j0Ok Date: Thu, 28 Mar 2024 16:41:40 -0300 Subject: [PATCH 3/5] Changed build context folder --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bd85242..eab2c8d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -31,6 +31,6 @@ jobs: name: Build and push to Docker Hub uses: docker/build-push-action@v5 with: - context: ./backend + context: ./src tags: ${{ steps.meta.outputs.tags }} push: true \ No newline at end of file From f75952264ae2fdd6d1dca1772f7ac375f5b0b4b0 Mon Sep 17 00:00:00 2001 From: 4l3j0Ok Date: Thu, 28 Mar 2024 16:42:02 -0300 Subject: [PATCH 4/5] Added volume to dev compose --- stack/dev.compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stack/dev.compose.yml b/stack/dev.compose.yml index 97c90a5..67a7f6b 100644 --- a/stack/dev.compose.yml +++ b/stack/dev.compose.yml @@ -6,6 +6,8 @@ services: build: context: ../src/ dockerfile: ../src/Dockerfile + volumes: + - ../src:/app/src restart: unless-stopped ports: - "8080:8080" From 35548e0954349db0ab8163c4f613f5f80f1c02c1 Mon Sep 17 00:00:00 2001 From: 4l3j0Ok Date: Thu, 28 Mar 2024 16:42:42 -0300 Subject: [PATCH 5/5] Remove bank mongo id --- src/models/bank.py | 10 ---------- src/models/response.py | 4 +--- src/modules/bank.py | 28 +++++++++++++--------------- src/modules/messages.py | 3 ++- src/routers/bank.py | 2 -- 5 files changed, 16 insertions(+), 31 deletions(-) diff --git a/src/models/bank.py b/src/models/bank.py index e08d168..4b1f18c 100644 --- a/src/models/bank.py +++ b/src/models/bank.py @@ -1,5 +1,4 @@ from pydantic import BaseModel -from bson import ObjectId class Bank(BaseModel): @@ -7,15 +6,6 @@ class Bank(BaseModel): bcra_id: str -class BankDB(Bank): - id: str | None - - def __init__(self, **bank): - if "_id" in bank and isinstance(bank["_id"], ObjectId): - bank["id"] = str(bank["_id"]) - super().__init__(**bank) - - class AdminCredentials(BaseModel): username: str password: str diff --git a/src/models/response.py b/src/models/response.py index 0e065d7..9964920 100644 --- a/src/models/response.py +++ b/src/models/response.py @@ -8,15 +8,13 @@ class Detail(TypedDict): class Success(BaseModel): - input: str | int | dict | None detail: Detail class Failed(BaseModel): - input: str | int | dict | None detail: Detail class Result(BaseModel): success: list[Success] | list - failed: list[Failed] | list \ No newline at end of file + failed: list[Failed] | list diff --git a/src/modules/bank.py b/src/modules/bank.py index c8ef027..d4d44c3 100644 --- a/src/modules/bank.py +++ b/src/modules/bank.py @@ -1,26 +1,20 @@ from modules import soup, messages, config from modules.logger import logger -from models.bank import BankDB +from models.bank import Bank from db.client import client -from bson import ObjectId import os db = client.bcra_scraper.banks -def get_banks(bank_id: str = None, as_dict: bool = False): - logger.info(f"Validando ID {bank_id}.") - if bank_id: - if not ObjectId.is_valid(bank_id): - logger.warning(f"ID {bank_id} inválido.") - return False, messages.ERR_SCHEMA.format(err_args="id") +def get_banks(bcra_id: str = None, as_dict: bool = False): logger.info("Buscando bancos en la DB.") - result = list(db.find({"_id":ObjectId(bank_id)} if bank_id else None)) + result = list(db.find({"bcra_id": bcra_id} if bcra_id else None)) if not result: return False, messages.ERR_BANK_NOT_FOUND - banks = [BankDB(**bank) for bank in result] if not as_dict \ - else [BankDB(**bank).model_dump() for bank in result] + banks = [Bank(**bank) for bank in result] if not as_dict \ + else [Bank(**bank).model_dump() for bank in result] return True, banks if len(banks) != 1 else banks[0] @@ -28,15 +22,19 @@ def update_banks(): logger.info("Actualizando la lista de bancos con SOUP...") data = soup.get_banks_from_url(config.BCRA_URL) delete_query = {"bcra_id": {"$in": [bank["bcra_id"] for bank in data]}} - db.delete_many(delete_query) - result = db.insert_many(data) - return True, [str(bank_id) for bank_id in result.inserted_ids] + try: + db.delete_many(delete_query) + db.insert_many(data) + except Exception as ex: + logger.exception(ex) + return False, messages.ERR_FAILED_TO_UPDATE + return True, messages.MSG_SUCCESS_SAVE def add_bank(bank): logger.info("Agregando un nuevo banco...") result = db.insert_one(bank.dict()) - return True, str(result.inserted_id) + return True, messages.MSG_SUCCESS_SAVE def validate_admin(credentials): diff --git a/src/modules/messages.py b/src/modules/messages.py index 617d0cf..406c8a1 100755 --- a/src/modules/messages.py +++ b/src/modules/messages.py @@ -1,7 +1,8 @@ ERR_BANK_NOT_FOUND = "Banco no encontrado." ERR_BANK_ALREADY_EXISTS = "El banco con id {id} ya existe." ERR_BANK_NOT_CHANGED = "El banco ingresado posee los mismos datos." +ERR_FAILED_TO_UPDATE = "No se pudo actualizar la lista de bancos." ERR_SCHEMA = "Los datos ingresados no tienen la estructura adecuada. Argumentos faltantes o incorrectos: {err_args}." MSG_SUCCESS_GENERIC = "Éxito al realizar la operación." MSG_SUCCESS_SAVE = "Exito al guardar los datos." -MSG_ERROR_UNAUTHORIZED = "No autorizado." \ No newline at end of file +MSG_ERROR_UNAUTHORIZED = "No autorizado." diff --git a/src/routers/bank.py b/src/routers/bank.py index ac3add6..af23d90 100644 --- a/src/routers/bank.py +++ b/src/routers/bank.py @@ -50,7 +50,6 @@ async def update_banks_list(credentials: AdminCredentials) -> Success: ) ) return Success( - input=None, detail=Detail( payload={"ids": result}, message=messages.MSG_SUCCESS_SAVE @@ -79,7 +78,6 @@ def add_bank(bank: Bank, credentials: AdminCredentials) -> Success: ) ) return Success( - input=None, detail=Detail( payload={"id": result}, message=messages.MSG_SUCCESS_SAVE