add minio implementation (#2817)

* integrate minio added docker configuration and environment variables for minio integrated minio client fastify plugin instance into the talawa api server and the mercurius graphql handler exposed a new `/objects` route for streaming the files from minio server to the clients * add graphql implementation for file upload scalar * remove noUncheckedIndexAccess from tsconfig.json * add graphql implementation for deleting minio objects added the implementation for deleting minio objects associated to the postgres entities at the time of their deletion in a transaction * replace _ with - in minio docker service names
PalisadoesFoundation · Dec 31, 2024 · 9a5a8c6 · 9a5a8c6
1 parent 8bdf2f4
commit 9a5a8c6
Show file tree

Hide file tree

Showing 210 changed files with 4,756 additions and 1,317 deletions.
diff --git a/compose.yaml b/compose.yaml
@@ -2,6 +2,7 @@ name: ${COMPOSE_PROJECT_NAME:?error}
 # https://docs.docker.com/reference/compose-file/networks/
 networks:
   api:
+  minio:
   postgres:
 # https://docs.docker.com/reference/compose-file/services
 services:
@@ -20,6 +21,10 @@ services:
       target: production
     # https://docs.docker.com/reference/compose-file/services/#depends_on
     depends_on:
+      minio:
+        condition: service_healthy
+        # Should be set to false when a third party minio service could be used in the api service and the `minio` compose profile is disabled.
+        required: false
       postgres:
         condition: service_healthy
         # Should be set to false when a third party postgres service could be used in the api service the and the `postgres` compose profile is disabled.
@@ -29,13 +34,19 @@ services:
     environment:
       - API_ADMINISTRATOR_USER_EMAIL_ADDRESS=${API_ADMINISTRATOR_USER_EMAIL_ADDRESS:?error}
       - API_ADMINISTRATOR_USER_PASSWORD=${API_ADMINISTRATOR_USER_PASSWORD:?error}
+      - API_BASE_URL=${API_BASE_URL:?error}
       - API_HOST=${API_HOST:?error}
       - API_IS_APPLY_DRIZZLE_MIGRATIONS=${API_IS_APPLY_DRIZZLE_MIGRATIONS:?error}
       - API_IS_GRAPHIQL=${API_IS_GRAPHIQL:?error}
       - API_IS_PINO_PRETTY=${API_IS_PINO_PRETTY:?error}
       - API_JWT_EXPIRES_IN=${API_JWT_EXPIRES_IN:?error}
       - API_JWT_SECRET=${API_JWT_SECRET:?error}
       - API_LOG_LEVEL=${API_LOG_LEVEL:?error}
+      - API_MINIO_ACCESS_KEY=${API_MINIO_ACCESS_KEY:?error}
+      - API_MINIO_END_POINT=${API_MINIO_END_POINT:?error}
+      - API_MINIO_PORT=${API_MINIO_PORT:?error}
+      - API_MINIO_SECRET_KEY=${API_MINIO_SECRET_KEY:?error}
+      - API_MINIO_USE_SSL=${API_MINIO_USE_SSL:?error}
       - API_PORT=${API_PORT:?error}
       - API_POSTGRES_DATABASE=${API_POSTGRES_DATABASE:?error}
       - API_POSTGRES_HOST=${API_POSTGRES_HOST:?error}
@@ -64,6 +75,7 @@ services:
     # https://docs.docker.com/reference/compose-file/services/#networks
     networks:
       - api
+      - minio
       - postgres
     # https://docs.docker.com/reference/compose-file/services/#profiles
     profiles:
@@ -121,6 +133,44 @@ services:
       # - source: ./docker/admin
       #   target: /srv
       #   type: bind
+  minio:
+    # https://docs.docker.com/compose/environment-variables/
+    # https://docs.docker.com/reference/compose-file/services/#environment
+    environment:
+      - MINIO_BROWSER=${MINIO_BROWSER:?error}
+        # https://min.io/docs/minio/linux/reference/minio-server/settings/root-credentials.html#envvar.MINIO_ROOT_PASSWORD
+      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?error}
+        # https://min.io/docs/minio/linux/reference/minio-server/settings/root-credentials.html#root-user
+      - MINIO_ROOT_USER=${MINIO_ROOT_USER:?error}
+    # https://docs.docker.com/reference/compose-file/services/#command
+    command: server /data --console-address ":9001"
+    # https://docs.docker.com/reference/dockerfile/#healthcheck
+    # https://docs.docker.com/reference/compose-file/services/#healthcheck
+    healthcheck:
+      interval: 10s
+      retries: 3
+      start_interval: 1s
+      start_period: 5s
+      # https://github.com/minio/minio/issues/18373
+      # https://min.io/docs/minio/linux/reference/minio-mc/mc-ready.html
+      test: ["CMD-SHELL", "mc ready local"]
+      timeout: 10s
+    # https://docs.docker.com/reference/compose-file/services/#image
+    # https://hub.docker.com/r/minio/minio
+    image: minio/minio:RELEASE.2024-12-18T13-15-44Z
+    # https://docs.docker.com/reference/compose-file/services/#networks
+    networks:
+      - minio
+    # https://docs.docker.com/reference/compose-file/services/#profiles
+    profiles:
+      - minio
+    # https://docs.docker.com/reference/compose-file/services/#restart
+    restart: unless-stopped
+    # https://docs.docker.com/reference/compose-file/services/#volumes
+    volumes:
+      - source: minio_data
+        target: /data
+        type: volume
   postgres:
     # https://docs.docker.com/compose/environment-variables/
     # https://docs.docker.com/reference/compose-file/services/#environment
@@ -144,7 +194,7 @@ services:
       timeout: 10s
     # https://docs.docker.com/reference/compose-file/services/#image
     # https://hub.docker.com/_/postgres
-    image: postgres:17.0-alpine3.20
+    image: postgres:17.2-alpine3.21
     # https://docs.docker.com/reference/compose-file/services/#networks
     networks:
       - postgres
@@ -162,4 +212,5 @@ services:
 volumes:
   caddy_config:
   caddy_data:
+  minio_data:
   postgres_data:
diff --git a/docker/compose.development.yaml b/docker/compose.development.yaml
@@ -12,6 +12,9 @@ services:
       # https://docs.docker.com/reference/compose-file/build/#target
       target: devcontainer
     depends_on:
+      minio:
+        condition: service_healthy
+        required: true
       postgres:
         condition: service_healthy
         required: true
@@ -81,7 +84,7 @@ services:
       timeout: 10s
     # https://dbeaver.com/docs/cloudbeaver/Run-Docker-Container/
     # https://hub.docker.com/r/dbeaver/cloudbeaver
-    image: dbeaver/cloudbeaver:24.2.0
+    image: dbeaver/cloudbeaver:24.3.1
     # https://docs.docker.com/reference/compose-file/services/#networks
     networks:
       - postgres
@@ -102,6 +105,29 @@ services:
       - source: cloudbeaver_data
         target: /opt/cloudbeaver/workspace
         type: volume
+  minio:
+    # https://docs.docker.com/reference/compose-file/services/#ports
+    ports:
+      - host_ip: ${MINIO_API_MAPPED_HOST_IP:?error}
+        name: minio_api
+        published: ${MINIO_API_MAPPED_PORT:?error}
+        target: 9000
+      - host_ip: ${MINIO_CONSOLE_MAPPED_HOST_IP:?error}
+        name: minio_console
+        published: ${MINIO_CONSOLE_MAPPED_PORT:?error}
+        target: 9001
+  # https://github.com/minio/minio/issues/13025
+  minio-test:
+    # https://docs.docker.com/reference/compose-file/services/#ports
+    ports:
+      - host_ip: ${MINIO_TEST_API_MAPPED_HOST_IP:?error}
+        name: minio_api
+        published: ${MINIO_TEST_API_MAPPED_PORT:?error}
+        target: 9000
+      - host_ip: ${MINIO_TEST_CONSOLE_MAPPED_HOST_IP:?error}
+        name: minio_console
+        published: ${MINIO_TEST_CONSOLE_MAPPED_PORT:?error}
+        target: 9001
   postgres:
     # https://docs.docker.com/reference/compose-file/services/#ports
     ports:

diff --git a/docker/compose.testing.yaml b/docker/compose.testing.yaml
@@ -1,5 +1,6 @@
 # https://docs.docker.com/reference/compose-file/networks/
 networks:
+  minio_test:
   postgres_test:
 # https://docs.docker.com/reference/compose-file/services
 services:
@@ -11,23 +12,68 @@ services:
     command: pnpm run_tests
     # https://docs.docker.com/reference/compose-file/services/#depends_on
     depends_on:
+      minio-test:
+        condition: service_healthy
+        # Should be set to false when a third party minio test service could be used in the api service and the `minio_test` compose profile is disabled.
+        required: true
       postgres_test:
         condition: service_healthy
         # Should be set to false when a third party postgres test service could be used in the api service the and the `postgres_test` compose profile is disabled.
         required: true
     # https://docs.docker.com/compose/environment-variables/
     # https://docs.docker.com/reference/compose-file/services/#environment
     environment:
+      - API_MINIO_TEST_END_POINT=${API_MINIO_TEST_END_POINT:?error}
       - API_POSTGRES_TEST_HOST=${API_POSTGRES_TEST_HOST:?error}
     # https://docs.docker.com/reference/dockerfile/#healthcheck
     # https://docs.docker.com/reference/compose-file/services/#healthcheck
     healthcheck:
       disable: true
     # https://docs.docker.com/reference/compose-file/services/#networks
     networks:
+      - minio_test
       - postgres_test
     # https://docs.docker.com/reference/compose-file/services/#restart
     restart: no
+  # https://github.com/minio/minio/issues/13025
+  minio-test:
+    # https://docs.docker.com/compose/environment-variables/
+    # https://docs.docker.com/reference/compose-file/services/#environment
+    environment:
+      - MINIO_BROWSER=${MINIO_TEST_BROWSER:?error}
+        # https://min.io/docs/minio/linux/reference/minio-server/settings/root-credentials.html#envvar.MINIO_ROOT_PASSWORD
+      - MINIO_ROOT_PASSWORD=${MINIO_TEST_ROOT_PASSWORD:?error}
+        # https://min.io/docs/minio/linux/reference/minio-server/settings/root-credentials.html#root-user
+      - MINIO_ROOT_USER=${MINIO_TEST_ROOT_USER:?error}
+    # https://docs.docker.com/reference/compose-file/services/#command
+    command: server /data --console-address ":9001"
+    # https://docs.docker.com/reference/dockerfile/#healthcheck
+    # https://docs.docker.com/reference/compose-file/services/#healthcheck
+    healthcheck:
+      interval: 10s
+      retries: 3
+      start_interval: 1s
+      start_period: 5s
+      # https://github.com/minio/minio/issues/18373
+      # https://min.io/docs/minio/linux/reference/minio-mc/mc-ready.html
+      test: ["CMD-SHELL", "mc ready local"]
+      timeout: 10s
+    # https://docs.docker.com/reference/compose-file/services/#image
+    # https://hub.docker.com/r/minio/minio
+    image: minio/minio:RELEASE.2024-12-18T13-15-44Z
+    # https://docs.docker.com/reference/compose-file/services/#networks
+    networks:
+      - minio_test
+    # https://docs.docker.com/reference/compose-file/services/#profiles
+    profiles:
+      - minio_test
+    # https://docs.docker.com/reference/compose-file/services/#restart
+    restart: unless-stopped
+    # https://docs.docker.com/reference/compose-file/services/#volumes
+    volumes:
+      - source: minio_test_data
+        target: /data
+        type: volume
   postgres_test:
     # https://docs.docker.com/compose/environment-variables/
     # https://docs.docker.com/reference/compose-file/services/#environment
@@ -51,7 +97,7 @@ services:
       timeout: 10s
     # https://docs.docker.com/reference/compose-file/services/#image
     # https://hub.docker.com/_/postgres
-    image: postgres:17.0-alpine3.20
+    image: postgres:17.2-alpine3.21
     # https://docs.docker.com/reference/compose-file/services/#networks
     networks:
       - postgres_test
@@ -67,4 +113,5 @@ services:
         type: volume
 # https://docs.docker.com/reference/compose-file/volumes/
 volumes:
+  minio_test_data:
   postgres_test_data:
diff --git a/.../20241226151206_massive_grey_gargoyle.sql → ...tions/20241231123945_true_dorian_gray.sql b/.../20241226151206_massive_grey_gargoyle.sql → ...tions/20241231123945_true_dorian_gray.sql
@@ -31,10 +31,10 @@ CREATE TABLE "advertisement_attachments" (
 	"advertisement_id" uuid NOT NULL,
 	"created_at" timestamp (3) with time zone DEFAULT now() NOT NULL,
 	"creator_id" uuid,
-	"type" text NOT NULL,
+	"mime_type" text NOT NULL,
+	"name" text NOT NULL,
 	"updated_at" timestamp (3) with time zone,
-	"updater_id" uuid,
-	"uri" text NOT NULL
+	"updater_id" uuid
 );
 --> statement-breakpoint
 CREATE TABLE "advertisements" (
@@ -99,7 +99,8 @@ CREATE TABLE "chat_messages" (
 );
 --> statement-breakpoint
 CREATE TABLE "chats" (
-	"avatar_uri" text,
+	"avatar_mime_type" text,
+	"avatar_name" text,
 	"created_at" timestamp (3) with time zone DEFAULT now() NOT NULL,
 	"creator_id" uuid,
 	"description" text,
@@ -135,10 +136,10 @@ CREATE TABLE "event_attachments" (
 	"created_at" timestamp (3) with time zone DEFAULT now() NOT NULL,
 	"creator_id" uuid,
 	"event_id" uuid NOT NULL,
-	"type" text NOT NULL,
+	"mime_type" text NOT NULL,
+	"name" text NOT NULL,
 	"updated_at" timestamp (3) with time zone,
-	"updater_id" uuid,
-	"uri" text NOT NULL
+	"updater_id" uuid
 );
 --> statement-breakpoint
 CREATE TABLE "event_attendances" (
@@ -236,7 +237,8 @@ CREATE TABLE "organization_memberships" (
 --> statement-breakpoint
 CREATE TABLE "organizations" (
 	"address" text,
-	"avatar_uri" text,
+	"avatar_mime_type" text,
+	"avatar_name" text,
 	"city" text,
 	"country_code" text,
 	"created_at" timestamp (3) with time zone DEFAULT now() NOT NULL,
@@ -255,10 +257,10 @@ CREATE TABLE "post_attachments" (
 	"created_at" timestamp (3) with time zone DEFAULT now() NOT NULL,
 	"creator_id" uuid,
 	"post_id" uuid NOT NULL,
-	"type" text NOT NULL,
+	"mime_type" text NOT NULL,
+	"name" text NOT NULL,
 	"updated_at" timestamp (3) with time zone,
-	"updater_id" uuid,
-	"uri" text NOT NULL
+	"updater_id" uuid
 );
 --> statement-breakpoint
 CREATE TABLE "post_votes" (
@@ -314,7 +316,8 @@ CREATE TABLE "tags" (
 --> statement-breakpoint
 CREATE TABLE "users" (
 	"address" text,
-	"avatar_uri" text,
+	"avatar_mime_type" text,
+	"avatar_name" text,
 	"birth_date" date,
 	"city" text,
 	"country_code" text,
@@ -344,10 +347,10 @@ CREATE TABLE "users" (
 CREATE TABLE "venue_attachments" (
 	"created_at" timestamp (3) with time zone DEFAULT now() NOT NULL,
 	"creator_id" uuid,
-	"type" text NOT NULL,
+	"mime_type" text NOT NULL,
+	"name" text NOT NULL,
 	"updated_at" timestamp (3) with time zone,
 	"updater_id" uuid,
-	"uri" text NOT NULL,
 	"venue_id" uuid NOT NULL
 );
 --> statement-breakpoint