Updated event schema for auth

docker-compose.yml

@@ -379,6 +379,7 @@ services:
       service: conduktor-console
       file: conduktor.yml
 
+  # MongoDB Database Server
   mongo:
     image: mongo
     container_name: mongo
@@ -390,6 +391,7 @@ services:
     volumes:
       - mongo_data:/data/db
 
+  # Mongo Express Web Interface
   mongo-express:
     image: mongo-express
     container_name: mongo-express
@@ -401,6 +403,7 @@ services:
     ports:
       - "8085:8081"
 
+  # MongoDB Kafka Consumer
   mongo-consumer:
     build:
       context: esgf-consumer
@@ -417,7 +420,7 @@ services:
       MONGODB_COLLECTION: esgf_data_collection
       SCRIPT: "esgf_consumer/mongo_consumer.py"
 
-
+  # PostgreSQL Database for Keycloak
   postgres:
     image: postgres:15
     container_name: postgres
@@ -429,6 +432,7 @@ services:
     volumes:
       - postgres:/var/lib/postgresql/data
 
+  # Keycloak Identity and Access Management Server
   keycloak:
     image: quay.io/keycloak/keycloak:25.0.0
     container_name: keycloak

esgf-consumer/esgf_consumer/items.py

@@ -17,7 +17,7 @@ async def create_item(
     url = urljoin(str(settings.stac_server), path)
 
     logger.critical("Posting %s to %s", getattr(item.properties, "instance_id"), url)
-    result = await client.post(url, content=(item.json()), timeout=5)
+    result = await client.post(url, content=(item.model_dump_json()), timeout=5)
     if result.status_code < 300:
         logger.critical("Item posted")
 
@@ -37,7 +37,7 @@ async def update_item(
     url = urljoin(str(settings.stac_server), path)
 
     logger.critical("Updating %s to %s", getattr(item.properties, "instance_id"), url)
-    result = await client.put(url, content=(item.json()), timeout=5)
+    result = await client.put(url, content=(item.model_dump_json()), timeout=5)
     if result.status_code < 300:
         logger.critical("Item updated")
 

esgf-generator/esgf_generator/cli.py

@@ -142,16 +142,14 @@ def esgf_generator(
             click.echo(
                 f"Sending {instance.properties.instance_id}, collection: {instance.collection} to ESGF node '{node}'"
             )
-            click.echo()
-
             with httpx.Client() as client:
                 result = client.post(
                     f"http://localhost:{NODE_PORTS[node]}/{instance.collection}/items",
                     headers={"Authorization": f"Bearer {token}"},
                     content=instance.model_dump_json(),
                 )
-                click.echo()
 
+                click.echo()
                 if result.status_code == 401:
                     click.echo("You are not Authorised")
                 elif result.status_code == 403:
@@ -160,15 +158,14 @@ def esgf_generator(
                     click.echo("Item already exists")
                 elif result.status_code >= 300:
                     raise Exception(result.content)
-
                 else:
                     click.echo(instance.model_dump_json(indent=2))
-
+                    click.echo()
                     if delay:
                         click.echo("Pausing for random sub-second time")
                         time.sleep(random.random())
                         click.echo()
-                        click.echo("Done")
+                    click.echo("Done")
 
 
 @click.command()
@@ -236,6 +233,7 @@ def esgf_update(
                     content=item.model_dump_json(),
                 )
 
+            click.echo()
             if result.status_code == 401:
                 click.echo("You are not Authorised")
             elif result.status_code == 403:
@@ -246,7 +244,6 @@ def esgf_update(
                 raise Exception(result.content)
 
             else:
-                click.echo()
                 click.echo("Done")
 
 
@@ -279,8 +276,8 @@ def esgf_delete(
     """
     token = authenticate()
 
-    click.echo(f"Deleting item {item_id} in collection {collection_id}")
     click.echo()
+    click.echo(f"Deleting item {item_id} in collection {collection_id}")
 
     if publish:
         with httpx.Client() as client:
@@ -290,8 +287,8 @@ def esgf_delete(
                     headers={"Authorization": f"Bearer {token}"},
                 )
             else:
-                click.echo("Soft deleting item")
                 click.echo()
+                click.echo("Soft deleting item")
 
                 content = {"properties": {"retracted": True}}
                 result = client.patch(
@@ -300,6 +297,7 @@ def esgf_delete(
                     content=json.dumps(content),
                 )
 
+            click.echo()
             if result.status_code == 401:
                 click.echo("You are not Authorised")
             elif result.status_code == 403:
@@ -310,7 +308,6 @@ def esgf_delete(
                 raise Exception(result.content)
 
             else:
-                click.echo()
                 click.echo("Done")
 
 

esgf-generator/esgf_generator/test_cli.py

@@ -44,7 +44,7 @@ def get_item_details(result: Result) -> None:
 
 
 def check_elasticsearch_index(expected_properties: dict[str, Any]) -> None:
-    time.sleep(8)
+    time.sleep(12)
     response = es.get(
         index=f"items_{collection_id}-000001", id=f"{item_id}|{collection_id}"
     )