diff --git a/README.md b/README.md index 8e6648e..16235a2 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Welcome to the Diaspora Event Fabric Service documentation! Our goal in the Dias - [Diaspora Action Provider](https://diaspora-action-provider.ml22sevubfnks.us-east-1.cs.amazonlightsail.com) ![7-day response time](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fhaochenpan%2Fdiaspora-uptime-monitor%2FHEAD%2Fapi%2Fdiaspora-action-provider%2Fresponse-time-week.json) ![7-day uptime](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fhaochenpan%2Fdiaspora-uptime-monitor%2FHEAD%2Fapi%2Fdiaspora-action-provider%2Fuptime-week.json) - - [Diaspora Admin Console](http://100.27.155.7/) + - [Diaspora Kafka Console](http://100.27.155.7/) ![7-day response time](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fhaochenpan%2Fdiaspora-uptime-monitor%2FHEAD%2Fapi%2Fdiaspora-kafka-console%2Fresponse-time-week.json) ![7-day uptime](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fhaochenpan%2Fdiaspora-uptime-monitor%2FHEAD%2Fapi%2Fdiaspora-kafka-console%2Fuptime-week.json) - [Diaspora Grafana Console](http://100.29.137.186/) diff --git a/docs/kafka-console/intro.md b/docs/kafka-console/intro.md index bde20e1..adf61ae 100644 --- a/docs/kafka-console/intro.md +++ b/docs/kafka-console/intro.md @@ -14,4 +14,83 @@ This script sets up a Docker environment, installs Docker Compose, and configure ### Startup Script -See [startup_script.sh](https://github.com/haochenpan/diaspora-service/blob/main/kafka_console/startup_script.sh) +```bash +#!/bin/bash + +# Startup script for Lightsail admin console at http://100.27.155.7 +# adapted from https://raw.githubusercontent.com/mikegcoleman/todo/master/lightsail-compose.sh +# and https://stackoverflow.com/a/33370375 + +# Set the front-end for apt to noninteractive to avoid prompts during package installation +export DEBIAN_FRONTEND=noninteractive + +# Update package list and perform a dist-upgrade with forced configuration file overwrite +# apt-get update && apt-get -o Dpkg::Options::="--force-confold" dist-upgrade -q -y --force-yes +apt-get update && apt-get -o Dpkg::Options::="--force-confold" dist-upgrade -q -y --allow-downgrades + +# Install Docker +curl -sSL https://get.docker.com | sh + +# Add the ubuntu user to the docker group +usermod -aG docker ubuntu + +# Install Docker Compose +curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose +chmod +x /usr/local/bin/docker-compose + +# Create a directory for Docker configuration +mkdir /srv/docker + +# Create a Docker Compose configuration file +cat < /srv/docker/docker-compose.yml +version: '3' +services: + kafka-ui: + container_name: kafka-ui + image: provectuslabs/kafka-ui:latest + ports: + - "8080:8080" + environment: + - DYNAMIC_CONFIG_ENABLED=true + - AUTH_TYPE=LOGIN_FORM + - SPRING_SECURITY_USER_NAME=******** + - SPRING_SECURITY_USER_PASSWORD=******** + - AWS_ACCESS_KEY_ID=******************** + - AWS_SECRET_ACCESS_KEY=**************************** + volumes: + - /srv/docker/config.yml:/etc/kafkaui/dynamic_config.yaml +EOF + +# Create a Kafka UI configuration file +cat < /srv/docker/config.yml +kafka: + clusters: + - name: diaspora + bootstrapServers: b-1-public.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9198,b-2-public.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9198 + properties: + security.protocol: SASL_SSL + sasl.mechanism: AWS_MSK_IAM + sasl.client.callback.handler.class: software.amazon.msk.auth.iam.IAMClientCallbackHandler + sasl.jaas.config: software.amazon.msk.auth.iam.IAMLoginModule required awsProfileName="default"; +EOF + +# Download the systemd service file for Docker Compose application +curl -o /etc/systemd/system/docker-compose-app.service https://raw.githubusercontent.com/zahedahmed/todo/master/docker-compose-app.service + +# Enable the Docker Compose application service +systemctl enable docker-compose-app + +# Start the Docker Compose application in detached mode +docker-compose -f /srv/docker/docker-compose.yml up -d + +# Redirect port 80 to port 8080 using iptables +iptables -t nat -A PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 8080 + +# Install iptables-persistent to save iptables rules +apt-get install -y iptables-persistent +netfilter-persistent save + +# Schedule a reboot every 3 days at 4:30 AM +(crontab -l 2>/dev/null; echo "30 4 */3 * * /sbin/reboot") | crontab - + +``` diff --git a/kafka-console/startup_script.sh b/kafka-console/startup_script.sh deleted file mode 100644 index 6016b3a..0000000 --- a/kafka-console/startup_script.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -# Startup script for Lightsail admin console at http://100.27.155.7 -# adapted from https://raw.githubusercontent.com/mikegcoleman/todo/master/lightsail-compose.sh -# and https://stackoverflow.com/a/33370375 - -# Set the front-end for apt to noninteractive to avoid prompts during package installation -export DEBIAN_FRONTEND=noninteractive - -# Update package list and perform a dist-upgrade with forced configuration file overwrite -# apt-get update && apt-get -o Dpkg::Options::="--force-confold" dist-upgrade -q -y --force-yes -apt-get update && apt-get -o Dpkg::Options::="--force-confold" dist-upgrade -q -y --allow-downgrades - -# Install Docker -curl -sSL https://get.docker.com | sh - -# Add the ubuntu user to the docker group -usermod -aG docker ubuntu - -# Install Docker Compose -curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose -chmod +x /usr/local/bin/docker-compose - -# Create a directory for Docker configuration -mkdir /srv/docker - -# Create a Docker Compose configuration file -cat < /srv/docker/docker-compose.yml -version: '3' -services: - kafka-ui: - container_name: kafka-ui - image: provectuslabs/kafka-ui:latest - ports: - - "8080:8080" - environment: - - DYNAMIC_CONFIG_ENABLED=true - - AUTH_TYPE=LOGIN_FORM - - SPRING_SECURITY_USER_NAME=******** - - SPRING_SECURITY_USER_PASSWORD=******** - - AWS_ACCESS_KEY_ID=******************** - - AWS_SECRET_ACCESS_KEY=**************************** - volumes: - - /srv/docker/config.yml:/etc/kafkaui/dynamic_config.yaml -EOF - -# Create a Kafka UI configuration file -cat < /srv/docker/config.yml -kafka: - clusters: - - name: diaspora - bootstrapServers: b-1-public.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9198,b-2-public.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9198 - properties: - security.protocol: SASL_SSL - sasl.mechanism: AWS_MSK_IAM - sasl.client.callback.handler.class: software.amazon.msk.auth.iam.IAMClientCallbackHandler - sasl.jaas.config: software.amazon.msk.auth.iam.IAMLoginModule required awsProfileName="default"; -EOF - -# Download the systemd service file for Docker Compose application -curl -o /etc/systemd/system/docker-compose-app.service https://raw.githubusercontent.com/zahedahmed/todo/master/docker-compose-app.service - -# Enable the Docker Compose application service -systemctl enable docker-compose-app - -# Start the Docker Compose application in detached mode -docker-compose -f /srv/docker/docker-compose.yml up -d - -# Redirect port 80 to port 8080 using iptables -iptables -t nat -A PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 8080 - -# Install iptables-persistent to save iptables rules -apt-get install -y iptables-persistent -netfilter-persistent save - -# Schedule a reboot every 3 hours using cron -(crontab -l 2>/dev/null; echo "0 */3 * * * /sbin/reboot") | crontab - diff --git a/pyproject.toml b/pyproject.toml index 191e0c0..0938c6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "diaspora-service" -version = "0.3.1" +version = "0.3.3" dependencies = [ "aws-msk-iam-sasl-signer-python", "boto3", diff --git a/web_service/utils.py b/web_service/utils.py index 9c99604..33e1397 100644 --- a/web_service/utils.py +++ b/web_service/utils.py @@ -1353,6 +1353,6 @@ def list_unused_topics(self): 'us-east-1', 'diaspora', '0b48e9a3-c32b-4783-9993-30798cdda646-9', - 'b-1.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9098', + 'b-1-public.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9198,b-2-public.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9198', 'b-1-public.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9198,b-2-public.diaspora.fy49oq.c9.kafka.us-east-1.amazonaws.com:9198', )