Aloitettu Lampi-siirtäjän muutos Node-sovelluksesta Java-sovellukseksi

.github/workflows/build-lampi-siirtaja.yml

@@ -36,5 +36,8 @@ jobs:
           IMAGE_TAG: ga-${{ github.run_number }}
         run: |
             cd lampi-siirtaja-container
+            cd lampi-siirtaja
+            mvn -B clean package
+            cd -
             docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG .
             docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG

.gitignore

@@ -17,3 +17,5 @@ cdk.out
 #Mac metadata
 .DS_Store
 dbt/package-lock.yml
+
+target

cdk/lib/ecs-stack.ts

@@ -410,6 +410,7 @@ export class EcsStack extends cdk.Stack {
           memoryLimitMiB: 6144,
           environment: {
             POSTGRES_HOST: `raportointi.db.${config.publicHostedZone}`,
+            POSTGRES_PORT: '5432',
             DB_USERNAME: 'app',
             LAMPI_S3_BUCKET: lampiSiirtajaTempS3Bucket.bucketName,
             OVARA_LAMPI_SIIRTAJA_BUCKET: lampiSiirtajaS3Bucket.bucketName,

lampi-siirtaja-container/Dockerfile

@@ -1,11 +1,11 @@
-FROM node:20-alpine3.20
+FROM amazoncorretto:21-alpine-full
 RUN apk upgrade --no-cache
 RUN apk --no-cache add bash
 
 WORKDIR /root/
 COPY ./run.sh ./run.sh
 COPY ./install.sh ./install.sh
-ADD ./lampi-siirtaja ./lampi-siirtaja
+ADD ./lampi-siirtaja/target/ovara-lampi-siirtaja-jar-with-dependencies.jar ./ovara-lampi-siirtaja.jar
 RUN \
   bash install.sh && \
   rm install.sh

lampi-siirtaja-container/install.sh

@@ -9,20 +9,20 @@ case "$(uname -m)" in
 esac
 echo $ARCHITECTURE
 
-echo "Installing needed software"
-apk --no-cache add \
-  python3 \
-  py3-pip \
-  libpq-dev \
-  g++ \
-  make
+#echo "Installing needed software"
+#apk --no-cache add \
+#  python3 \
+#  py3-pip \
+#  libpq-dev \
+#  g++ \
+#  make
 
 echo "Listing contents of /root folder"
 ls -Al /root
 
-echo "Listing contents of /root/lampi-siirtaja folder"
-ls -Al /root/lampi-siirtaja
+#echo "Listing contents of /root/lampi-siirtaja folder"
+#ls -Al /root/lampi-siirtaja
 
-echo "Installing dependencies with npm"
-cd /root/lampi-siirtaja
-npm ci
+#echo "Installing dependencies with npm"
+#cd /root/lampi-siirtaja
+#npm ci

lampi-siirtaja-container/lampi-siirtaja/pom.xml

@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>fi.oph.opintopolku.ovara</groupId>
+  <artifactId>lampi-siirtaja</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <name>lampi-siirtaja</name>
+  <!-- FIXME change it to the project's website -->
+  <url>http://www.example.com</url>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.release>21</maven.compiler.release>
+  </properties>
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>org.junit</groupId>
+        <artifactId>junit-bom</artifactId>
+        <version>5.11.0</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-s3</artifactId>
+      <version>1.12.780</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>33.4.0-jre</version>
+    </dependency>
+    <dependency>
+      <groupId>org.javatuples</groupId>
+      <artifactId>javatuples</artifactId>
+      <version>1.2</version>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.5.16</version>
+    </dependency>
+
+
+    <dependency>
+      <groupId>commons-dbutils</groupId>
+      <artifactId>commons-dbutils</artifactId>
+      <version>1.8.1</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.postgresql</groupId>
+      <artifactId>postgresql</artifactId>
+      <version>42.7.4</version>
+    </dependency>
+
+
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <!-- Optionally: parameterized tests support -->
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <finalName>ovara-lampi-siirtaja</finalName>
+    <plugins>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>3.1.1</version>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <configuration>
+              <archive>
+                <manifest>
+                  <mainClass>fi.oph.opintopolku.ovara.App</mainClass>
+                </manifest>
+              </archive>
+              <descriptorRefs>
+                <descriptorRef>jar-with-dependencies</descriptorRef>
+              </descriptorRefs>
+              <appendAssemblyId>true</appendAssemblyId>
+            </configuration>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+
+    <pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
+      <plugins>
+        <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
+        <plugin>
+          <artifactId>maven-clean-plugin</artifactId>
+          <version>3.4.0</version>
+        </plugin>
+        <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
+        <plugin>
+          <artifactId>maven-resources-plugin</artifactId>
+          <version>3.3.1</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.13.0</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>3.3.0</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-jar-plugin</artifactId>
+          <version>3.4.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-install-plugin</artifactId>
+          <version>3.1.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-deploy-plugin</artifactId>
+          <version>3.1.2</version>
+        </plugin>
+        <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
+        <plugin>
+          <artifactId>maven-site-plugin</artifactId>
+          <version>3.12.1</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-project-info-reports-plugin</artifactId>
+          <version>3.6.1</version>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+</project>

lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/App.java

@@ -0,0 +1,40 @@
+package fi.oph.opintopolku.ovara;
+
+import com.amazonaws.regions.Regions;
+import fi.oph.opintopolku.ovara.db.DatabaseToS3;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+public class App {
+
+    public static final Logger LOG = LoggerFactory.getLogger(App.class);
+
+    public static void main(String[] args) throws Exception {
+
+        Config config = new Config(
+                System.getProperty("POSTGRES_HOST"),
+                Integer.valueOf(System.getProperty("POSTGRES_PORT")),
+                System.getProperty("DB_USERNAME"),
+                System.getProperty("DB_PASSWORD"),
+                System.getProperty("OVARA_LAMPI_SIIRTAJA_BUCKET"),
+                System.getProperty("LAMPI_S3_BUCKET"),
+                Regions.EU_WEST_1.getName()
+        );
+
+        DatabaseToS3 db = new DatabaseToS3(config);
+
+        Stream.of("pub").forEach(schemaName -> {
+            try {
+                LOG.info("Haetaan scheman {} taulut", schemaName);
+                List<String> tableNames = db.getTableNames("pub");
+                LOG.info("Scheman {} taulut: ", tableNames);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+    }
+}

lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/Config.java

@@ -0,0 +1,11 @@
+package fi.oph.opintopolku.ovara;
+
+public record Config (
+    String postgresHost,
+    Integer postgresPort,
+    String postgresUser,
+    String postgresPassword,
+    String ovaraS3Bucket,
+    String lampiS3Bucket,
+    String awsRegion
+) { }

lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/DatabaseToS3.java

@@ -0,0 +1,100 @@
+package fi.oph.opintopolku.ovara.db;
+
+import fi.oph.opintopolku.ovara.Config;
+import fi.oph.opintopolku.ovara.db.domain.S3ExportResult;
+import fi.oph.opintopolku.ovara.db.domain.Table;
+import org.apache.commons.dbutils.DbUtils;
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.ResultSetHandler;
+import org.apache.commons.dbutils.handlers.BeanHandler;
+import org.apache.commons.dbutils.handlers.BeanListHandler;
+import org.javatuples.Pair;
+
+import java.sql.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class DatabaseToS3 {
+
+    private final Config config;
+
+    public DatabaseToS3(Config config) throws Exception {
+        this.config = config;
+        Class.forName("org.postgresql.Driver");
+    }
+
+    private static final String GET_TABLE_NAMES_SQL = """
+        select table_name as tablename
+        from information_schema.tables
+        where table_schema = ?;
+    """;
+
+    private static final String EXPORT_TABLE_TO_S3_SQL = """
+        select *
+        from aws_s3.query_export_to_s3(
+            'select * from ?',
+            aws_commons.create_s3_uri(
+                ?, 
+                ?, 
+                ?
+            ),
+            options := 'FORMAT CSV, HEADER TRUE' 
+        );    
+    """;
+
+    public List<String> getTableNames(String schemaName) throws Exception{
+        ResultSetHandler<List<Table>> h = new BeanListHandler<Table>(Table.class);
+
+        Connection connection = getConnection();
+        try {
+
+            QueryRunner run = new QueryRunner();
+            List<Table> tables = run.query(connection, GET_TABLE_NAMES_SQL, h, schemaName);
+
+            return tables.stream().map(Table::getTablename).toList();
+
+        } finally {
+            DbUtils.close(connection);
+        }
+
+    }
+
+    public List<Pair<String, S3ExportResult>> exportTablesToS3(String schemaName, List<String> tableNames) throws Exception {
+        List<Pair<String, S3ExportResult>> results = tableNames.stream().map(tableName -> {
+            try {
+                return exportTableToS3(schemaName, tableName);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }).toList();
+        return results;
+    }
+
+    private Pair<String, S3ExportResult> exportTableToS3(String schemaName, String tableName) throws Exception {
+        ResultSetHandler<S3ExportResult> h = new BeanHandler<S3ExportResult>(S3ExportResult.class);
+
+        Connection connection = getConnection();
+        try {
+
+            QueryRunner run = new QueryRunner();
+            S3ExportResult s3ExportResult = run.query(
+                    connection,
+                    EXPORT_TABLE_TO_S3_SQL,
+                    h,
+                    String.format("%s.%s", schemaName, tableName),
+                    config.ovaraS3Bucket(),
+                    String.format("%s.csv", tableName),
+                    config.awsRegion());
+
+            return new Pair<>(tableName, s3ExportResult);
+
+        } finally {
+            DbUtils.close(connection);
+        }
+    }
+
+    private Connection getConnection() throws Exception {
+        return DriverManager.getConnection(String.format("jdbc:postgresql://%s:%s/ovara", config.postgresHost(), config.postgresPort()), config.postgresUser(), config.postgresPassword());
+    }
+
+}