diff --git a/.github/workflows/build-lampi-siirtaja.yml b/.github/workflows/build-lampi-siirtaja.yml index f03ce76..10c2ca1 100644 --- a/.github/workflows/build-lampi-siirtaja.yml +++ b/.github/workflows/build-lampi-siirtaja.yml @@ -18,6 +18,24 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'corretto' + + - uses: szenius/set-timezone@v1.0 + with: + timezoneLinux: "Europe/Helsinki" + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -36,5 +54,8 @@ jobs: IMAGE_TAG: ga-${{ github.run_number }} run: | cd lampi-siirtaja-container + cd lampi-siirtaja + mvn -B clean package + cd - docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG diff --git a/.gitignore b/.gitignore index 5861f04..656cf14 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ cdk.out #Mac metadata .DS_Store dbt/package-lock.yml + +target diff --git a/.java-version b/.java-version new file mode 100644 index 0000000..5f39e91 --- /dev/null +++ b/.java-version @@ -0,0 +1 @@ +21.0 diff --git a/cdk/lib/ecs-stack.ts b/cdk/lib/ecs-stack.ts index 8899666..a11b8fd 100644 --- a/cdk/lib/ecs-stack.ts +++ b/cdk/lib/ecs-stack.ts @@ -410,6 +410,7 @@ export class EcsStack extends cdk.Stack { memoryLimitMiB: 6144, environment: { POSTGRES_HOST: `raportointi.db.${config.publicHostedZone}`, + POSTGRES_PORT: '5432', DB_USERNAME: 'app', LAMPI_S3_BUCKET: lampiSiirtajaTempS3Bucket.bucketName, OVARA_LAMPI_SIIRTAJA_BUCKET: lampiSiirtajaS3Bucket.bucketName, diff --git a/lampi-siirtaja-container/Dockerfile b/lampi-siirtaja-container/Dockerfile index e94c7c4..7f24aea 100644 --- a/lampi-siirtaja-container/Dockerfile +++ b/lampi-siirtaja-container/Dockerfile @@ -1,11 +1,11 @@ -FROM node:20-alpine3.20 +FROM amazoncorretto:21-alpine-full RUN apk upgrade --no-cache RUN apk --no-cache add bash WORKDIR /root/ COPY ./run.sh ./run.sh COPY ./install.sh ./install.sh -ADD ./lampi-siirtaja ./lampi-siirtaja +ADD ./lampi-siirtaja/target/ovara-lampi-siirtaja-jar-with-dependencies.jar ./ovara-lampi-siirtaja.jar RUN \ bash install.sh && \ rm install.sh diff --git a/lampi-siirtaja-container/install.sh b/lampi-siirtaja-container/install.sh index e79835a..14fb67b 100644 --- a/lampi-siirtaja-container/install.sh +++ b/lampi-siirtaja-container/install.sh @@ -9,20 +9,20 @@ case "$(uname -m)" in esac echo $ARCHITECTURE -echo "Installing needed software" -apk --no-cache add \ - python3 \ - py3-pip \ - libpq-dev \ - g++ \ - make +#echo "Installing needed software" +#apk --no-cache add \ +# python3 \ +# py3-pip \ +# libpq-dev \ +# g++ \ +# make echo "Listing contents of /root folder" ls -Al /root -echo "Listing contents of /root/lampi-siirtaja folder" -ls -Al /root/lampi-siirtaja +#echo "Listing contents of /root/lampi-siirtaja folder" +#ls -Al /root/lampi-siirtaja -echo "Installing dependencies with npm" -cd /root/lampi-siirtaja -npm ci +#echo "Installing dependencies with npm" +#cd /root/lampi-siirtaja +#npm ci diff --git a/lampi-siirtaja-container/lampi-siirtaja/dist/run.js.map b/lampi-siirtaja-container/lampi-siirtaja-node/dist/run.js.map similarity index 100% rename from lampi-siirtaja-container/lampi-siirtaja/dist/run.js.map rename to lampi-siirtaja-container/lampi-siirtaja-node/dist/run.js.map diff --git a/lampi-siirtaja-container/lampi-siirtaja/package-lock.json b/lampi-siirtaja-container/lampi-siirtaja-node/package-lock.json similarity index 100% rename from lampi-siirtaja-container/lampi-siirtaja/package-lock.json rename to lampi-siirtaja-container/lampi-siirtaja-node/package-lock.json diff --git a/lampi-siirtaja-container/lampi-siirtaja/package.json b/lampi-siirtaja-container/lampi-siirtaja-node/package.json similarity index 100% rename from lampi-siirtaja-container/lampi-siirtaja/package.json rename to lampi-siirtaja-container/lampi-siirtaja-node/package.json diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/run.ts b/lampi-siirtaja-container/lampi-siirtaja-node/src/run.ts similarity index 100% rename from lampi-siirtaja-container/lampi-siirtaja/src/run.ts rename to lampi-siirtaja-container/lampi-siirtaja-node/src/run.ts diff --git a/lampi-siirtaja-container/lampi-siirtaja/tsconfig.json b/lampi-siirtaja-container/lampi-siirtaja-node/tsconfig.json similarity index 100% rename from lampi-siirtaja-container/lampi-siirtaja/tsconfig.json rename to lampi-siirtaja-container/lampi-siirtaja-node/tsconfig.json diff --git a/lampi-siirtaja-container/lampi-siirtaja/pom.xml b/lampi-siirtaja-container/lampi-siirtaja/pom.xml new file mode 100644 index 0000000..766523f --- /dev/null +++ b/lampi-siirtaja-container/lampi-siirtaja/pom.xml @@ -0,0 +1,154 @@ + + + 4.0.0 + + fi.oph.opintopolku.ovara + lampi-siirtaja + 1.0-SNAPSHOT + + lampi-siirtaja + + http://www.example.com + + + UTF-8 + 21 + + + + + + org.junit + junit-bom + 5.11.0 + pom + import + + + + + + + com.amazonaws + aws-java-sdk-s3 + 1.12.780 + + + com.google.guava + guava + 33.4.0-jre + + + org.javatuples + javatuples + 1.2 + + + ch.qos.logback + logback-classic + 1.5.16 + + + + + commons-dbutils + commons-dbutils + 1.8.1 + + + + org.postgresql + postgresql + 42.7.4 + + + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + + ovara-lampi-siirtaja + + + maven-assembly-plugin + 3.1.1 + + + make-assembly + package + + + + fi.oph.opintopolku.ovara.App + + + + jar-with-dependencies + + true + + + single + + + + + + + + + + + maven-clean-plugin + 3.4.0 + + + + maven-resources-plugin + 3.3.1 + + + maven-compiler-plugin + 3.13.0 + + + maven-surefire-plugin + 3.3.0 + + + maven-jar-plugin + 3.4.2 + + + maven-install-plugin + 3.1.2 + + + maven-deploy-plugin + 3.1.2 + + + + maven-site-plugin + 3.12.1 + + + maven-project-info-reports-plugin + 3.6.1 + + + + + diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/App.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/App.java new file mode 100644 index 0000000..19e1307 --- /dev/null +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/App.java @@ -0,0 +1,40 @@ +package fi.oph.opintopolku.ovara; + +import com.amazonaws.regions.Regions; +import fi.oph.opintopolku.ovara.db.DatabaseToS3; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Stream; + +public class App { + + public static final Logger LOG = LoggerFactory.getLogger(App.class); + + public static void main(String[] args) throws Exception { + + Config config = new Config( + System.getenv("POSTGRES_HOST"), + Integer.valueOf(System.getenv("POSTGRES_PORT")), + System.getenv("DB_USERNAME"), + System.getenv("DB_PASSWORD"), + System.getenv("OVARA_LAMPI_SIIRTAJA_BUCKET"), + System.getenv("LAMPI_S3_BUCKET"), + Regions.EU_WEST_1.getName() + ); + + DatabaseToS3 db = new DatabaseToS3(config); + + Stream.of("pub").forEach(schemaName -> { + try { + LOG.info("Haetaan scheman {} taulut", schemaName); + List tableNames = db.getTableNames("pub"); + LOG.info("Scheman {} taulut: ", tableNames); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + } +} diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/Config.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/Config.java new file mode 100644 index 0000000..e71461f --- /dev/null +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/Config.java @@ -0,0 +1,11 @@ +package fi.oph.opintopolku.ovara; + +public record Config ( + String postgresHost, + Integer postgresPort, + String postgresUser, + String postgresPassword, + String ovaraS3Bucket, + String lampiS3Bucket, + String awsRegion +) { } diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/DatabaseToS3.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/DatabaseToS3.java new file mode 100644 index 0000000..0723a66 --- /dev/null +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/DatabaseToS3.java @@ -0,0 +1,100 @@ +package fi.oph.opintopolku.ovara.db; + +import fi.oph.opintopolku.ovara.Config; +import fi.oph.opintopolku.ovara.db.domain.S3ExportResult; +import fi.oph.opintopolku.ovara.db.domain.Table; +import org.apache.commons.dbutils.DbUtils; +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.dbutils.ResultSetHandler; +import org.apache.commons.dbutils.handlers.BeanHandler; +import org.apache.commons.dbutils.handlers.BeanListHandler; +import org.javatuples.Pair; + +import java.sql.*; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabaseToS3 { + + private final Config config; + + public DatabaseToS3(Config config) throws Exception { + this.config = config; + Class.forName("org.postgresql.Driver"); + } + + private static final String GET_TABLE_NAMES_SQL = """ + select table_name as tablename + from information_schema.tables + where table_schema = ?; + """; + + private static final String EXPORT_TABLE_TO_S3_SQL = """ + select * + from aws_s3.query_export_to_s3( + 'select * from ?', + aws_commons.create_s3_uri( + ?, + ?, + ? + ), + options := 'FORMAT CSV, HEADER TRUE' + ); + """; + + public List getTableNames(String schemaName) throws Exception{ + ResultSetHandler> h = new BeanListHandler(Table.class); + + Connection connection = getConnection(); + try { + + QueryRunner run = new QueryRunner(); + List
tables = run.query(connection, GET_TABLE_NAMES_SQL, h, schemaName); + + return tables.stream().map(Table::getTablename).toList(); + + } finally { + DbUtils.close(connection); + } + + } + + public List> exportTablesToS3(String schemaName, List tableNames) throws Exception { + List> results = tableNames.stream().map(tableName -> { + try { + return exportTableToS3(schemaName, tableName); + } catch (Exception e) { + throw new RuntimeException(e); + } + }).toList(); + return results; + } + + private Pair exportTableToS3(String schemaName, String tableName) throws Exception { + ResultSetHandler h = new BeanHandler(S3ExportResult.class); + + Connection connection = getConnection(); + try { + + QueryRunner run = new QueryRunner(); + S3ExportResult s3ExportResult = run.query( + connection, + EXPORT_TABLE_TO_S3_SQL, + h, + String.format("%s.%s", schemaName, tableName), + config.ovaraS3Bucket(), + String.format("%s.csv", tableName), + config.awsRegion()); + + return new Pair<>(tableName, s3ExportResult); + + } finally { + DbUtils.close(connection); + } + } + + private Connection getConnection() throws Exception { + return DriverManager.getConnection(String.format("jdbc:postgresql://%s:%s/ovara", config.postgresHost(), config.postgresPort()), config.postgresUser(), config.postgresPassword()); + } + +} diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/domain/S3ExportResult.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/domain/S3ExportResult.java new file mode 100644 index 0000000..0341ab2 --- /dev/null +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/domain/S3ExportResult.java @@ -0,0 +1,40 @@ +package fi.oph.opintopolku.ovara.db.domain; + +public class S3ExportResult { + private Long rows_uploaded; + private Long files_uploaded; + private Long bytes_uploaded; + + public Long getRows_uploaded() { + return rows_uploaded; + } + + public void setRows_uploaded(Long rows_uploaded) { + this.rows_uploaded = rows_uploaded; + } + + public Long getFiles_uploaded() { + return files_uploaded; + } + + public void setFiles_uploaded(Long files_uploaded) { + this.files_uploaded = files_uploaded; + } + + public Long getBytes_uploaded() { + return bytes_uploaded; + } + + public void setBytes_uploaded(Long bytes_uploaded) { + this.bytes_uploaded = bytes_uploaded; + } + + @Override + public String toString() { + return "S3ExportResult{" + + "rows_uploaded=" + rows_uploaded + + ", files_uploaded=" + files_uploaded + + ", bytes_uploaded=" + bytes_uploaded + + '}'; + } +} diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/domain/Table.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/domain/Table.java new file mode 100644 index 0000000..155bec3 --- /dev/null +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/domain/Table.java @@ -0,0 +1,14 @@ +package fi.oph.opintopolku.ovara.db.domain; + +public class Table { + + private String tablename; + + public String getTablename() { + return tablename; + } + + public void setTablename(String tablename) { + this.tablename = tablename; + } +} diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/io/MultiInputStream.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/io/MultiInputStream.java new file mode 100644 index 0000000..ac0266e --- /dev/null +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/io/MultiInputStream.java @@ -0,0 +1,125 @@ +package fi.oph.opintopolku.ovara.io; + +import com.amazonaws.services.s3.model.S3ObjectInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Enumeration; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class MultiInputStream extends InputStream { + private final Enumeration> e; + private InputStream in; + + public MultiInputStream(Enumeration> e) throws IOException { + this.e = e; + peekNextStream(); + } + + final void nextStream() throws IOException { + if (in != null) { + in.close(); + } + peekNextStream(); + } + + private void peekNextStream() throws IOException { + if (e.hasMoreElements()) { + Future f = e.nextElement(); + try { + in = f.get(); + } catch (ExecutionException | InterruptedException e) { + throw new IOException(e); + } + if (in == null) + throw new NullPointerException(); + } else { + in = null; + } + } + + @Override + public int available() throws IOException { + if (in == null) { + return 0; // no way to signal EOF from available() + } + return in.available(); + } + + @Override + public int read() throws IOException { + while (in != null) { + int c = in.read(); + if (c != -1) { + return c; + } + nextStream(); + } + return -1; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (in == null) { + return -1; + } else if (b == null) { + throw new NullPointerException(); + } + Objects.checkFromIndexSize(off, len, b.length); + if (len == 0) { + return 0; + } + do { + int n = in.read(b, off, len); + if (n > 0) { + return n; + } + nextStream(); + } while (in != null); + return -1; + } + + @Override + public void close() throws IOException { + IOException ioe = null; + while (in != null) { + try { + in.close(); + } catch (IOException e) { + if (ioe == null) { + ioe = e; + } else { + ioe.addSuppressed(e); + } + } + peekNextStream(); + } + if (ioe != null) { + throw ioe; + } + } + + @Override + public long transferTo(OutputStream out) throws IOException { + Objects.requireNonNull(out, "out"); + if (getClass() == MultiInputStream.class) { + long transferred = 0; + while (in != null) { + if (transferred < Long.MAX_VALUE) { + try { + transferred = Math.addExact(transferred, in.transferTo(out)); + } catch (ArithmeticException ignore) { + return Long.MAX_VALUE; + } + } + nextStream(); + } + return transferred; + } else { + return super.transferTo(out); + } + } +} diff --git a/lampi-siirtaja-container/run.sh b/lampi-siirtaja-container/run.sh index 750672f..cf03490 100755 --- a/lampi-siirtaja-container/run.sh +++ b/lampi-siirtaja-container/run.sh @@ -6,8 +6,8 @@ echo "Running Lampi-siirtäjä..." start=$(date +%s) -cd /root/lampi-siirtaja -npm start +cd /root +java -jar ovara-lampi-siirtaja.jar echo "Ajon kesto `expr $(date +%s) - ${start}` s"