Skip to content

Commit

Permalink
cleanup script, build.sh -> scripts/build.sh 💥
Browse files Browse the repository at this point in the history
fixes #9
  • Loading branch information
derhuerst committed Jun 29, 2021
1 parent a9a76e7 commit 2c1b231
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 22 deletions.
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ A Docker image [is available as `stadtnavi/delay-prediction-service`](https://hu

### building

[`build.sh`](build.sh) is designed to allow continuous deployments. Given the name and URL of a GTFS feed, it will generate all necessary data for `delay-prediction-service` to work.
[`scripts/build.sh`](scripts/build.sh) is designed to allow continuous deployments. Given the name and URL of a GTFS feed, it will generate all necessary data for `delay-prediction-service` to work.

The following is an example with the [VVS feed](https://www.openvvs.de/dataset/gtfs-daten), cleaned up and served by [`gtfs.mfdz.de`](https://gtfs.mfdz.de).

Expand All @@ -32,7 +32,7 @@ docker run --rm -it \
-e PGHOST -e PGUSER -e PGPASSWORD \
-e GTFS_NAME=vss -e GTFS_URL='https://gtfs.mfdz.de/VVS.filtered.gtfs.zip'
stadtnavi/delay-prediction-service \
./build.sh
./scripts/build.sh
```

### running
Expand Down Expand Up @@ -84,7 +84,7 @@ npm install

```shell
# build step
env GTFS_NAME=vss -e GTFS_URL='https://gtfs.mfdz.de/VVS.filtered.gtfs.zip' ./build.sh
env GTFS_NAME=vss -e GTFS_URL='https://gtfs.mfdz.de/VVS.filtered.gtfs.zip' ./scripts/build.sh

# run step
node index.js
Expand Down
4 changes: 2 additions & 2 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
This service needs access to the following infrastructure

- a **PostgreSQL database**, contains
- the schedules data, generated from GTFS-Static data by [`build.sh`](../build.sh)
- the schedules data, generated from GTFS-Static data by [`scripts/build.sh`](../scripts/build.sh)
- a **data directory**, contains
- *trajectories*, generated from GTFS-Static data by [`build.sh`](../build.sh)
- *trajectories*, generated from GTFS-Static data by [`scripts/build.sh`](../scripts/build.sh)
- information about the GTFS feed being used
- **City of Herrenberg's [Thingsboard](https://thingsboard.io/)** instance
- provides the realtime vehicle positions
Expand Down
18 changes: 1 addition & 17 deletions build.sh → scripts/build.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
set -e
set -o pipefail
cd $(dirname $(realpath $0))
cd $(dirname $(dirname $(realpath $0))) # change to project root

# This scripts is written for continuous deployment environments.
# Because it is idempotent and cleans up after itself (except in
Expand Down Expand Up @@ -36,12 +36,6 @@ env | grep '^PG' || true

PATH="$(realpath node_modules/.bin):$PATH"

prev_GTFS_ID=''
if [ -s 'data/gtfs_id' ]; then
prev_GTFS_ID="$(cat data/gtfs_id | tr -d '\n')"
echo "previous \$GTFS_ID: $prev_GTFS_ID"
fi

# download GTFS feed to tmp file
user_agent='https://github.com/stadtnavi/delay-prediction-service'
gtfs_file="$(mktemp)"
Expand Down Expand Up @@ -114,13 +108,3 @@ fi

# store current $GTFS_ID
echo -n "$GTFS_ID" >data/gtfs_id

if [[ ! -z "$prev_GTFS_ID" && "$GTFS_ID" != "$prev_GTFS_ID" ]]; then
# delete old database if present
psql -c "DROP DATABASE IF EXISTS $prev_GTFS_ID"

# delete old trajectories if present
if [ -d "data/trajectories-$prev_GTFS_ID" ]; then
rm -r "data/trajectories-$prev_GTFS_ID"
fi
fi
73 changes: 73 additions & 0 deletions scripts/cleanup.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/usr/bin/env node
'use strict'

// this is the counterpart to build.sh

const {join: pathJoin} = require('path')
const {Client: PostgresClient} = require('pg')
const formatSQL = require('pg-format')
const {readdir, rm} = require('fs/promises')

const GTFS_NAME = process.env.GTFS_NAME
if (!GTFS_NAME) {
console.error('missing/empty GTFS_NAME env var')
process.exit(1)
}

const DATA_DIR = pathJoin(__dirname, '..', 'data')

// keep the most n DBs called $GTFS_NAME…
const NR_OF_DBS_KEPT = 2
// keep the most n trajectories directories called trajectories-$GTFS_NAME…
const NR_OF_TRAJECTORIES_DIRS_KEPT = 2

;(async (cfg) => {
{ // delete old DBs
const prefix = `${GTFS_NAME}_`

const db = new PostgresClient({
database: 'postgres'
})
await db.connect()

const {rows} = await db.query(`
SELECT
datname
FROM pg_database
`)
const dbs = rows
.map(row => row.datname)
.filter(db => db.slice(0, prefix.length) === prefix)
.sort()

const dbsToDelete = dbs.slice(0, -NR_OF_DBS_KEPT)
for (const dbToDelete of dbsToDelete) {
console.info(`deleting database ${dbToDelete}`)
await db.query(formatSQL('DROP DATABASE %I', dbToDelete))
}

db.end()
}

{ // delete old trajectories directories
const prefix = `trajectories-${GTFS_NAME}_`

const trajectoriesDirs = (await readdir(DATA_DIR))
.filter(db => db.slice(0, prefix.length) === prefix)
.sort()
console.error('trajectoriesDirs', trajectoriesDirs)

const dirsToDelete = trajectoriesDirs.slice(0, -NR_OF_TRAJECTORIES_DIRS_KEPT)
for (const dirToDelete of dirsToDelete) {
const fullPath = pathJoin(DATA_DIR, dirToDelete)
console.info(`deleting trajectories directory ${fullPath}`)
await rm(fullPath, {recursive: true})
}
}

console.debug('done')
})()
.catch((err) => {
console.error(err)
process.exit(1)
})

0 comments on commit 2c1b231

Please sign in to comment.