Skip to content

Commit

Permalink
[RFR-1213] Fix files size changes between attachment uploads
Browse files Browse the repository at this point in the history
  • Loading branch information
hb0 committed Feb 12, 2025
1 parent faaf788 commit 8b767ea
Show file tree
Hide file tree
Showing 12 changed files with 537 additions and 66 deletions.
414 changes: 414 additions & 0 deletions persistence/schemas/de.cyface.persistence.Database/20.json

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2019-2023 Cyface GmbH
* Copyright 2019-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand Down Expand Up @@ -60,7 +60,7 @@ import java.nio.ByteBuffer
* should be the same as they were in that version to really test the migration as it would happen in real.
*
* @author Armin Schnabel
* @version 2.0.0
* @version 2.1.0
* @since 4.0.0
*/
@RunWith(AndroidJUnit4::class)
Expand Down Expand Up @@ -101,6 +101,7 @@ class DatabaseMigratorTest {
DatabaseMigrator.MIGRATION_16_17,
migrator!!.MIGRATION_17_18,
DatabaseMigrator.MIGRATION_18_19,
DatabaseMigrator.MIGRATION_19_20,
)
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023 Cyface GmbH
* Copyright 2023-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand Down Expand Up @@ -37,7 +37,7 @@ import kotlin.io.path.Path
* Utility methods used in different [de.cyface.persistence.dao] `androidTest`s.
*
* @author Armin Schnabel
* @version 1.1.0
* @version 1.1.1
* @since 7.5.0
*/
class TestUtils {
Expand All @@ -59,7 +59,8 @@ class TestUtils {
Modality.BICYCLE,
DefaultPersistenceLayer.PERSISTENCE_FILE_FORMAT_VERSION,
0.0,
1000L
1000L,
0,
)
}

Expand Down
10 changes: 6 additions & 4 deletions persistence/src/main/kotlin/de/cyface/persistence/Database.kt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023 Cyface GmbH
* Copyright 2023-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand Down Expand Up @@ -50,7 +50,7 @@ import de.cyface.persistence.model.Pressure
* For this, see: https://developer.android.com/training/data-storage/room/async-queries
*
* @author Armin Schnabel
* @version 1.1.0
* @version 1.1.1
* @since 7.5.0
*/
@androidx.room.Database(
Expand All @@ -64,7 +64,8 @@ import de.cyface.persistence.model.Pressure
],
// version 18 imported data from `v6.1` database into `measures.17` and migrated `measures` to Room
// version 19 adds the attachments table
version = 19
// version 20 adds filesSize to the measurement table [RFR-1213]
version = 20
//autoMigrations = [] // test this feature on the next version change
)
@TypeConverters(PathTypeConverter::class)
Expand Down Expand Up @@ -133,7 +134,8 @@ abstract class Database : RoomDatabase() {
DatabaseMigrator.MIGRATION_15_16,
DatabaseMigrator.MIGRATION_16_17,
migrator.MIGRATION_17_18,
DatabaseMigrator.MIGRATION_18_19
DatabaseMigrator.MIGRATION_18_19,
DatabaseMigrator.MIGRATION_19_20,
)
.build()
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023 Cyface GmbH
* Copyright 2023-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand Down Expand Up @@ -51,7 +51,7 @@ import java.nio.ByteBuffer
* provide a Migration object to the builder*!
*
* @author Armin Schnabel
* @version 1.1.0
* @version 1.2.0
* @since 7.5.0
* @property context The `Context` required to import data from a secondary data source.
*/
Expand Down Expand Up @@ -92,6 +92,17 @@ class DatabaseMigrator(val context: Context) {
val MIGRATION_9_10: Migration = migrationFrom9To10()

companion object {
/**
* Adds the [de.cyface.persistence.model.Measurement.filesSize] column.
*/
val MIGRATION_19_20 = object : Migration(19, 20) {
override fun migrate(database: SupportSQLiteDatabase) {
// Add the new filesSize column with default value 0 for existing rows
// This should be fine as we don't upload attachments in production, yet.
database.execSQL("ALTER TABLE Measurement ADD COLUMN filesSize INTEGER NOT NULL DEFAULT 0")
}
}

/**
* Adds the [de.cyface.persistence.model.Attachment] table.
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2024 Cyface GmbH
* Copyright 2017-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand Down Expand Up @@ -65,6 +65,8 @@ import kotlin.math.max
*
* @author Klemens Muthmann
* @author Armin Schnabel
* @version 19.1.1
* @since 2.0.0
* @property persistenceBehaviour The [PersistenceBehaviour] defines how the `Persistence` layer works.
* We need this behaviour to differentiate if the [DefaultPersistenceLayer] is used for live capturing
* and or to load existing data.
Expand Down Expand Up @@ -179,7 +181,8 @@ class DefaultPersistenceLayer<B : PersistenceBehaviour?> : PersistenceLayer<B> {
modality,
PERSISTENCE_FILE_FORMAT_VERSION,
0.0,
timestamp
timestamp,
0,
)
runBlocking {
val measurementId = withContext(scope.coroutineContext) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2023 Cyface GmbH
* Copyright 2017-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand All @@ -22,11 +22,11 @@ import android.net.Uri
import de.cyface.persistence.model.GeoLocation

/**
* This class represents the table containing all the [de.cyface.persistence.model.Measurement]s currently
* stored on this device.
* This class represents the table containing all the [de.cyface.persistence.model.Measurement]s
* currently stored on this device.
*
* @author Armin Schnabel
* @version 5.0.1
* @version 5.1.0
* @since 1.0.0
*/
class MeasurementTable : AbstractCyfaceTable(URI_PATH) {
Expand Down Expand Up @@ -60,6 +60,11 @@ class MeasurementTable : AbstractCyfaceTable(URI_PATH) {
*/
const val COLUMN_DISTANCE = "distance"

/**
* Column name for the number of bytes of all attachments collected for this measurement.
*/
const val COLUMN_FILES_SIZE = "filesSize"

/**
* Returns the URI which identifies the table represented by this class.
*
Expand All @@ -75,7 +80,12 @@ class MeasurementTable : AbstractCyfaceTable(URI_PATH) {

override val databaseTableColumns: Array<String>
get() = arrayOf(
BaseColumns.ID, COLUMN_STATUS, COLUMN_MODALITY,
COLUMN_PERSISTENCE_FILE_FORMAT_VERSION, COLUMN_DISTANCE, BaseColumns.TIMESTAMP
BaseColumns.ID,
COLUMN_STATUS,
COLUMN_MODALITY,
COLUMN_PERSISTENCE_FILE_FORMAT_VERSION,
COLUMN_DISTANCE,
BaseColumns.TIMESTAMP,
COLUMN_FILES_SIZE,
)
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023-2024 Cyface GmbH
* Copyright 2023-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand Down Expand Up @@ -31,18 +31,20 @@ import de.cyface.protos.model.File.FileType
* Data access object which provides the API to interact with the [Attachment] database table.
*
* @author Armin Schnabel
* @version 1.0.1
* @since 7.10.0
*/
@Dao
interface AttachmentDao {
/**
* Inserts a reference to an [Attachment] into the database.
*
* @param attachment The [Attachment] to create the reference for.
* @return The identifier of the [Attachment] reference in the database.
*/
@Insert
suspend fun insert(attachment: Attachment): Long

// This is just a workaround until we succeed to port JpegSafer to Kotlin. We tried this before
// but image capturing got flaky afterwards so we reverted it to do the refactoring separately.
// It's hard to handle suspend functions from Java, so we just have this non-suspend sibling.
@Insert
fun insertJava(attachment: Attachment): Long

@Insert
suspend fun insertAll(vararg attachments: Attachment)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023 Cyface GmbH
* Copyright 2023-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand Down Expand Up @@ -31,12 +31,17 @@ import kotlinx.coroutines.flow.Flow
* Data access object which provides the API to interact with the [Measurement] database table.
*
* @author Armin Schnabel
* @version 2.0.0
* @version 2.1.0
* @since 7.5.0
*/
@Dao
interface MeasurementDao {

/**
* Inserts a reference to a [Measurement] into the database.
*
* @param measurement The [Measurement] to create the reference for.
* @return The identifier of the [Measurement] reference in the database.
*/
@Insert
suspend fun insert(measurement: Measurement): Long

Expand All @@ -47,14 +52,18 @@ interface MeasurementDao {
* Loads all measurements which are not in the [MeasurementStatus.OPEN] or
* [MeasurementStatus.PAUSED] state starting with the newest measurement.
*/
@Query("SELECT * FROM ${MeasurementTable.URI_PATH} WHERE ${MeasurementTable.COLUMN_STATUS} NOT IN ('OPEN', 'PAUSED') ORDER BY ${BaseColumns.ID} DESC")
@Query("SELECT * FROM ${MeasurementTable.URI_PATH} " +
"WHERE ${MeasurementTable.COLUMN_STATUS} NOT IN ('OPEN', 'PAUSED') " +
"ORDER BY ${BaseColumns.ID} DESC")
suspend fun loadAllCompleted(): List<Measurement>

/**
* Loads and observes all measurements which are not in the [MeasurementStatus.OPEN] or
* [MeasurementStatus.PAUSED] state starting with the newest measurement.
*/
@Query("SELECT * FROM ${MeasurementTable.URI_PATH} WHERE ${MeasurementTable.COLUMN_STATUS} NOT IN ('OPEN', 'PAUSED') ORDER BY ${BaseColumns.ID} DESC")
@Query("SELECT * FROM ${MeasurementTable.URI_PATH} " +
"WHERE ${MeasurementTable.COLUMN_STATUS} NOT IN ('OPEN', 'PAUSED') " +
"ORDER BY ${BaseColumns.ID} DESC")
fun observeAllCompleted(): Flow<List<Measurement>>

@Query("SELECT * FROM ${MeasurementTable.URI_PATH} WHERE ${BaseColumns.ID} = :id")
Expand All @@ -76,18 +85,44 @@ interface MeasurementDao {
//@Update
//fun update(vararg measurements: Measurement)

@Query("UPDATE ${MeasurementTable.URI_PATH} SET ${MeasurementTable.COLUMN_PERSISTENCE_FILE_FORMAT_VERSION} = :fileFormatVersion WHERE ${BaseColumns.ID} = :id")
@Query("UPDATE ${MeasurementTable.URI_PATH} " +
"SET ${MeasurementTable.COLUMN_PERSISTENCE_FILE_FORMAT_VERSION} = :fileFormatVersion " +
"WHERE ${BaseColumns.ID} = :id")
suspend fun updateFileFormatVersion(id: Long, fileFormatVersion: Short): Int

@Query("UPDATE ${MeasurementTable.URI_PATH} SET ${MeasurementTable.COLUMN_STATUS} = :status WHERE ${BaseColumns.ID} = :id")
@Query("UPDATE ${MeasurementTable.URI_PATH} " +
"SET ${MeasurementTable.COLUMN_STATUS} = :status " +
"WHERE ${BaseColumns.ID} = :id")
suspend fun update(id: Long, status: MeasurementStatus): Int

@Query("UPDATE ${MeasurementTable.URI_PATH} SET ${MeasurementTable.COLUMN_DISTANCE} = :distance WHERE ${BaseColumns.ID} = :id")
/**
* Updates the measurement distance entry of a measurement in the database.
*
* @param id The device-unique identifier of the measurement to update.
* @param distance The measurement distance in meters to write.
* @return The number of database entries updated.
*/
@Query("UPDATE ${MeasurementTable.URI_PATH} " +
"SET ${MeasurementTable.COLUMN_DISTANCE} = :distance " +
"WHERE ${BaseColumns.ID} = :id")
suspend fun updateDistance(id: Long, distance: Double): Int

/**
* Increments the number of bytes of all attachments collected for a measurement.
*
* @param id The device-unique identifier of the measurement to update.
* @param addedBytes The number of bytes to add to the existing `filesSize` count.
* @return The number of database entries updated.
*/
@Query("UPDATE ${MeasurementTable.URI_PATH} " +
"SET ${MeasurementTable.COLUMN_FILES_SIZE} = ${MeasurementTable.COLUMN_FILES_SIZE} + :addedBytes " +
"WHERE ${BaseColumns.ID} = :id")
suspend fun incrementFilesSize(id: Long, addedBytes: Long): Int


@Query("DELETE FROM ${MeasurementTable.URI_PATH} WHERE ${BaseColumns.ID} = :id")
suspend fun deleteItemById(id: Long): Int

@Query("DELETE FROM ${MeasurementTable.URI_PATH}")
suspend fun deleteAll(): Int
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023 Cyface GmbH
* Copyright 2023-2025 Cyface GmbH
*
* This file is part of the Cyface SDK for Android.
*
Expand Down Expand Up @@ -28,7 +28,7 @@ import androidx.room.PrimaryKey
* An instance of this class represents one row in a database table containing the measurement data.
*
* @author Armin Schnabel
* @version 6.0.0
* @version 6.1.0
* @since 1.0.0
* @property id The system-wide unique identifier of this entity, generated by the data store.
* It's `0`, which equals `null` in the non-nullable column `Long` when the entry is not yet persisted.
Expand All @@ -39,6 +39,8 @@ import androidx.room.PrimaryKey
* used to serialize the data in the file-based persistence layer of this measurement.
* @property distance The distance this measurement has captured in meters.
* @property timestamp The Unix timestamp in milliseconds indicating the start time of the measurement.
* @property filesSize The number of bytes of the attachments collected for this measurement (log,
* image and video data).
*/
@Entity
data class Measurement(
Expand All @@ -49,7 +51,8 @@ data class Measurement(
val modality: Modality,
val fileFormatVersion: Short,
val distance: Double,
val timestamp: Long
val timestamp: Long,
val filesSize: Long,
) {

/**
Expand All @@ -61,14 +64,17 @@ data class Measurement(
* used to serialize the data in the file-based persistence layer of this measurement.
* @param distance The distance this measurement has captured in meters.
* @param timestamp The Unix timestamp in milliseconds indicating the start time of the measurement.
* @param filesSize The number of bytes of the attachments collected this measurement (log,
* image and video data).
*/
constructor(
status: MeasurementStatus,
modality: Modality,
fileFormatVersion: Short,
distance: Double,
timestamp: Long
) : this(0, status, modality, fileFormatVersion, distance, timestamp)
timestamp: Long,
filesSize: Long,
) : this(0, status, modality, fileFormatVersion, distance, timestamp, filesSize)

override fun equals(other: Any?): Boolean {
if (this === other) return true
Expand All @@ -82,6 +88,7 @@ data class Measurement(
if (fileFormatVersion != other.fileFormatVersion) return false
if (distance != other.distance) return false
if (timestamp != other.timestamp) return false
if (filesSize != other.filesSize) return false

return true
}
Expand Down
Loading

0 comments on commit 8b767ea

Please sign in to comment.