Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat batch upsert updates #517

Merged
merged 3 commits into from
Feb 4, 2025
Merged

Feat batch upsert updates #517

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8b54fb5
Ensure updatedAt is updated for specific attribute upsert increase
abnegate Feb 4, 2025
8cbfb79
Catch PDO level unknown collection
abnegate Feb 4, 2025
7afa1fe
Fix hardcoded test values
abnegate Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions src/Database/Adapter/MariaDB.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1621,7 +1621,7 @@ public function createOrUpdateDocuments(

$documentIds = array_map(fn ($doc) => $doc->getId(), $batch);

foreach ($batch as $index => $document) {
foreach ($batch as $document) {
/**
* @var array<string, mixed> $attributes
*/
Expand Down Expand Up @@ -1665,12 +1665,13 @@ public function createOrUpdateDocuments(
if (!empty($attribute)) {
// Increment specific column by its new value in place
$updateColumns = [
"`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)"
"`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)",
"`_updatedAt` = VALUES(`_updatedAt`)"
];
} else {
// Update all columns
$updateColumns = [];
foreach (\array_keys($attributes) as $key => $attr) {
foreach (\array_keys($attributes) as $attr) {
$updateColumns[] = "`{$this->filter($attr)}` = VALUES(`{$this->filter($attr)}`)";
}
}
Expand Down Expand Up @@ -2689,6 +2690,11 @@ protected function processException(PDOException $e): \Exception
return new NotFoundException('Database not found', $e->getCode(), $e);
}

// Unknown collection
if ($e->getCode() === '42S02' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1049) {
return new NotFoundException('Collection not found', $e->getCode(), $e);
}

return $e;
}

Expand Down
162 changes: 81 additions & 81 deletions src/Database/Database.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,125 +30,125 @@

class Database
{
public const VAR_STRING = 'string';
public const string VAR_STRING = 'string';
// Simple Types
public const VAR_INTEGER = 'integer';
public const VAR_FLOAT = 'double';
public const VAR_BOOLEAN = 'boolean';
public const VAR_DATETIME = 'datetime';
public const string VAR_INTEGER = 'integer';
public const string VAR_FLOAT = 'double';
public const string VAR_BOOLEAN = 'boolean';
public const string VAR_DATETIME = 'datetime';

public const INT_MAX = 2147483647;
public const BIG_INT_MAX = PHP_INT_MAX;
public const DOUBLE_MAX = PHP_FLOAT_MAX;
public const int INT_MAX = 2147483647;
public const int BIG_INT_MAX = PHP_INT_MAX;
public const float DOUBLE_MAX = PHP_FLOAT_MAX;

// Relationship Types
public const VAR_RELATIONSHIP = 'relationship';
public const string VAR_RELATIONSHIP = 'relationship';

// Index Types
public const INDEX_KEY = 'key';
public const INDEX_FULLTEXT = 'fulltext';
public const INDEX_UNIQUE = 'unique';
public const INDEX_SPATIAL = 'spatial';
public const ARRAY_INDEX_LENGTH = 255;
public const string INDEX_KEY = 'key';
public const string INDEX_FULLTEXT = 'fulltext';
public const string INDEX_UNIQUE = 'unique';
public const string INDEX_SPATIAL = 'spatial';
public const int ARRAY_INDEX_LENGTH = 255;

// Relation Types
public const RELATION_ONE_TO_ONE = 'oneToOne';
public const RELATION_ONE_TO_MANY = 'oneToMany';
public const RELATION_MANY_TO_ONE = 'manyToOne';
public const RELATION_MANY_TO_MANY = 'manyToMany';
public const string RELATION_ONE_TO_ONE = 'oneToOne';
public const string RELATION_ONE_TO_MANY = 'oneToMany';
public const string RELATION_MANY_TO_ONE = 'manyToOne';
public const string RELATION_MANY_TO_MANY = 'manyToMany';

// Relation Actions
public const RELATION_MUTATE_CASCADE = 'cascade';
public const RELATION_MUTATE_RESTRICT = 'restrict';
public const RELATION_MUTATE_SET_NULL = 'setNull';
public const string RELATION_MUTATE_CASCADE = 'cascade';
public const string RELATION_MUTATE_RESTRICT = 'restrict';
public const string RELATION_MUTATE_SET_NULL = 'setNull';

// Relation Sides
public const RELATION_SIDE_PARENT = 'parent';
public const RELATION_SIDE_CHILD = 'child';
public const string RELATION_SIDE_PARENT = 'parent';
public const string RELATION_SIDE_CHILD = 'child';

public const RELATION_MAX_DEPTH = 3;
public const int RELATION_MAX_DEPTH = 3;

// Orders
public const ORDER_ASC = 'ASC';
public const ORDER_DESC = 'DESC';
public const string ORDER_ASC = 'ASC';
public const string ORDER_DESC = 'DESC';

// Permissions
public const PERMISSION_CREATE = 'create';
public const PERMISSION_READ = 'read';
public const PERMISSION_UPDATE = 'update';
public const PERMISSION_DELETE = 'delete';
public const string PERMISSION_CREATE = 'create';
public const string PERMISSION_READ = 'read';
public const string PERMISSION_UPDATE = 'update';
public const string PERMISSION_DELETE = 'delete';

// Aggregate permissions
public const PERMISSION_WRITE = 'write';
public const string PERMISSION_WRITE = 'write';

public const PERMISSIONS = [
public const array PERMISSIONS = [
self::PERMISSION_CREATE,
self::PERMISSION_READ,
self::PERMISSION_UPDATE,
self::PERMISSION_DELETE,
];

// Collections
public const METADATA = '_metadata';
public const string METADATA = '_metadata';

// Cursor
public const CURSOR_BEFORE = 'before';
public const CURSOR_AFTER = 'after';
public const string CURSOR_BEFORE = 'before';
public const string CURSOR_AFTER = 'after';

// Lengths
public const LENGTH_KEY = 255;
public const int LENGTH_KEY = 255;

// Cache
public const TTL = 60 * 60 * 24; // 24 hours
public const int|float TTL = 60 * 60 * 24; // 24 hours

// Events
public const EVENT_ALL = '*';

public const EVENT_DATABASE_LIST = 'database_list';
public const EVENT_DATABASE_CREATE = 'database_create';
public const EVENT_DATABASE_DELETE = 'database_delete';

public const EVENT_COLLECTION_LIST = 'collection_list';
public const EVENT_COLLECTION_CREATE = 'collection_create';
public const EVENT_COLLECTION_UPDATE = 'collection_update';
public const EVENT_COLLECTION_READ = 'collection_read';
public const EVENT_COLLECTION_DELETE = 'collection_delete';

public const EVENT_DOCUMENT_FIND = 'document_find';
public const EVENT_DOCUMENT_CREATE = 'document_create';
public const EVENT_DOCUMENT_PURGE = 'document_purge';
public const EVENT_DOCUMENTS_CREATE = 'documents_create';
public const EVENT_DOCUMENTS_DELETE = 'documents_delete';
public const EVENT_DOCUMENT_READ = 'document_read';
public const EVENT_DOCUMENT_UPDATE = 'document_update';
public const EVENT_DOCUMENTS_UPDATE = 'documents_update';
public const EVENT_DOCUMENT_DELETE = 'document_delete';
public const EVENT_DOCUMENT_COUNT = 'document_count';
public const EVENT_DOCUMENT_SUM = 'document_sum';
public const EVENT_DOCUMENT_INCREASE = 'document_increase';
public const EVENT_DOCUMENT_DECREASE = 'document_decrease';

public const EVENT_PERMISSIONS_CREATE = 'permissions_create';
public const EVENT_PERMISSIONS_READ = 'permissions_read';
public const EVENT_PERMISSIONS_DELETE = 'permissions_delete';

public const EVENT_ATTRIBUTE_CREATE = 'attribute_create';
public const EVENT_ATTRIBUTE_UPDATE = 'attribute_update';
public const EVENT_ATTRIBUTE_DELETE = 'attribute_delete';

public const EVENT_INDEX_RENAME = 'index_rename';
public const EVENT_INDEX_CREATE = 'index_create';
public const EVENT_INDEX_DELETE = 'index_delete';

public const INSERT_BATCH_SIZE = 100;
public const DELETE_BATCH_SIZE = 100;
public const string EVENT_ALL = '*';

public const string EVENT_DATABASE_LIST = 'database_list';
public const string EVENT_DATABASE_CREATE = 'database_create';
public const string EVENT_DATABASE_DELETE = 'database_delete';

public const string EVENT_COLLECTION_LIST = 'collection_list';
public const string EVENT_COLLECTION_CREATE = 'collection_create';
public const string EVENT_COLLECTION_UPDATE = 'collection_update';
public const string EVENT_COLLECTION_READ = 'collection_read';
public const string EVENT_COLLECTION_DELETE = 'collection_delete';

public const string EVENT_DOCUMENT_FIND = 'document_find';
public const string EVENT_DOCUMENT_CREATE = 'document_create';
public const string EVENT_DOCUMENT_PURGE = 'document_purge';
public const string EVENT_DOCUMENTS_CREATE = 'documents_create';
public const string EVENT_DOCUMENTS_DELETE = 'documents_delete';
public const string EVENT_DOCUMENT_READ = 'document_read';
public const string EVENT_DOCUMENT_UPDATE = 'document_update';
public const string EVENT_DOCUMENTS_UPDATE = 'documents_update';
public const string EVENT_DOCUMENT_DELETE = 'document_delete';
public const string EVENT_DOCUMENT_COUNT = 'document_count';
public const string EVENT_DOCUMENT_SUM = 'document_sum';
public const string EVENT_DOCUMENT_INCREASE = 'document_increase';
public const string EVENT_DOCUMENT_DECREASE = 'document_decrease';

public const string EVENT_PERMISSIONS_CREATE = 'permissions_create';
public const string EVENT_PERMISSIONS_READ = 'permissions_read';
public const string EVENT_PERMISSIONS_DELETE = 'permissions_delete';

public const string EVENT_ATTRIBUTE_CREATE = 'attribute_create';
public const string EVENT_ATTRIBUTE_UPDATE = 'attribute_update';
public const string EVENT_ATTRIBUTE_DELETE = 'attribute_delete';

public const string EVENT_INDEX_RENAME = 'index_rename';
public const string EVENT_INDEX_CREATE = 'index_create';
public const string EVENT_INDEX_DELETE = 'index_delete';

public const int INSERT_BATCH_SIZE = 10_000;
public const int DELETE_BATCH_SIZE = 10_000;

/**
* List of Internal attributes
*
* @var array<array<string, mixed>>
*/
public const INTERNAL_ATTRIBUTES = [
public const array INTERNAL_ATTRIBUTES = [
[
'$id' => '$id',
'type' => self::VAR_STRING,
Expand Down Expand Up @@ -211,7 +211,7 @@ class Database
[
'$id' => '$permissions',
'type' => Database::VAR_STRING,
'size' => 1000000,
'size' => 1_000_000,
'signed' => true,
'required' => false,
'default' => [],
Expand All @@ -220,7 +220,7 @@ class Database
],
];

public const INTERNAL_INDEXES = [
public const array INTERNAL_INDEXES = [
'_id',
'_uid',
'_createdAt',
Expand All @@ -235,7 +235,7 @@ class Database
*
* @var array<string, mixed>
*/
protected const COLLECTION = [
protected const array COLLECTION = [
'$id' => self::METADATA,
'$collection' => self::METADATA,
'name' => 'collections',
Expand Down
24 changes: 11 additions & 13 deletions tests/e2e/Adapter/Base.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16318,7 +16318,7 @@ public function testTransformations(): void
$this->assertTrue($result->isEmpty());
}

public function propegateBulkDocuments(string $collection, int $amount = 10, bool $documentSecurity = false): void
public function propagateBulkDocuments(string $collection, int $amount = 10, bool $documentSecurity = false): void
{
for ($i = 0; $i < $amount; $i++) {
static::getDatabase()->createDocument($collection, new Document(
Expand Down Expand Up @@ -16367,7 +16367,7 @@ public function testDeleteBulkDocuments(): void
]
);

$this->propegateBulkDocuments('bulk_delete');
$this->propagateBulkDocuments('bulk_delete');

$docs = static::getDatabase()->find('bulk_delete');
$this->assertCount(10, $docs);
Expand All @@ -16379,7 +16379,7 @@ public function testDeleteBulkDocuments(): void
$this->assertCount(0, $docs);

// TEST: Bulk delete documents with queries.
$this->propegateBulkDocuments('bulk_delete');
$this->propagateBulkDocuments('bulk_delete');

$modified = static::getDatabase()->deleteDocuments('bulk_delete', [
Query::greaterThanEqual('integer', 5)
Expand Down Expand Up @@ -16426,7 +16426,7 @@ public function testDeleteBulkDocuments(): void
static::getDatabase()->updateCollection('bulk_delete', [
Permission::create(Role::any()),
], true);
$this->propegateBulkDocuments('bulk_delete', documentSecurity: true);
$this->propagateBulkDocuments('bulk_delete', documentSecurity: true);

$this->assertCount(0, static::getDatabase()->deleteDocuments('bulk_delete'));

Expand Down Expand Up @@ -16480,7 +16480,7 @@ public function testDeleteBulkDocumentsQueries(): void
);

// Test limit
$this->propegateBulkDocuments('bulk_delete_queries');
$this->propagateBulkDocuments('bulk_delete_queries');

$this->assertCount(5, static::getDatabase()->deleteDocuments('bulk_delete_queries', [Query::limit(5)]));
$this->assertCount(5, static::getDatabase()->find('bulk_delete_queries'));
Expand All @@ -16489,16 +16489,14 @@ public function testDeleteBulkDocumentsQueries(): void
$this->assertCount(0, static::getDatabase()->find('bulk_delete_queries'));

// Test Limit more than batchSize
$this->propegateBulkDocuments('bulk_delete_queries', Database::DELETE_BATCH_SIZE * 2);
$this->assertCount(Database::DELETE_BATCH_SIZE * 2, static::getDatabase()->find('bulk_delete_queries', [Query::limit(200)]));

$this->propagateBulkDocuments('bulk_delete_queries', Database::DELETE_BATCH_SIZE * 2);
$this->assertCount(Database::DELETE_BATCH_SIZE * 2, static::getDatabase()->find('bulk_delete_queries', [Query::limit(Database::DELETE_BATCH_SIZE * 2)]));
$this->assertCount(Database::DELETE_BATCH_SIZE + 2, static::getDatabase()->deleteDocuments('bulk_delete_queries', [Query::limit(Database::DELETE_BATCH_SIZE + 2)]));

$this->assertCount(Database::DELETE_BATCH_SIZE - 2, static::getDatabase()->find('bulk_delete_queries', [Query::limit(200)]));
$this->assertCount(Database::DELETE_BATCH_SIZE - 2, static::getDatabase()->find('bulk_delete_queries', [Query::limit(Database::DELETE_BATCH_SIZE * 2)]));
$this->assertCount(Database::DELETE_BATCH_SIZE - 2, $this->getDatabase()->deleteDocuments('bulk_delete_queries'));

// Test Offset
$this->propegateBulkDocuments('bulk_delete_queries', 100);
$this->propagateBulkDocuments('bulk_delete_queries', 100);
$this->assertCount(50, static::getDatabase()->deleteDocuments('bulk_delete_queries', [Query::offset(50)]));

$docs = static::getDatabase()->find('bulk_delete_queries', [Query::limit(100)]);
Expand Down Expand Up @@ -17260,7 +17258,7 @@ public function testUpdateDocumentsQueries(): void
], documentSecurity: true);

// Test limit
$this->propegateBulkDocuments($collection, 100);
$this->propagateBulkDocuments($collection, 100);

$this->assertCount(10, static::getDatabase()->updateDocuments($collection, new Document([
'text' => 'text📝 updated',
Expand All @@ -17270,7 +17268,7 @@ public function testUpdateDocumentsQueries(): void
$this->assertCount(0, static::getDatabase()->find($collection));

// Test Offset
$this->propegateBulkDocuments($collection, 100);
$this->propagateBulkDocuments($collection, 100);
$this->assertCount(50, static::getDatabase()->updateDocuments($collection, new Document([
'text' => 'text📝 updated',
]), [Query::offset(50)]));
Expand Down