Skip to content

Commit

Permalink
Merge pull request #517 from utopia-php/feat-batch-upsert
Browse files Browse the repository at this point in the history 
Feat batch upsert updates
  • Loading branch information
@abnegate
abnegate authored Feb 4, 2025
2 parents bd6f080 + 7afa1fe commit d8ad527
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 97 deletions.
12 changes: 9 additions & 3 deletions src/Database/Adapter/MariaDB.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1621,7 +1621,7 @@ public function createOrUpdateDocuments(

$documentIds = array_map(fn ($doc) => $doc->getId(), $batch);

foreach ($batch as $index => $document) {
foreach ($batch as $document) {
/**
* @var array<string, mixed> $attributes
*/
Expand Down Expand Up @@ -1665,12 +1665,13 @@ public function createOrUpdateDocuments(
if (!empty($attribute)) {
// Increment specific column by its new value in place
$updateColumns = [
"`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)"
"`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)",
"`_updatedAt` = VALUES(`_updatedAt`)"
];
} else {
// Update all columns
$updateColumns = [];
foreach (\array_keys($attributes) as $key => $attr) {
foreach (\array_keys($attributes) as $attr) {
$updateColumns[] = "`{$this->filter($attr)}` = VALUES(`{$this->filter($attr)}`)";
}
}
Expand Down Expand Up @@ -2689,6 +2690,11 @@ protected function processException(PDOException $e): \Exception
return new NotFoundException('Database not found', $e->getCode(), $e);
}

// Unknown collection
if ($e->getCode() === '42S02' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1049) {
return new NotFoundException('Collection not found', $e->getCode(), $e);
}

return $e;
}

Expand Down
162 changes: 81 additions & 81 deletions src/Database/Database.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,125 +30,125 @@

class Database
{
public const VAR_STRING = 'string';
public const string VAR_STRING = 'string';
// Simple Types
public const VAR_INTEGER = 'integer';
public const VAR_FLOAT = 'double';
public const VAR_BOOLEAN = 'boolean';
public const VAR_DATETIME = 'datetime';
public const string VAR_INTEGER = 'integer';
public const string VAR_FLOAT = 'double';
public const string VAR_BOOLEAN = 'boolean';
public const string VAR_DATETIME = 'datetime';

public const INT_MAX = 2147483647;
public const BIG_INT_MAX = PHP_INT_MAX;
public const DOUBLE_MAX = PHP_FLOAT_MAX;
public const int INT_MAX = 2147483647;
public const int BIG_INT_MAX = PHP_INT_MAX;
public const float DOUBLE_MAX = PHP_FLOAT_MAX;

// Relationship Types
public const VAR_RELATIONSHIP = 'relationship';
public const string VAR_RELATIONSHIP = 'relationship';

// Index Types
public const INDEX_KEY = 'key';
public const INDEX_FULLTEXT = 'fulltext';
public const INDEX_UNIQUE = 'unique';
public const INDEX_SPATIAL = 'spatial';
public const ARRAY_INDEX_LENGTH = 255;
public const string INDEX_KEY = 'key';
public const string INDEX_FULLTEXT = 'fulltext';
public const string INDEX_UNIQUE = 'unique';
public const string INDEX_SPATIAL = 'spatial';
public const int ARRAY_INDEX_LENGTH = 255;

// Relation Types
public const RELATION_ONE_TO_ONE = 'oneToOne';
public const RELATION_ONE_TO_MANY = 'oneToMany';
public const RELATION_MANY_TO_ONE = 'manyToOne';
public const RELATION_MANY_TO_MANY = 'manyToMany';
public const string RELATION_ONE_TO_ONE = 'oneToOne';
public const string RELATION_ONE_TO_MANY = 'oneToMany';
public const string RELATION_MANY_TO_ONE = 'manyToOne';
public const string RELATION_MANY_TO_MANY = 'manyToMany';

// Relation Actions
public const RELATION_MUTATE_CASCADE = 'cascade';
public const RELATION_MUTATE_RESTRICT = 'restrict';
public const RELATION_MUTATE_SET_NULL = 'setNull';
public const string RELATION_MUTATE_CASCADE = 'cascade';
public const string RELATION_MUTATE_RESTRICT = 'restrict';
public const string RELATION_MUTATE_SET_NULL = 'setNull';

// Relation Sides
public const RELATION_SIDE_PARENT = 'parent';
public const RELATION_SIDE_CHILD = 'child';
public const string RELATION_SIDE_PARENT = 'parent';
public const string RELATION_SIDE_CHILD = 'child';

public const RELATION_MAX_DEPTH = 3;
public const int RELATION_MAX_DEPTH = 3;

// Orders
public const ORDER_ASC = 'ASC';
public const ORDER_DESC = 'DESC';
public const string ORDER_ASC = 'ASC';
public const string ORDER_DESC = 'DESC';

// Permissions
public const PERMISSION_CREATE = 'create';
public const PERMISSION_READ = 'read';
public const PERMISSION_UPDATE = 'update';
public const PERMISSION_DELETE = 'delete';
public const string PERMISSION_CREATE = 'create';
public const string PERMISSION_READ = 'read';
public const string PERMISSION_UPDATE = 'update';
public const string PERMISSION_DELETE = 'delete';

// Aggregate permissions
public const PERMISSION_WRITE = 'write';
public const string PERMISSION_WRITE = 'write';

public const PERMISSIONS = [
public const array PERMISSIONS = [
self::PERMISSION_CREATE,
self::PERMISSION_READ,
self::PERMISSION_UPDATE,
self::PERMISSION_DELETE,
];

// Collections
public const METADATA = '_metadata';
public const string METADATA = '_metadata';

// Cursor
public const CURSOR_BEFORE = 'before';
public const CURSOR_AFTER = 'after';
public const string CURSOR_BEFORE = 'before';
public const string CURSOR_AFTER = 'after';

// Lengths
public const LENGTH_KEY = 255;
public const int LENGTH_KEY = 255;

// Cache
public const TTL = 60 * 60 * 24; // 24 hours
public const int|float TTL = 60 * 60 * 24; // 24 hours

// Events
public const EVENT_ALL = '*';

public const EVENT_DATABASE_LIST = 'database_list';
public const EVENT_DATABASE_CREATE = 'database_create';
public const EVENT_DATABASE_DELETE = 'database_delete';

public const EVENT_COLLECTION_LIST = 'collection_list';
public const EVENT_COLLECTION_CREATE = 'collection_create';
public const EVENT_COLLECTION_UPDATE = 'collection_update';
public const EVENT_COLLECTION_READ = 'collection_read';
public const EVENT_COLLECTION_DELETE = 'collection_delete';

public const EVENT_DOCUMENT_FIND = 'document_find';
public const EVENT_DOCUMENT_CREATE = 'document_create';
public const EVENT_DOCUMENT_PURGE = 'document_purge';
public const EVENT_DOCUMENTS_CREATE = 'documents_create';
public const EVENT_DOCUMENTS_DELETE = 'documents_delete';
public const EVENT_DOCUMENT_READ = 'document_read';
public const EVENT_DOCUMENT_UPDATE = 'document_update';
public const EVENT_DOCUMENTS_UPDATE = 'documents_update';
public const EVENT_DOCUMENT_DELETE = 'document_delete';
public const EVENT_DOCUMENT_COUNT = 'document_count';
public const EVENT_DOCUMENT_SUM = 'document_sum';
public const EVENT_DOCUMENT_INCREASE = 'document_increase';
public const EVENT_DOCUMENT_DECREASE = 'document_decrease';

public const EVENT_PERMISSIONS_CREATE = 'permissions_create';
public const EVENT_PERMISSIONS_READ = 'permissions_read';
public const EVENT_PERMISSIONS_DELETE = 'permissions_delete';

public const EVENT_ATTRIBUTE_CREATE = 'attribute_create';
public const EVENT_ATTRIBUTE_UPDATE = 'attribute_update';
public const EVENT_ATTRIBUTE_DELETE = 'attribute_delete';

public const EVENT_INDEX_RENAME = 'index_rename';
public const EVENT_INDEX_CREATE = 'index_create';
public const EVENT_INDEX_DELETE = 'index_delete';

public const INSERT_BATCH_SIZE = 100;
public const DELETE_BATCH_SIZE = 100;
public const string EVENT_ALL = '*';

public const string EVENT_DATABASE_LIST = 'database_list';
public const string EVENT_DATABASE_CREATE = 'database_create';
public const string EVENT_DATABASE_DELETE = 'database_delete';

public const string EVENT_COLLECTION_LIST = 'collection_list';
public const string EVENT_COLLECTION_CREATE = 'collection_create';
public const string EVENT_COLLECTION_UPDATE = 'collection_update';
public const string EVENT_COLLECTION_READ = 'collection_read';
public const string EVENT_COLLECTION_DELETE = 'collection_delete';

public const string EVENT_DOCUMENT_FIND = 'document_find';
public const string EVENT_DOCUMENT_CREATE = 'document_create';
public const string EVENT_DOCUMENT_PURGE = 'document_purge';
public const string EVENT_DOCUMENTS_CREATE = 'documents_create';
public const string EVENT_DOCUMENTS_DELETE = 'documents_delete';
public const string EVENT_DOCUMENT_READ = 'document_read';
public const string EVENT_DOCUMENT_UPDATE = 'document_update';
public const string EVENT_DOCUMENTS_UPDATE = 'documents_update';
public const string EVENT_DOCUMENT_DELETE = 'document_delete';
public const string EVENT_DOCUMENT_COUNT = 'document_count';
public const string EVENT_DOCUMENT_SUM = 'document_sum';
public const string EVENT_DOCUMENT_INCREASE = 'document_increase';
public const string EVENT_DOCUMENT_DECREASE = 'document_decrease';

public const string EVENT_PERMISSIONS_CREATE = 'permissions_create';
public const string EVENT_PERMISSIONS_READ = 'permissions_read';
public const string EVENT_PERMISSIONS_DELETE = 'permissions_delete';

public const string EVENT_ATTRIBUTE_CREATE = 'attribute_create';
public const string EVENT_ATTRIBUTE_UPDATE = 'attribute_update';
public const string EVENT_ATTRIBUTE_DELETE = 'attribute_delete';

public const string EVENT_INDEX_RENAME = 'index_rename';
public const string EVENT_INDEX_CREATE = 'index_create';
public const string EVENT_INDEX_DELETE = 'index_delete';

public const int INSERT_BATCH_SIZE = 10_000;
public const int DELETE_BATCH_SIZE = 10_000;

/**
* List of Internal attributes
*
* @var array<array<string, mixed>>
*/
public const INTERNAL_ATTRIBUTES = [
public const array INTERNAL_ATTRIBUTES = [
[
'$id' => '$id',
'type' => self::VAR_STRING,
Expand Down Expand Up @@ -211,7 +211,7 @@ class Database
[
'$id' => '$permissions',
'type' => Database::VAR_STRING,
'size' => 1000000,
'size' => 1_000_000,
'signed' => true,
'required' => false,
'default' => [],
Expand All @@ -220,7 +220,7 @@ class Database
],
];

public const INTERNAL_INDEXES = [
public const array INTERNAL_INDEXES = [
'_id',
'_uid',
'_createdAt',
Expand All @@ -235,7 +235,7 @@ class Database
*
* @var array<string, mixed>
*/
protected const COLLECTION = [
protected const array COLLECTION = [
'$id' => self::METADATA,
'$collection' => self::METADATA,
'name' => 'collections',
Expand Down
24 changes: 11 additions & 13 deletions tests/e2e/Adapter/Base.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16326,7 +16326,7 @@ public function testTransformations(): void
$this->assertTrue($result->isEmpty());
}

public function propegateBulkDocuments(string $collection, int $amount = 10, bool $documentSecurity = false): void
public function propagateBulkDocuments(string $collection, int $amount = 10, bool $documentSecurity = false): void
{
for ($i = 0; $i < $amount; $i++) {
static::getDatabase()->createDocument($collection, new Document(
Expand Down Expand Up @@ -16375,7 +16375,7 @@ public function testDeleteBulkDocuments(): void
]
);

$this->propegateBulkDocuments('bulk_delete');
$this->propagateBulkDocuments('bulk_delete');

$docs = static::getDatabase()->find('bulk_delete');
$this->assertCount(10, $docs);
Expand All @@ -16387,7 +16387,7 @@ public function testDeleteBulkDocuments(): void
$this->assertCount(0, $docs);

// TEST: Bulk delete documents with queries.
$this->propegateBulkDocuments('bulk_delete');
$this->propagateBulkDocuments('bulk_delete');

$modified = static::getDatabase()->deleteDocuments('bulk_delete', [
Query::greaterThanEqual('integer', 5)
Expand Down Expand Up @@ -16434,7 +16434,7 @@ public function testDeleteBulkDocuments(): void
static::getDatabase()->updateCollection('bulk_delete', [
Permission::create(Role::any()),
], true);
$this->propegateBulkDocuments('bulk_delete', documentSecurity: true);
$this->propagateBulkDocuments('bulk_delete', documentSecurity: true);

$this->assertCount(0, static::getDatabase()->deleteDocuments('bulk_delete'));

Expand Down Expand Up @@ -16488,7 +16488,7 @@ public function testDeleteBulkDocumentsQueries(): void
);

// Test limit
$this->propegateBulkDocuments('bulk_delete_queries');
$this->propagateBulkDocuments('bulk_delete_queries');

$this->assertCount(5, static::getDatabase()->deleteDocuments('bulk_delete_queries', [Query::limit(5)]));
$this->assertCount(5, static::getDatabase()->find('bulk_delete_queries'));
Expand All @@ -16497,16 +16497,14 @@ public function testDeleteBulkDocumentsQueries(): void
$this->assertCount(0, static::getDatabase()->find('bulk_delete_queries'));

// Test Limit more than batchSize
$this->propegateBulkDocuments('bulk_delete_queries', Database::DELETE_BATCH_SIZE * 2);
$this->assertCount(Database::DELETE_BATCH_SIZE * 2, static::getDatabase()->find('bulk_delete_queries', [Query::limit(200)]));

$this->propagateBulkDocuments('bulk_delete_queries', Database::DELETE_BATCH_SIZE * 2);
$this->assertCount(Database::DELETE_BATCH_SIZE * 2, static::getDatabase()->find('bulk_delete_queries', [Query::limit(Database::DELETE_BATCH_SIZE * 2)]));
$this->assertCount(Database::DELETE_BATCH_SIZE + 2, static::getDatabase()->deleteDocuments('bulk_delete_queries', [Query::limit(Database::DELETE_BATCH_SIZE + 2)]));

$this->assertCount(Database::DELETE_BATCH_SIZE - 2, static::getDatabase()->find('bulk_delete_queries', [Query::limit(200)]));
$this->assertCount(Database::DELETE_BATCH_SIZE - 2, static::getDatabase()->find('bulk_delete_queries', [Query::limit(Database::DELETE_BATCH_SIZE * 2)]));
$this->assertCount(Database::DELETE_BATCH_SIZE - 2, $this->getDatabase()->deleteDocuments('bulk_delete_queries'));

// Test Offset
$this->propegateBulkDocuments('bulk_delete_queries', 100);
$this->propagateBulkDocuments('bulk_delete_queries', 100);
$this->assertCount(50, static::getDatabase()->deleteDocuments('bulk_delete_queries', [Query::offset(50)]));

$docs = static::getDatabase()->find('bulk_delete_queries', [Query::limit(100)]);
Expand Down Expand Up @@ -17268,7 +17266,7 @@ public function testUpdateDocumentsQueries(): void
], documentSecurity: true);

// Test limit
$this->propegateBulkDocuments($collection, 100);
$this->propagateBulkDocuments($collection, 100);

$this->assertCount(10, static::getDatabase()->updateDocuments($collection, new Document([
'text' => 'text📝 updated',
Expand All @@ -17278,7 +17276,7 @@ public function testUpdateDocumentsQueries(): void
$this->assertCount(0, static::getDatabase()->find($collection));

// Test Offset
$this->propegateBulkDocuments($collection, 100);
$this->propagateBulkDocuments($collection, 100);
$this->assertCount(50, static::getDatabase()->updateDocuments($collection, new Document([
'text' => 'text📝 updated',
]), [Query::offset(50)]));
Expand Down

0 comments on commit d8ad527

Please sign in to comment.