Skip to content

Commit

Permalink
Finalize sharding support for 2.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@alcaeus
alcaeus committed Sep 24, 2018
1 parent a4b3c82 commit d87a76d
Show file tree
Hide file tree
Showing 9 changed files with 91 additions and 226 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
# Test against lowest dependencies, including driver and server versions
- stage: Test
php: 7.2
env: SERVER_VERSION="3.2" KEY_ID="EA312927" DRIVER_VERSION="1.3.0"
env: SERVER_VERSION="3.2" KEY_ID="EA312927" DRIVER_VERSION="1.5.0"
install:
- composer update --prefer-lowest

Expand Down
1 change: 1 addition & 0 deletions composer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
],
"require": {
"php": "^7.2",
"ext-mongodb": "^1.5",
"symfony/console": "^3.4|^4.1",
"doctrine/annotations": "^1.6",
"doctrine/collections": "^1.5",
Expand Down
8 changes: 7 additions & 1 deletion docs/en/reference/sharding.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ about sharding, please refer to the `MongoDB docs <https://docs.mongodb.com/manu

Once you have a `sharded cluster <https://docs.mongodb.com/manual/core/sharded-cluster-architectures-production/>`_,
you can enable sharding for a document. You can do this by defining a shard key in
the document:
the document as well as an appropriate index:

.. configuration-block::

Expand All @@ -19,6 +19,7 @@ the document:
/**
* @Document
* @Indexes(@Index(keys={"username"="asc"}))
* @ShardKey(keys={"username"="asc"})
*/
class User
Expand All @@ -44,6 +45,11 @@ the document:
<shard-key>
<key name="username" order="asc"/>
</shard-key>
<indexes>
<index>
<key name="username" order="asc" />
</index>
</indexes>
</document>
</doctrine-mongo-mapping>
Expand Down
74 changes: 34 additions & 40 deletions lib/Doctrine/ODM/MongoDB/SchemaManager.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,22 @@
use Doctrine\ODM\MongoDB\Mapping\ClassMetadata;
use Doctrine\ODM\MongoDB\Mapping\ClassMetadataFactory;
use MongoDB\Driver\Exception\RuntimeException;
use MongoDB\Driver\Exception\ServerException;
use MongoDB\Model\IndexInfo;
use function array_filter;
use function array_unique;
use function iterator_count;
use function iterator_to_array;
use function ksort;
use function strpos;

class SchemaManager
{
private const GRIDFS_FILE_COLLECTION_INDEX = ['files_id' => 1, 'n' => 1];

private const GRIDFS_CHUNKS_COLLECTION_INDEX = ['filename' => 1, 'uploadDate' => 1];

private const CODE_SHARDING_ALREADY_INITIALIZED = 23;

/** @var DocumentManager */
protected $dm;

Expand Down Expand Up @@ -522,68 +525,42 @@ private function isEquivalentTextIndexWeights($mongoIndex, array $documentIndex)
* Ensure collections are sharded for all documents that can be loaded with the
* metadata factory.
*
* @param array $indexOptions Options for `ensureIndex` command. It's performed on an existing collections
*
* @throws MongoDBException
*/
public function ensureSharding(array $indexOptions = []): void
public function ensureSharding(): void
{
foreach ($this->metadataFactory->getAllMetadata() as $class) {
if ($class->isMappedSuperclass || ! $class->isSharded()) {
continue;
}

$this->ensureDocumentSharding($class->name, $indexOptions);
$this->ensureDocumentSharding($class->name);
}
}

/**
* Ensure sharding for collection by document name.
*
* @param array $indexOptions Options for `ensureIndex` command. It's performed on an existing collections.
*
* @throws MongoDBException
*/
public function ensureDocumentSharding(string $documentName, array $indexOptions = []): void
public function ensureDocumentSharding(string $documentName): void
{
$class = $this->dm->getClassMetadata($documentName);
if (! $class->isSharded()) {
return;
}

$this->enableShardingForDbByDocumentName($documentName);

$try = 0;
do {
try {
$result = $this->runShardCollectionCommand($documentName);
$done = true;

// Need to check error message because MongoDB 3.0 does not return a code for this error
if (! (bool) $result['ok'] && strpos($result['errmsg'], 'please create an index that starts') !== false) {
// The proposed key is not returned when using mongo-php-adapter with ext-mongodb.
// See https://github.com/mongodb/mongo-php-driver/issues/296 for details
$key = $result['proposedKey'] ?? $this->dm->getClassMetadata($documentName)->getShardKey()['keys'];

$this->dm->getDocumentCollection($documentName)->ensureIndex($key, $indexOptions);
$done = false;
}
} catch (RuntimeException $e) {
if ($e->getCode() === 20 || $e->getCode() === 23 || $e->getMessage() === 'already sharded') {
return;
}

throw $e;
}
} while (! $done && $try < 2);

// Starting with MongoDB 3.2, this command returns code 20 when a collection is already sharded.
// For older MongoDB versions, check the error message
if ((bool) $result['ok'] || (isset($result['code']) && $result['code'] === 20) || $result['errmsg'] === 'already sharded') {
if ($this->collectionIsSharded($documentName)) {
return;
}

throw MongoDBException::failedToEnsureDocumentSharding($documentName, $result['errmsg']);
$this->enableShardingForDbByDocumentName($documentName);

try {
$this->runShardCollectionCommand($documentName);
} catch (RuntimeException $e) {
throw MongoDBException::failedToEnsureDocumentSharding($documentName, $e->getMessage());
}
}

/**
Expand All @@ -598,10 +575,13 @@ public function enableShardingForDbByDocumentName(string $documentName): void

try {
$adminDb->command(['enableSharding' => $dbName]);
} catch (RuntimeException $e) {
if ($e->getCode() !== 23 || $e->getMessage() === 'already enabled') {
} catch (ServerException $e) {
// Don't throw an exception if sharding is already enabled; there's just no other way to check this
if ($e->getCode() !== self::CODE_SHARDING_ALREADY_INITIALIZED) {
throw MongoDBException::failedToEnableSharding($dbName, $e->getMessage());
}
} catch (RuntimeException $e) {
throw MongoDBException::failedToEnableSharding($dbName, $e->getMessage());
}
}

Expand Down Expand Up @@ -651,4 +631,18 @@ private function ensureFilesIndex(ClassMetadata $class): void

$filesCollection->createIndex(self::GRIDFS_CHUNKS_COLLECTION_INDEX);
}

private function collectionIsSharded(string $documentName): bool
{
$class = $this->dm->getClassMetadata($documentName);

$database = $this->dm->getDocumentDatabase($documentName);
$collections = $database->listCollections(['filter' => ['name' => $class->getCollection()]]);
if (! iterator_count($collections)) {
return false;
}

$stats = $database->command(['collstats' => $class->getCollection()])->toArray()[0];
return (bool) ($stats['sharded'] ?? false);
}
}
51 changes: 32 additions & 19 deletions tests/Doctrine/ODM/MongoDB/Tests/Functional/EnsureShardingTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace Doctrine\ODM\MongoDB\Tests\Functional;

use Doctrine\ODM\MongoDB\MongoDBException;
use Doctrine\ODM\MongoDB\Tests\BaseTest;
use Documents\Sharded\ShardedOne;
use Documents\Sharded\ShardedOneWithDifferentKey;
Expand All @@ -21,6 +22,7 @@ public function setUp()
public function testEnsureShardingForNewCollection()
{
$class = ShardedOne::class;
$this->dm->getSchemaManager()->ensureDocumentIndexes($class);
$this->dm->getSchemaManager()->ensureDocumentSharding($class);

$collection = $this->dm->getDocumentCollection($class);
Expand All @@ -32,16 +34,12 @@ public function testEnsureShardingForNewCollection()
$this->assertTrue($stats['sharded']);
}

public function testEnsureShardingForCollectionWithDocuments()
public function testEnsureShardingForNewCollectionWithoutCreatingIndexes()
{
$this->markTestSkipped('Test does not pass due to https://github.com/mongodb/mongo-php-driver/issues/296');
$class = ShardedOne::class;
$collection = $this->dm->getDocumentCollection($class);
$doc = ['title' => 'hey', 'k' => 'hi'];
$collection->insertOne($doc);

$this->dm->getSchemaManager()->ensureDocumentSharding($class);

$collection = $this->dm->getDocumentCollection($class);
$indexes = iterator_to_array($collection->listIndexes());
$stats = $this->dm->getDocumentDatabase($class)->command(['collstats' => $collection->getCollectionName()])->toArray()[0];

Expand All @@ -50,38 +48,53 @@ public function testEnsureShardingForCollectionWithDocuments()
$this->assertTrue($stats['sharded']);
}

public function testEnsureShardingForCollectionWithShardingEnabled()
public function testEnsureShardingForCollectionWithDocuments()
{
$class = ShardedOneWithDifferentKey::class;
$this->dm->getSchemaManager()->ensureDocumentSharding($class);
$class = ShardedOne::class;

$this->dm->getSchemaManager()->ensureDocumentSharding(ShardedOne::class);
$document = new ShardedOne();
$this->dm->persist($document);
$this->dm->flush();

$this->dm->getSchemaManager()->ensureDocumentIndexes($class);
$this->dm->getSchemaManager()->ensureDocumentSharding($class);

$collection = $this->dm->getDocumentCollection($class);
$indexes = iterator_to_array($collection->listIndexes());
$stats = $this->dm->getDocumentDatabase($class)->command(['collstats' => $collection->getCollectionName()])->toArray()[0];

$this->assertCount(2, $indexes);
$this->assertSame(['v' => 1], $indexes[1]['key']);
$this->assertTrue($stats['sharded']);
}

public function testEnsureShardingForCollectionWithData()
public function testEnsureShardingForCollectionWithDocumentsThrowsIndexError()
{
$this->markTestSkipped('Test does not pass due to https://github.com/mongodb/mongo-php-driver/issues/296');
$class = ShardedOne::class;

$document = new ShardedOne();
$this->dm->persist($document);
$this->dm->flush();

$class = ShardedOne::class;
$this->expectException(MongoDBException::class);
$this->expectExceptionMessage('Failed to ensure sharding for document');

$this->dm->getSchemaManager()->ensureDocumentSharding($class);

$collection = $this->dm->getDocumentCollection($class);
$indexes = iterator_to_array($collection->listIndexes());
$stats = $this->dm->getDocumentDatabase($class)->command(['collstats' => $collection->getCollectionName()])->toArray()[0];

$this->assertCount(2, $indexes);
$this->assertSame(['k' => 1], $indexes[1]['key']);
$this->assertFalse($stats['sharded']);
}

public function testEnsureShardingForCollectionWithShardingEnabled()
{
$class = ShardedOneWithDifferentKey::class;
$this->dm->getSchemaManager()->ensureDocumentIndexes($class);
$this->dm->getSchemaManager()->ensureDocumentSharding($class);

$this->dm->getSchemaManager()->ensureDocumentSharding(ShardedOne::class);

$collection = $this->dm->getDocumentCollection($class);
$stats = $this->dm->getDocumentDatabase($class)->command(['collstats' => $collection->getCollectionName()])->toArray()[0];

$this->assertTrue($stats['sharded']);
}
}
Loading

0 comments on commit d87a76d

Please sign in to comment.