From 0dec04b6bb4d02fb808d571bf1f7550942fce71c Mon Sep 17 00:00:00 2001 From: Giorgio Franceschetti Date: Sun, 12 Mar 2023 09:25:30 +0100 Subject: [PATCH] Added delete method to GridOut + more methods --- CHANGELOG.md | 8 +++ analysis_options.yaml | 7 ++- example/manual/gridfs/delete.dart | 99 +++++++++++++++++++++++++++++++ lib/mongo_dart.dart | 3 + lib/src/extensions/file_ext.dart | 38 ++++++++++++ lib/src/gridfs/grid_fs_file.dart | 2 +- lib/src/gridfs/grid_out.dart | 45 ++++++++++++++ lib/src/gridfs/gridfs.dart | 17 +++++- pubspec.yaml | 2 +- test/file_ext_test.dart | 91 ++++++++++++++++++++++++++++ 10 files changed, 304 insertions(+), 8 deletions(-) create mode 100644 example/manual/gridfs/delete.dart create mode 100644 lib/src/extensions/file_ext.dart create mode 100644 test/file_ext_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index a71ce24f..1fefd214 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.9.1 + +- Added a `delete()` method to `GridOut` class +- Added a `toFile()` method to `GridOut` class (allows to write safely on disk - adds a `(n)` suffix if the file already exists) +- Added a `clearBucket()` method to class `GridFs` +- Added a `dropBucket()` mehthod to class `GridFs` +- Fixed an issue on `GridFsFile` `numChunks()` method + ## 0.9.0 - Fixed an issue on GridFs `save()` method using MongoDb 6.0 diff --git a/analysis_options.yaml b/analysis_options.yaml index d6048080..123e8177 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -5,9 +5,10 @@ include: package:lints/recommended.yaml # For lint rules and documentation, see http://dart-lang.github.io/linter/lints. # Uncomment to specify additional rules. -# linter: -# rules: -# - camel_case_types +linter: + rules: + - await_only_futures + - unawaited_futures # analyzer: diff --git a/example/manual/gridfs/delete.dart b/example/manual/gridfs/delete.dart new file mode 100644 index 00000000..03ed3fed --- /dev/null +++ b/example/manual/gridfs/delete.dart @@ -0,0 +1,99 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:mongo_dart/mongo_dart.dart'; +import 'package:path/path.dart'; + +const dbName = 'mongo-dart-example'; +const dbAddress = '127.0.0.1'; + +const defaultUri = 'mongodb://$dbAddress:27017/$dbName'; + +class MockConsumer implements StreamConsumer> { + List data = []; + + Future consume(Stream> stream) { + var completer = Completer(); + stream.listen(_onData, onDone: () => completer.complete(null)); + return completer.future; + } + + void _onData(List chunk) { + data.addAll(chunk); + } + + @override + Future addStream(Stream> stream) { + var completer = Completer(); + stream.listen(_onData, onDone: () => completer.complete(null)); + return completer.future; + } + + @override + Future close() => Future.value(true); +} + +void main() async { + var db = Db(defaultUri); + await db.open(); + + Future cleanupDatabase() async { + await db.close(); + } + + if (!db.masterConnection.serverCapabilities.supportsOpMsg) { + return; + } + + var collectionName = 'delete-gridfs'; + var gridFS = GridFS(db, collectionName); + await gridFS.dropBucket(); + + // **** data preparation + var smallData = [ + 0x00, + 0x01, + 0x10, + 0x11, + 0x7e, + 0x7f, + 0x80, + 0x81, + 0xfe, + 0xff + ]; + + // Set small chunks + GridFS.defaultChunkSize = 9; + + // assures at least 3 chunks + var target = GridFS.defaultChunkSize * 3; + var data = []; + while (data.length < target) { + data.addAll(smallData); + } + print('Expected chunks: ${(data.length / GridFS.defaultChunkSize).ceil()}'); + var extraData = { + 'test': [1, 2, 3], + 'extraData': 'Test', + 'map': {'a': 1} + }; + + var inputStream = Stream.fromIterable([data]); + var input = gridFS.createFile(inputStream, 'test'); + input.extraData = extraData; + await input.save(); + + var gridOut = await gridFS.findOne(where.eq('_id', input.id)); + var consumer = MockConsumer(); + var out = IOSink(consumer); + await gridOut?.writeTo(out); + + await gridOut?.delete(); + + print('Out Chunk size: ${gridOut?.chunkSize}'); // 9 + print('Out Chunk lengt: ${gridOut?.length}'); + print('Out Chunk num: ${gridOut?.numChunks()}'); + + await cleanupDatabase(); +} diff --git a/lib/mongo_dart.dart b/lib/mongo_dart.dart index 7dbe3621..23c32c86 100644 --- a/lib/mongo_dart.dart +++ b/lib/mongo_dart.dart @@ -14,6 +14,7 @@ import 'dart:io' File, FileMode, IOSink, + Platform, SecureSocket, SecurityContext, Socket, @@ -38,6 +39,7 @@ import 'package:mongo_dart/src/database/utils/dns_lookup.dart'; import 'package:mongo_dart/src/database/utils/map_keys.dart'; import 'package:mongo_dart/src/database/utils/parms_utils.dart'; import 'package:mongo_dart/src/database/utils/split_hosts.dart'; +import 'package:mongo_dart/src/extensions/file_ext.dart'; import 'package:mongo_dart_query/mongo_dart_query.dart'; import 'package:pool/pool.dart'; import 'package:mongo_dart/src/auth/auth.dart' @@ -66,6 +68,7 @@ import 'src/database/commands/aggregation_commands/count/count_options.dart'; import 'src/database/commands/aggregation_commands/count/count_result.dart'; import 'src/database/commands/base/command_operation.dart'; import 'src/database/commands/diagnostic_commands/ping_command/ping_command.dart'; +import 'package:path/path.dart' as p; export 'package:bson/bson.dart'; export 'package:mongo_dart_query/mongo_aggregation.dart'; diff --git a/lib/src/extensions/file_ext.dart b/lib/src/extensions/file_ext.dart new file mode 100644 index 00000000..c1b66580 --- /dev/null +++ b/lib/src/extensions/file_ext.dart @@ -0,0 +1,38 @@ +import 'dart:io'; +import 'package:path/path.dart' as p; + +extension FileExt on File { + String get name => + path.substring(path.lastIndexOf(Platform.pathSeparator) + 1); + + String newPathByName(String newFileName) => + path.substring(0, path.lastIndexOf(Platform.pathSeparator) + 1) + + newFileName; + + Future get safePath async => toSafePath(path); + + Future toSafePath(String newPath) async { + var basename = p.basenameWithoutExtension(newPath); + var dirname = p.dirname(newPath) + Platform.pathSeparator; + var ext = p.extension(newPath); + + String tryPath = newPath; + File newFile = File(tryPath); + var count = 1; + while (await newFile.exists()) { + tryPath = '$dirname$basename($count)$ext'; + count++; + newFile = File(tryPath); + } + return tryPath; + } + + Future changeFileNameOnly(String newFileName) async => + rename(newPathByName(newFileName)); + + Future changeFileNameOnlySafe(String newFileName) async => + renameSafe(newPathByName(newFileName)); + + Future renameSafe(String newPath) async => + rename(await toSafePath(newPath)); +} diff --git a/lib/src/gridfs/grid_fs_file.dart b/lib/src/gridfs/grid_fs_file.dart index e2aa7881..d361118b 100644 --- a/lib/src/gridfs/grid_fs_file.dart +++ b/lib/src/gridfs/grid_fs_file.dart @@ -47,7 +47,7 @@ abstract class GridFSFile { return completer.future; } - int numChunks() => (length?.toDouble() ?? 0.0 / (chunkSize)).ceil().toInt(); + int numChunks() => ((length ?? 0.0) / chunkSize).ceil(); List get aliases => extraData['aliases'] as List; diff --git a/lib/src/gridfs/grid_out.dart b/lib/src/gridfs/grid_out.dart index 4c08e3b0..324a5fd6 100644 --- a/lib/src/gridfs/grid_out.dart +++ b/lib/src/gridfs/grid_out.dart @@ -13,6 +13,45 @@ class GridOut extends GridFSFile { return sink.done; } + /// This method uses a different approach in writing to a file. + /// It uses a temp file to store the data and then renames it to + /// the correct name. If overwriteExisting file is true, + /// and a file with the same name exists, it is overwitten, + /// otherwise a suffix like "(n)" is appended to the file name, where n is + /// a progressive number not yet assigned to any existing file in the system + Future toFile(File file, + {FileMode? mode, bool? overwriteExistingFile}) async { + overwriteExistingFile ??= false; + mode ??= FileMode.writeOnly; + if (mode == FileMode.read) { + throw ArgumentError('Read file mode not valid for method "toFile()"'); + } + File tempFile; + String tempFilePath = '${p.dirname(file.path)}${Platform.pathSeparator}' + '${p.basenameWithoutExtension(file.path)}_${Uuid().v4()}' + '${p.extension(file.path)}'; + if (mode == FileMode.append || mode == FileMode.writeOnlyAppend) { + tempFile = await file.copy(tempFilePath); + } else { + tempFile = File(tempFilePath); + } + Future addToFile(Map chunk) async { + final bytes = chunk['data'] as BsonBinary; + await tempFile.writeAsBytes(bytes.byteList, + mode: FileMode.writeOnlyAppend, flush: true); + } + + var chunkList = + await fs.chunks.find(where.eq('files_id', id).sortBy('n')).toList(); + for (var chunk in chunkList) { + await addToFile(chunk); + } + if (overwriteExistingFile) { + return tempFile.changeFileNameOnly(file.name); + } + return tempFile.changeFileNameOnlySafe(file.name); + } + Future writeTo(IOSink out) { var length = 0; var completer = Completer(); @@ -28,4 +67,10 @@ class GridOut extends GridFSFile { .then((_) => completer.complete(length)); return completer.future; } + + /// Removes this document from the bucket + Future delete() async { + await fs.files.deleteOne(where.id(id)); + await fs.chunks.deleteMany(where.eq('files_id', id)); + } } diff --git a/lib/src/gridfs/gridfs.dart b/lib/src/gridfs/gridfs.dart index ece30d71..ee3e3666 100644 --- a/lib/src/gridfs/gridfs.dart +++ b/lib/src/gridfs/gridfs.dart @@ -16,9 +16,8 @@ class GridFS { // T O D O (tsander): Ensure index. - Stream> getFileList(SelectorBuilder selectorBuilder) { - return files.find(selectorBuilder.sortBy('filename', descending: true)); - } + Stream> getFileList(SelectorBuilder selectorBuilder) => + files.find(selectorBuilder.sortBy('filename', descending: true)); Future findOne(selector) async { //var completer = Completer(); @@ -43,4 +42,16 @@ class GridFS { GridIn createFile(Stream> input, String filename) => GridIn._(this, filename, input); + + /// **Beware!** This method removes all the documents in this bucket + Future clearBucket() async { + await files.deleteMany({}); + await chunks.deleteMany({}); + } + + /// **Beware!** This method drops this bucket + Future dropBucket() async { + await files.drop(); + await chunks.drop(); + } } diff --git a/pubspec.yaml b/pubspec.yaml index 03ace49b..ee28a05a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,5 +1,5 @@ name: mongo_dart -version: 0.9.0 +version: 0.9.1 description: MongoDB driver, implemented in pure Dart. All CRUD operations, aggregation pipeline and more! homepage: https://github.com/mongo-dart/mongo_dart diff --git a/test/file_ext_test.dart b/test/file_ext_test.dart new file mode 100644 index 00000000..4f454c07 --- /dev/null +++ b/test/file_ext_test.dart @@ -0,0 +1,91 @@ +import 'dart:io'; + +import 'package:mongo_dart/src/extensions/file_ext.dart'; +import 'package:path/path.dart'; +import 'package:test/test.dart'; +import 'package:uuid/uuid.dart'; + +Future main() async { + var dir = Directory(current); + var filename = Uuid().v4(); + var file = File('${dir.path}${Platform.pathSeparator}$filename'); + await file.writeAsString('Test'); + group('Base methods', () { + test('Name', () { + expect(file.name, filename); + }); + test('New Path by Name', () { + expect(file.newPathByName('prova'), + '${dir.path}${Platform.pathSeparator}prova'); + }); + test('Safe Path', () async { + var newPath = await file.safePath; + expect(newPath, '${dir.path}${Platform.pathSeparator}$filename(1)'); + }); + test('To Safe Path', () async { + var newPath = await file.toSafePath(file.newPathByName('prova')); + expect(newPath, '${dir.path}${Platform.pathSeparator}prova'); + }); + }); + + group('Change File Name', () { + test('Change File Name Only', () async { + var changedFile = await file.copy(file.newPathByName('chgf')); + var newFile = await changedFile.changeFileNameOnly('testo'); + var name = newFile.name; + await newFile.delete(); + expect(name, 'testo'); + }); + + test('Change File Name Only - two tries', () async { + var changedFile = await file.copy(file.newPathByName('chgf2')); + var newFile = await changedFile.changeFileNameOnly('test2'); + changedFile = await file.copy(file.newPathByName('chgf2')); + var newFile2 = await changedFile.changeFileNameOnly('test2'); + var name = newFile2.name; + var exists = await changedFile.exists(); + var exists2 = await newFile2.exists(); + + await newFile2.delete(); + expect(name, 'test2'); + expect(exists, isFalse); + expect(exists2, isTrue); + }); + }); + group('Safe Change File Name', () { + test('Safe Change File Name Only', () async { + var changedFile = await file.copy(file.newPathByName('chgf4')); + var newFile = await changedFile.changeFileNameOnlySafe('test4.ts'); + var name = newFile.name; + await newFile.delete(); + expect(name, 'test4.ts'); + }); + + test('Change File Name Only - two tries', () async { + var changedFile = await file.copy(file.newPathByName('chgf3')); + var newFile = await changedFile.changeFileNameOnlySafe('test3.1.ts'); + changedFile = await file.copy(file.newPathByName('chgf3')); + var newFile2 = await changedFile.changeFileNameOnlySafe('test3.1.ts'); + var name = newFile.name; + var name2 = newFile2.name; + + var exists = await changedFile.exists(); + var exists2 = await newFile.exists(); + var exists3 = await newFile2.exists(); + + await newFile.delete(); + await newFile2.delete(); + + expect(name, 'test3.1.ts'); + expect(name2, 'test3.1(1).ts'); + + expect(exists, isFalse); + expect(exists2, isTrue); + expect(exists3, isTrue); + }); + }); + + tearDownAll(() async { + await file.delete(); + }); +}