diff --git a/velox/common/file/FileSystems.cpp b/velox/common/file/FileSystems.cpp index 8c53cb52e5200..c7f72d6b77751 100644 --- a/velox/common/file/FileSystems.cpp +++ b/velox/common/file/FileSystems.cpp @@ -172,7 +172,8 @@ class LocalFileSystem : public FileSystem { return filePaths; } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& /*options*/) + override { std::error_code ec; std::filesystem::create_directories(path, ec); VELOX_CHECK_EQ( diff --git a/velox/common/file/FileSystems.h b/velox/common/file/FileSystems.h index 3f3acc6fcba46..e21b13520f95a 100644 --- a/velox/common/file/FileSystems.h +++ b/velox/common/file/FileSystems.h @@ -67,6 +67,13 @@ struct FileOptions { bool bufferWrite{true}; }; +/// Defines directory options +struct DirectoryOptions : FileOptions { + /// This is similar to kFileCreateConfig + static constexpr folly::StringPiece kMakeDirectoryConfig{ + "make-directory-config"}; +}; + // Metrics structure to support file system metrics. struct FileSystemMetrics { virtual uint64_t getActiveConnections() const = 0; @@ -127,7 +134,9 @@ class FileSystem { virtual std::vector list(std::string_view path) = 0; /// Create a directory (recursively). Throws velox exception on failure. - virtual void mkdir(std::string_view path) = 0; + virtual void mkdir( + std::string_view path, + const DirectoryOptions& options = {}) = 0; /// Remove a directory (all the files and sub-directories underneath /// recursively). Throws velox exception on failure. diff --git a/velox/common/file/tests/FaultyFileSystem.cpp b/velox/common/file/tests/FaultyFileSystem.cpp index 39b4f6b09e158..6e346af61731c 100644 --- a/velox/common/file/tests/FaultyFileSystem.cpp +++ b/velox/common/file/tests/FaultyFileSystem.cpp @@ -112,9 +112,11 @@ std::vector FaultyFileSystem::list(std::string_view path) { return files; } -void FaultyFileSystem::mkdir(std::string_view path) { +void FaultyFileSystem::mkdir( + std::string_view path, + const DirectoryOptions& options) { const auto delegatedDirPath = extractPath(path); - getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath); + getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath, options); } void FaultyFileSystem::rmdir(std::string_view path) { diff --git a/velox/common/file/tests/FaultyFileSystem.h b/velox/common/file/tests/FaultyFileSystem.h index 9d8aeb3dda24d..ae3e876c00f76 100644 --- a/velox/common/file/tests/FaultyFileSystem.h +++ b/velox/common/file/tests/FaultyFileSystem.h @@ -96,7 +96,7 @@ class FaultyFileSystem : public FileSystem { std::vector list(std::string_view path) override; - void mkdir(std::string_view path) override; + void mkdir(std::string_view path, const DirectoryOptions& options) override; void rmdir(std::string_view path) override; diff --git a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h index 319a85b7a382c..903aa80226065 100644 --- a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h @@ -66,7 +66,9 @@ class AbfsFileSystem : public FileSystem { VELOX_UNSUPPORTED("list for abfs not implemented"); } - void mkdir(std::string_view path) override { + void mkdir( + std::string_view path, + const filesystems::DirectoryOptions& options = {}) override { VELOX_UNSUPPORTED("mkdir for abfs not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp index 6bc7aaf0cb82a..f5d0830d3de35 100644 --- a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp +++ b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp @@ -433,7 +433,9 @@ void GcsFileSystem::rename(std::string_view, std::string_view, bool) { VELOX_UNSUPPORTED("rename for GCS not implemented"); } -void GcsFileSystem::mkdir(std::string_view path) { +void GcsFileSystem::mkdir( + std::string_view path, + const DirectoryOptions& options) { VELOX_UNSUPPORTED("mkdir for GCS not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h index 99e4f94ab99ef..34daff8d6c64f 100644 --- a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h @@ -81,7 +81,8 @@ class GcsFileSystem : public FileSystem { void rename(std::string_view, std::string_view, bool) override; /// Unsupported - void mkdir(std::string_view path) override; + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override; /// Unsupported void rmdir(std::string_view path) override; diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h index 1ae7d28b94e5f..a3e9a8efe5d10 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h @@ -78,7 +78,8 @@ class HdfsFileSystem : public FileSystem { VELOX_UNSUPPORTED("list for HDFS not implemented"); } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override { VELOX_UNSUPPORTED("mkdir for HDFS not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h index cb9b32ca9e449..3471ea94caaa1 100644 --- a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h +++ b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h @@ -64,7 +64,8 @@ class S3FileSystem : public FileSystem { VELOX_UNSUPPORTED("list for S3 not implemented"); } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override { VELOX_UNSUPPORTED("mkdir for S3 not implemented"); }