Storage Transfer Manager: Job Engine Implementation (Azure#3742)

sdk/storage/azure-storage-datamovement/CMakeLists.txt

@@ -43,33 +43,35 @@ set(
   inc/azure/storage/datamovement/blob_folder.hpp
   inc/azure/storage/datamovement/blob_transfer_manager.hpp
   inc/azure/storage/datamovement/datamovement_options.hpp
-  inc/azure/storage/datamovement/directory_iterator.hpp
   inc/azure/storage/datamovement/dll_import_export.hpp
+  inc/azure/storage/datamovement/filesystem.hpp
+  inc/azure/storage/datamovement/job_engine.hpp
   inc/azure/storage/datamovement/job_properties.hpp
   inc/azure/storage/datamovement/rtti.hpp
-  inc/azure/storage/datamovement/scheduler.hpp
   inc/azure/storage/datamovement/storage_transfer_manager.hpp
   inc/azure/storage/datamovement/task.hpp
-  inc/azure/storage/datamovement/tasks/download_blob_directory_task.hpp
+  inc/azure/storage/datamovement/task_shared_status.hpp
   inc/azure/storage/datamovement/tasks/download_blob_to_file_task.hpp
   inc/azure/storage/datamovement/tasks/upload_blob_from_file_task.hpp
-  inc/azure/storage/datamovement/tasks/upload_blobs_from_directory_task.hpp
+  inc/azure/storage/datamovement/transfer_engine.hpp
   inc/azure/storage/datamovement/utilities.hpp
 )
 
 set(
   AZURE_STORAGE_DATAMOVEMENT_SOURCE
   src/blob_folder.cpp
   src/blob_transfer_manager.cpp
-  src/directory_iterator.cpp
+  src/filesystem.cpp
+  src/job_engine.cpp
+  src/job_engine_serialization.cpp
+  src/job_engine_task_generation.cpp
   src/job_properties.cpp
   src/private/package_version.hpp
-  src/scheduler.cpp
   src/storage_transfer_manager.cpp
-  src/tasks/download_blob_directory_task.cpp
+  src/task_shared_status.cpp
   src/tasks/download_blob_to_file_task.cpp
   src/tasks/upload_blob_from_file_task.cpp
-  src/tasks/upload_blobs_from_directory_task.cpp
+  src/transfer_engine.cpp
   src/utilities.cpp
 )
 

sdk/storage/azure-storage-datamovement/inc/azure/storage/datamovement/blob_folder.hpp

@@ -7,26 +7,29 @@
 
 #include <azure/storage/blobs.hpp>
 
-namespace Azure { namespace Storage { namespace Blobs {
-  namespace _detail {
-    struct DownloadBlobDirectoryTask;
+namespace Azure { namespace Storage {
+
+  namespace _internal {
+    class TransferEnd;
   }
+  namespace Blobs {
 
-  class BlobFolder final {
-  public:
-    explicit BlobFolder(BlobContainerClient blobContainerClient, std::string folderPath);
+    class BlobFolder final {
+    public:
+      explicit BlobFolder(BlobContainerClient blobContainerClient, std::string folderPath);
 
-    std::string GetUrl() const;
+      std::string GetUrl() const;
 
-    BlobFolder GetBlobFolder(const std::string& folderName) const;
+      BlobFolder GetBlobFolder(const std::string& folderName) const;
 
-    BlobClient GetBlobClient(const std::string& blobName) const;
+      BlobClient GetBlobClient(const std::string& blobName) const;
 
-  private:
-    BlobContainerClient m_blobContainerClient;
-    std::string m_folderPath;
+    private:
+      BlobContainerClient m_blobContainerClient;
+      std::string m_folderPath;
 
-    friend struct _detail::DownloadBlobDirectoryTask;
-  };
+      friend class _internal::TransferEnd;
+    };
 
-}}} // namespace Azure::Storage::Blobs
+  } // namespace Blobs
+}} // namespace Azure::Storage

sdk/storage/azure-storage-datamovement/inc/azure/storage/datamovement/datamovement_options.hpp

@@ -13,14 +13,13 @@
 namespace Azure { namespace Storage {
   struct StorageTransferManagerOptions final
   {
-    std::string TransferStateDirectoryPath;
+    std::string TransferStateDirectoryPath = ".";
     Nullable<int> NumThreads;
     Nullable<size_t> MaxMemorySize;
   };
 
   struct TransferCredential final
   {
-    std::string ConnectionString;
     std::string SasCredential; // TODO: or Azure SAS credential?
     std::shared_ptr<StorageSharedKeyCredential> SharedKeyCredential;
     std::shared_ptr<Core::Credentials::TokenCredential> TokenCredential;
@@ -36,7 +35,6 @@ namespace Azure { namespace Storage {
 
   struct TransferError final
   {
-    std::string JobId;
     std::string SourceUrl;
     std::string DestinationUrl;
     // TODO: error information: std::exception_ptr or descriptive string?

sdk/storage/azure-storage-datamovement/inc/azure/storage/datamovement/filesystem.hpp

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace Azure { namespace Storage { namespace _internal {
+
+  bool IsDirectory(const std::string& path);
+  bool IsRegularFile(const std::string& path);
+  bool PathExists(const std::string& path);
+  void CreateDirectory(const std::string& path);
+  void Rename(const std::string& oldPath, const std::string& newPath);
+  void Remove(const std::string& path);
+  int64_t GetFileSize(const std::string& path);
+
+  class DirectoryIterator final {
+  public:
+    struct DirectoryEntry final
+    {
+      std::string Name;
+      bool IsDirectory = false;
+      int64_t Size = -1;
+    };
+    explicit DirectoryIterator(const std::string& rootDirectory);
+    DirectoryIterator(const DirectoryIterator&) = delete;
+    DirectoryIterator& operator=(const DirectoryIterator&) = delete;
+    ~DirectoryIterator();
+
+    DirectoryEntry Next();
+
+  private:
+    std::string m_rootDirectory;
+    void* m_directroyObject;
+  };
+
+  class MemoryMap final {
+  public:
+    explicit MemoryMap(const std::string& filename);
+    MemoryMap(const MemoryMap&) = delete;
+    MemoryMap(MemoryMap&& other) noexcept { *this = std::move(other); }
+    MemoryMap& operator=(const MemoryMap&) = delete;
+    MemoryMap& operator=(MemoryMap&& other) noexcept;
+    ~MemoryMap();
+
+    void* Map(size_t offset, size_t size);
+
+  private:
+    void* m_fileHandle = nullptr;
+    std::vector<std::pair<void*, size_t>> m_mapped;
+  };
+
+}}} // namespace Azure::Storage::_internal