Create Storage DataMovement directory (Azure#3594)

cmake-modules/FolderList.cmake

@@ -27,6 +27,9 @@ macro(GetFolderList project)
     elseif(${project} STREQUAL STORAGE_QUEUES)
         DownloadDepVersion(sdk/core azure-core 1.5.0)
         DownloadDepVersion(sdk/storage/azure-storage-common azure-storage-common 12.2.3)
+    elseif(${project} STREQUAL STORAGE_DATAMOVEMENT)
+        DownloadDepVersion(sdk/core azure-core 1.5.0)
+        DownloadDepVersion(sdk/storage/azure-storage-blobs azure-storage-blobs 12.4.0)
     endif()
     list(REMOVE_DUPLICATES BUILD_FOLDERS)
 endmacro()

sdk/storage/CMakeLists.txt

@@ -10,3 +10,4 @@ add_subdirectory(azure-storage-blobs)
 add_subdirectory(azure-storage-files-datalake)
 add_subdirectory(azure-storage-files-shares)
 add_subdirectory(azure-storage-queues)
+add_subdirectory(azure-storage-datamovement)

sdk/storage/azure-storage-common/inc/azure/storage/common/internal/file_io.hpp

@@ -19,13 +19,16 @@ namespace Azure { namespace Storage { namespace _internal {
   class FileReader final {
   public:
     FileReader(const std::string& filename);
-
+    FileReader(const FileReader&) = delete;
+    FileReader& operator=(const FileReader&) = delete;
     ~FileReader();
 
     FileHandle GetHandle() const { return m_handle; }
 
     int64_t GetFileSize() const { return m_fileSize; }
 
+    size_t Read(uint8_t* buffer, size_t length, int64_t offset) const;
+
   private:
     FileHandle m_handle;
     int64_t m_fileSize;
@@ -34,12 +37,13 @@ namespace Azure { namespace Storage { namespace _internal {
   class FileWriter final {
   public:
     FileWriter(const std::string& filename);
-
+    FileWriter(const FileWriter&) = delete;
+    FileWriter& operator=(const FileWriter&) = delete;
     ~FileWriter();
 
     FileHandle GetHandle() const { return m_handle; }
 
-    void Write(const uint8_t* buffer, size_t length, int64_t offset);
+    void Write(const uint8_t* buffer, size_t length, int64_t offset) const;
 
   private:
     FileHandle m_handle;

sdk/storage/azure-storage-common/src/file_io.cpp

@@ -22,6 +22,7 @@
 #include <windows.h>
 #endif
 
+#include <algorithm>
 #include <limits>
 #include <stdexcept>
 
@@ -87,6 +88,29 @@ namespace Azure { namespace Storage { namespace _internal {
 
   FileReader::~FileReader() { CloseHandle(static_cast<HANDLE>(m_handle)); }
 
+  size_t FileReader::Read(uint8_t* buffer, size_t length, int64_t offset) const
+  {
+    length = std::min(length, static_cast<size_t>(std::max(0LL, m_fileSize - offset)));
+    if (length > std::numeric_limits<DWORD>::max())
+    {
+      throw std::runtime_error("Failed to read file.");
+    }
+
+    OVERLAPPED overlapped;
+    std::memset(&overlapped, 0, sizeof(overlapped));
+    overlapped.Offset = static_cast<DWORD>(static_cast<uint64_t>(offset));
+    overlapped.OffsetHigh = static_cast<DWORD>(static_cast<uint64_t>(offset) >> 32);
+
+    DWORD bytesRead;
+    BOOL ret = ReadFile(
+        static_cast<HANDLE>(m_handle), buffer, static_cast<DWORD>(length), &bytesRead, &overlapped);
+    if (!ret)
+    {
+      throw std::runtime_error("Failed to read file.");
+    }
+    return bytesRead;
+  }
+
   FileWriter::FileWriter(const std::string& filename)
   {
     int sizeNeeded = MultiByteToWideChar(
@@ -138,7 +162,7 @@ namespace Azure { namespace Storage { namespace _internal {
 
   FileWriter::~FileWriter() { CloseHandle(static_cast<HANDLE>(m_handle)); }
 
-  void FileWriter::Write(const uint8_t* buffer, size_t length, int64_t offset)
+  void FileWriter::Write(const uint8_t* buffer, size_t length, int64_t offset) const
   {
     if (length > std::numeric_limits<DWORD>::max())
     {
@@ -180,6 +204,21 @@ namespace Azure { namespace Storage { namespace _internal {
 
   FileReader::~FileReader() { close(m_handle); }
 
+  size_t FileReader::Read(uint8_t* buffer, size_t length, int64_t offset) const
+  {
+    if (offset > static_cast<int64_t>(std::numeric_limits<off_t>::max()))
+    {
+      throw std::runtime_error("Failed to read file.");
+    }
+    length = std::min<size_t>(length, m_fileSize - offset);
+    ssize_t bytesRead = pread(m_handle, buffer, length, static_cast<off_t>(offset));
+    if (bytesRead < 0)
+    {
+      throw std::runtime_error("Failed to read file.");
+    }
+    return bytesRead;
+  }
+
   FileWriter::FileWriter(const std::string& filename)
   {
     m_handle = open(
@@ -192,7 +231,7 @@ namespace Azure { namespace Storage { namespace _internal {
 
   FileWriter::~FileWriter() { close(m_handle); }
 
-  void FileWriter::Write(const uint8_t* buffer, size_t length, int64_t offset)
+  void FileWriter::Write(const uint8_t* buffer, size_t length, int64_t offset) const
   {
     if (offset > static_cast<int64_t>(std::numeric_limits<off_t>::max()))
     {

sdk/storage/azure-storage-datamovement/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Release History
+
+## 12.0.0-beta.1 (Unreleased)
+
+### Features Added
+
+### Breaking Changes
+
+### Bugs Fixed
+
+### Other Changes

sdk/storage/azure-storage-datamovement/CMakeLists.txt

@@ -0,0 +1,101 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: MIT
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake-modules")
+include(AzureVcpkg)
+az_vcpkg_integrate()
+
+cmake_minimum_required (VERSION 3.13)
+project(azure-storage-datamovement LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+
+option(FETCH_SOURCE_DEPS "build source dependencies" OFF)
+
+include(AzureVersion)
+include(AzureCodeCoverage)
+include(AzureTransportAdapters)
+include(AzureDoxygen)
+include(AzureGlobalCompileOptions)
+include(AzureConfigRTTI)
+include(AzureBuildTargetForCI)
+
+if(FETCH_SOURCE_DEPS)
+    set(AZ_ALL_LIBRARIES ON)
+    include(FolderList)
+    SetCompileOptions(STORAGE_DATAMOVEMENT)
+    GetFolderList(STORAGE_DATAMOVEMENT)
+    foreach(oneFolder IN LISTS BUILD_FOLDERS)
+        message("add folder ${oneFolder}")
+        add_subdirectory(${oneFolder} EXCLUDE_FROM_ALL)
+    endforeach()
+elseif(NOT AZ_ALL_LIBRARIES)
+  find_package(azure-storage-blobs-cpp "12.4.0" CONFIG QUIET)
+  if(NOT azure-storage-blobs-cpp_FOUND)
+    find_package(azure-storage-blobs-cpp "12.4.0" REQUIRED)
+  endif()
+endif()
+
+set(
+  AZURE_STORAGE_DATAMOVEMENT_HEADER
+  inc/azure/storage/datamovement/blob_folder.hpp
+  inc/azure/storage/datamovement/datamovement_options.hpp
+  inc/azure/storage/datamovement/dll_import_export.hpp
+  inc/azure/storage/datamovement/job_properties.hpp
+  inc/azure/storage/datamovement/rtti.hpp
+  inc/azure/storage/datamovement/scheduler.hpp
+  inc/azure/storage/datamovement/storage_transfer_manager.hpp
+  inc/azure/storage/datamovement/task.hpp
+  inc/azure/storage/datamovement/tasks/upload_blob_from_file_task.hpp
+  src/private/package_version.hpp
+)
+
+set(
+  AZURE_STORAGE_DATAMOVEMENT_SOURCE
+  src/scheduler.cpp
+  src/storage_transfer_manager.cpp
+  src/tasks/upload_blob_from_file_task.cpp
+)
+
+add_library(azure-storage-datamovement ${AZURE_STORAGE_DATAMOVEMENT_HEADER} ${AZURE_STORAGE_DATAMOVEMENT_SOURCE})
+create_per_service_target_build(storage azure-storage-datamovement)
+
+# make sure that users can consume the project as a library.
+add_library(Azure::azure-storage-datamovement ALIAS azure-storage-datamovement)
+
+target_include_directories(
+  azure-storage-datamovement
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/inc>
+      $<INSTALL_INTERFACE:include>
+)
+
+target_link_libraries(azure-storage-datamovement PUBLIC Azure::azure-storage-blobs)
+
+get_az_version("${CMAKE_CURRENT_SOURCE_DIR}/src/private/package_version.hpp")
+generate_documentation(azure-storage-datamovement ${AZ_LIBRARY_VERSION})
+
+az_vcpkg_export(
+  azure-storage-datamovement
+    STORAGE_DATAMOVEMENT
+    "azure/storage/datamovement/dll_import_export.hpp"
+  )
+
+az_rtti_setup(
+  azure-storage-datamovement
+  STORAGE_DATAMOVEMENT
+  "azure/storage/datamovement/rtti.hpp"
+)
+
+# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF
+create_code_coverage(storage azure-storage-datamovement azure-storage-test "tests?/*;samples?/*")
+
+if(BUILD_TESTING)
+  # add_subdirectory(test/ut)
+endif()
+
+if(BUILD_SAMPLES)
+  add_subdirectory(samples)
+endif()