Add creation expiry time

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp

@@ -23,6 +23,60 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     using SignedIdentifier = Blobs::Models::SignedIdentifier;
     using FileQueryArrowField = Blobs::Models::BlobQueryArrowField;
     using FileQueryArrowFieldType = Blobs::Models::BlobQueryArrowFieldType;
+
+    /**
+     * @brief An access control object.
+     */
+    struct Acl final
+    {
+      /**
+       * The scope of the ACL.
+       */
+      std::string Scope;
+
+      /**
+       * The type of the ACL.
+       */
+      std::string Type;
+
+      /**
+       * The ID of the ACL.
+       */
+      std::string Id;
+
+      /**
+       * The permissions of the ACL.
+       */
+      std::string Permissions;
+
+      /**
+       * @brief Creates an Acl based on acl input string.
+       * @param aclString the string to be parsed to Acl.
+       * @return Acl
+       */
+      static Acl FromString(const std::string& aclString);
+
+      /**
+       * @brief Creates a string from an Acl.
+       * @param acl the acl object to be serialized to a string.
+       * @return std::string
+       */
+      static std::string ToString(const Acl& acl);
+
+      /**
+       * @brief Creates a vector of Acl from a string that indicates multiple acls.
+       * @param aclsString the string that contains multiple acls.
+       * @return std::vector<Acl>
+       */
+      static std::vector<Acl> DeserializeAcls(const std::string& aclsString);
+
+      /**
+       * @brief Creates a string that contains several Acls.
+       * @param aclsArray the acls to be serialized into a string.
+       * @return std::string
+       */
+      static std::string SerializeAcls(const std::vector<Acl>& aclsArray);
+    };
   } // namespace Models
 
   using DownloadFileToOptions = Blobs::DownloadBlobToOptions;
@@ -259,6 +313,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
      * Specify the lease access conditions.
      */
     LeaseAccessConditions AccessConditions;
+
+    /**
+     * If true, the file will be flushed after the append.
+     */
+    Azure::Nullable<bool> Flush;
   };
 
   /**
@@ -373,6 +432,29 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     PathAccessConditions AccessConditions;
   };
 
+  using ScheduleFileExpiryOriginType = Blobs::Models::ScheduleBlobExpiryOriginType;
+
+  /**
+   * @brief Optional parameters for #Azure::Storage::Files::DataLake::FileClient::UploadFrom.
+   */
+  struct ScheduleFileDeletionOptions final
+  {
+    /**
+     * The expiry time from the specified origin. Only work if ExpiryOrigin is
+     * ScheduleFileExpiryOriginType::RelativeToCreation or
+     * ScheduleFileExpiryOriginType::RelativeToNow.
+     */
+    Azure::Nullable<std::chrono::milliseconds> TimeToExpire;
+
+    /**
+     * The expiry time in RFC1123 format. Only work if ExpiryOrigin is
+     * ScheduleFileExpiryOriginType::Absolute.
+     */
+    Azure::Nullable<DateTime> ExpiresOn;
+  };
+
+  using SchedulePathDeletionOptions = ScheduleFileDeletionOptions;
+
   /**
    * @brief Optional parameters for #Azure::Storage::Files::DataLake::PathClient::Create.
    * @remark Some optional parameter is mandatory in certain combination.
@@ -414,6 +496,39 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
      */
     Azure::Nullable<std::string> Permissions;
 
+    /**
+     * The owner of the file or directory.
+     */
+    Azure::Nullable<std::string> Owner;
+
+    /**
+     * The owning group of the file or directory.
+     */
+    Azure::Nullable<std::string> Group;
+
+    /**
+     * Sets POSIX access control rights on files and directories. Each access control entry (ACE)
+     * consists of a scope, a type, a user or group identifier, and permissions.
+     */
+    Azure::Nullable<std::vector<Models::Acl>> Acls;
+
+    /**
+     * Proposed LeaseId.
+     */
+    Azure::Nullable<std::string> LeaseId;
+
+    /**
+     * Specifies the duration of the lease, in seconds, or InfiniteLeaseDuration for a lease that
+     * never expires. A non-infinite lease can be between 15 and 60 seconds. A lease duration cannot
+     * be changed using renew or change. Does not apply to directories.
+     */
+    Azure::Nullable<std::chrono::seconds> LeaseDuration;
+
+    /**
+     * Optional parameters to schedule the file for deletion.
+     */
+    SchedulePathDeletionOptions ScheduleDeletionOptions;
+
     /**
      * Specify the access condition for the path.
      */
@@ -613,27 +728,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     } TransferOptions;
   };
 
-  using ScheduleFileExpiryOriginType = Blobs::Models::ScheduleBlobExpiryOriginType;
-
-  /**
-   * @brief Optional parameters for #Azure::Storage::Files::DataLake::FileClient::UploadFrom.
-   */
-  struct ScheduleFileDeletionOptions final
-  {
-    /**
-     * The expiry time from the specified origin. Only work if ExpiryOrigin is
-     * ScheduleFileExpiryOriginType::RelativeToCreation or
-     * ScheduleFileExpiryOriginType::RelativeToNow.
-     */
-    Azure::Nullable<std::chrono::milliseconds> TimeToExpire;
-
-    /**
-     * The expiry time in RFC1123 format. Only work if ExpiryOrigin is
-     * ScheduleFileExpiryOriginType::Absolute.
-     */
-    Azure::Nullable<DateTime> ExpiresOn;
-  };
-
   using AcquireLeaseOptions = Blobs::AcquireLeaseOptions;
   using BreakLeaseOptions = Blobs::BreakLeaseOptions;
   using RenewLeaseOptions = Blobs::RenewLeaseOptions;

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp

@@ -193,60 +193,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     using RehydratePriority = Blobs::Models::RehydratePriority;
     using ArchiveStatus = Blobs::Models::ArchiveStatus;
 
-    /**
-     * @brief An access control object.
-     */
-    struct Acl final
-    {
-      /**
-       * The scope of the ACL.
-       */
-      std::string Scope;
-
-      /**
-       * The type of the ACL.
-       */
-      std::string Type;
-
-      /**
-       * The ID of the ACL.
-       */
-      std::string Id;
-
-      /**
-       * The permissions of the ACL.
-       */
-      std::string Permissions;
-
-      /**
-       * @brief Creates an Acl based on acl input string.
-       * @param aclString the string to be parsed to Acl.
-       * @return Acl
-       */
-      static Acl FromString(const std::string& aclString);
-
-      /**
-       * @brief Creates a string from an Acl.
-       * @param acl the acl object to be serialized to a string.
-       * @return std::string
-       */
-      static std::string ToString(const Acl& acl);
-
-      /**
-       * @brief Creates a vector of Acl from a string that indicates multiple acls.
-       * @param aclsString the string that contains multiple acls.
-       * @return std::vector<Acl>
-       */
-      static std::vector<Acl> DeserializeAcls(const std::string& aclsString);
-
-      /**
-       * @brief Creates a string that contains several Acls.
-       * @param aclsArray the acls to be serialized into a string.
-       * @return std::string
-       */
-      static std::string SerializeAcls(const std::vector<Acl>& aclsArray);
-    };
-
     /**
      * @brief The properties of the path.
      */

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp

@@ -96,6 +96,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        * The name of the encryption scope under which the blob is encrypted.
        */
       Nullable<std::string> EncryptionScope;
+      Nullable<DateTime> CreatedOn;
+      Nullable<DateTime> ExpiresOn;
       std::string ETag;
     };
     namespace _detail {
@@ -379,6 +381,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
         Nullable<std::string> Acl;
         Nullable<std::string> ProposedLeaseId;
         Nullable<int64_t> LeaseDuration;
+        Nullable<std::string> ExpiryOptions;
         Nullable<std::string> ExpiresOn;
       };
       static Response<Models::CreatePathResult> Create(

sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp

@@ -81,6 +81,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
       }
     }
     protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
+    protocolLayerOptions.Flush = options.Flush;
     return _detail::FileClient::Append(
         *m_pipeline, m_pathUrl, content, protocolLayerOptions, context);
   }

sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp

@@ -202,6 +202,31 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     protocolLayerOptions.Properties = _detail::SerializeMetadata(options.Metadata);
     protocolLayerOptions.Umask = options.Umask;
     protocolLayerOptions.Permissions = options.Permissions;
+    protocolLayerOptions.Owner = options.Owner;
+    protocolLayerOptions.Group = options.Group;
+    protocolLayerOptions.ProposedLeaseId = options.LeaseId;
+    if (options.Acls.HasValue())
+    {
+      protocolLayerOptions.Acl = Models::Acl::SerializeAcls(options.Acls.Value());
+    }
+    if (options.LeaseDuration.HasValue())
+    {
+      protocolLayerOptions.LeaseDuration = static_cast<int64_t>(options.LeaseDuration->count());
+    }
+    if (options.ScheduleDeletionOptions.ExpiresOn.HasValue())
+    {
+      protocolLayerOptions.ExpiryOptions
+          = Files::DataLake::ScheduleFileExpiryOriginType::Absolute.ToString();
+      protocolLayerOptions.ExpiresOn = options.ScheduleDeletionOptions.ExpiresOn.Value().ToString(
+          Azure::DateTime::DateFormat::Rfc1123);
+    }
+    else if (options.ScheduleDeletionOptions.TimeToExpire.HasValue())
+    {
+      protocolLayerOptions.ExpiryOptions
+          = Files::DataLake::ScheduleFileExpiryOriginType::RelativeToNow.ToString();
+      protocolLayerOptions.ExpiresOn
+          = std::to_string(options.ScheduleDeletionOptions.TimeToExpire.Value().count());
+    }
     return _detail::PathClient::Create(*m_pipeline, m_pathUrl, protocolLayerOptions, context);
   }
 

sdk/storage/azure-storage-files-datalake/src/rest_client.cpp

@@ -114,6 +114,18 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
           {
             vectorElement2.EncryptionScope = var0["EncryptionScope"].get<std::string>();
           }
+          if (var0.count("creationTime") != 0)
+          {
+            vectorElement2.CreatedOn
+                = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime(
+                    std::stoll(var0["creationTime"].get<std::string>()));
+          }
+          if (var0.count("expiryTime") != 0)
+          {
+            vectorElement2.ExpiresOn
+                = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime(
+                    std::stoll(var0["expiryTime"].get<std::string>()));
+          }
           if (var0.count("etag") != 0)
           {
             vectorElement2.ETag = var0["etag"].get<std::string>();
@@ -274,6 +286,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
       {
         request.SetHeader("x-ms-lease-duration", std::to_string(options.LeaseDuration.Value()));
       }
+      if (options.ExpiryOptions.HasValue() && !options.ExpiryOptions.Value().empty())
+      {
+        request.SetHeader("x-ms-expiry-option", options.ExpiryOptions.Value());
+      }
       if (options.ExpiresOn.HasValue() && !options.ExpiresOn.Value().empty())
       {
         request.SetHeader("x-ms-expiry-time", options.ExpiresOn.Value());

sdk/storage/azure-storage-files-datalake/swagger/README.md

@@ -231,8 +231,8 @@ directive:
       $.Path.properties["contentLength"]["x-ms-client-name"] = "FileSize";
       $.Path.properties["isDirectory"]["x-ms-client-default"] = false;
       $.Path.properties["EncryptionScope"]["x-nullable"] = true;
-      delete $.Path.properties["creationTime"];
-      delete $.Path.properties["expiryTime"];
+      $.Path.properties["creationTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "CreatedOn", "x-nullable": true};
+      $.Path.properties["expiryTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "ExpiresOn", "x-nullable": true};
       $.Path.properties["etag"] = {"type": "string", "x-ms-format": "string", "x-ms-client-default": "", "x-ms-client-name": "ETag"};
       delete $.Path.properties["eTag"];
       $.PathList["x-namespace"] = "_detail";
@@ -250,10 +250,6 @@ directive:
 
 ```yaml
 directive:
-  - from: swagger-document
-    where: $["x-ms-paths"]["/{filesystem}/{path}"].put.parameters
-    transform: >
-      $ = $.filter(p => !(p["$ref"] && (p["$ref"].endsWith("#/parameters/PathExpiryOptionsOptional"))));
   - from: swagger-document
     where: $["x-ms-paths"]["/{filesystem}/{path}"].put.responses
     transform: >
@@ -281,7 +277,9 @@ directive:
     where: $.parameters
     transform: >
       delete $["PathExpiryOptions"];
-      delete $["PathExpiryOptionsOptional"];
+      delete $.PathExpiryOptionsOptional["enum"];
+      delete $.PathExpiryOptionsOptional["x-ms-enum"];
+
 ```
 
 ### DeletePath

sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp

@@ -245,6 +245,22 @@ namespace Azure { namespace Storage { namespace Test {
     EXPECT_EQ(buffer, downloaded);
   }
 
+  TEST_F(DataLakeFileClientTest, AppendFileWithFlush)
+  {
+    const int32_t bufferSize = 4 * 1024; // 4KB data size
+    std::vector<uint8_t> buffer(bufferSize, 'x');
+    auto bufferStream = std::make_unique<Azure::Core::IO::MemoryBodyStream>(
+        Azure::Core::IO::MemoryBodyStream(buffer));
+    auto properties1 = m_fileClient->GetProperties();
+
+    // Append with flush option
+    Files::DataLake::AppendFileOptions options;
+    options.Flush = true;
+    m_fileClient->Append(*bufferStream, 0, options);
+    auto properties2 = m_fileClient->GetProperties();
+    EXPECT_NE(properties1.Value.ETag, properties2.Value.ETag);
+  }
+
   TEST_F(DataLakeFileClientTest, FileReadReturns)
   {
     const int32_t bufferSize = 4 * 1024; // 4KB data size

sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp

@@ -289,6 +289,8 @@ namespace Azure { namespace Storage { namespace Test {
         EXPECT_NE(result.end(), iter);
         EXPECT_EQ(iter->Name, name);
         EXPECT_EQ(iter->Name.substr(0U, m_directoryA.size()), m_directoryA);
+        EXPECT_TRUE(iter->CreatedOn.HasValue());
+        EXPECT_TRUE(iter->ExpiresOn.HasValue());
       }
       for (const auto& name : m_pathNameSetB)
       {
@@ -299,6 +301,8 @@ namespace Azure { namespace Storage { namespace Test {
         EXPECT_NE(result.end(), iter);
         EXPECT_EQ(iter->Name, name);
         EXPECT_EQ(iter->Name.substr(0U, m_directoryB.size()), m_directoryB);
+        EXPECT_TRUE(iter->CreatedOn.HasValue());
+        EXPECT_TRUE(iter->ExpiresOn.HasValue());
       }
     }
     {