Add creation expiry time

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp

@@ -24,6 +24,60 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     using FileQueryArrowField = Blobs::Models::BlobQueryArrowField;
     using FileQueryArrowFieldType = Blobs::Models::BlobQueryArrowFieldType;
     using EncryptionAlgorithmType = Blobs::Models::EncryptionAlgorithmType;
+
+    /**
+     * @brief An access control object.
+     */
+    struct Acl final
+    {
+      /**
+       * The scope of the ACL.
+       */
+      std::string Scope;
+
+      /**
+       * The type of the ACL.
+       */
+      std::string Type;
+
+      /**
+       * The ID of the ACL.
+       */
+      std::string Id;
+
+      /**
+       * The permissions of the ACL.
+       */
+      std::string Permissions;
+
+      /**
+       * @brief Creates an Acl based on acl input string.
+       * @param aclString the string to be parsed to Acl.
+       * @return Acl
+       */
+      static Acl FromString(const std::string& aclString);
+
+      /**
+       * @brief Creates a string from an Acl.
+       * @param acl the acl object to be serialized to a string.
+       * @return std::string
+       */
+      static std::string ToString(const Acl& acl);
+
+      /**
+       * @brief Creates a vector of Acl from a string that indicates multiple acls.
+       * @param aclsString the string that contains multiple acls.
+       * @return std::vector<Acl>
+       */
+      static std::vector<Acl> DeserializeAcls(const std::string& aclsString);
+
+      /**
+       * @brief Creates a string that contains several Acls.
+       * @param aclsArray the acls to be serialized into a string.
+       * @return std::string
+       */
+      static std::string SerializeAcls(const std::vector<Acl>& aclsArray);
+    };
   } // namespace Models
 
   using DownloadFileToOptions = Blobs::DownloadBlobToOptions;
@@ -266,6 +320,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
      * Specify the lease access conditions.
      */
     LeaseAccessConditions AccessConditions;
+
+    /**
+     * If true, the file will be flushed after the append.
+     */
+    Azure::Nullable<bool> Flush;
   };
 
   /**
@@ -380,6 +439,33 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     PathAccessConditions AccessConditions;
   };
 
+  using ScheduleFileExpiryOriginType = Blobs::Models::ScheduleBlobExpiryOriginType;
+
+  /**
+   * @brief Options for scheduling the deletion of a path.
+   */
+  struct ScheduleFileDeletionOptions final
+  {
+    /**
+     * The expiry time from the specified origin. Only work if ExpiryOrigin is
+     * ScheduleFileExpiryOriginType::RelativeToCreation or
+     * ScheduleFileExpiryOriginType::RelativeToNow.
+     * Does not apply to directories.
+     * TimeToExpire and ExpiresOn cannot both be set.
+     */
+    Azure::Nullable<std::chrono::milliseconds> TimeToExpire;
+
+    /**
+     * The expiry time in RFC1123 format. Only works if ExpiryOrigin is
+     * ScheduleFileExpiryOriginType::Absolute.
+     * Does not apply to directories.
+     * ExpiresOn and TimeToExpire cannot both be set.
+     */
+    Azure::Nullable<DateTime> ExpiresOn;
+  };
+
+  using SchedulePathDeletionOptions = ScheduleFileDeletionOptions;
+
   /**
    * @brief Optional parameters for #Azure::Storage::Files::DataLake::PathClient::Create.
    * @remark Some optional parameter is mandatory in certain combination.
@@ -421,6 +507,39 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
      */
     Azure::Nullable<std::string> Permissions;
 
+    /**
+     * The owner of the file or directory.
+     */
+    Azure::Nullable<std::string> Owner;
+
+    /**
+     * The owning group of the file or directory.
+     */
+    Azure::Nullable<std::string> Group;
+
+    /**
+     * Sets POSIX access control rights on files and directories. Each access control entry (ACE)
+     * consists of a scope, a type, a user or group identifier, and permissions.
+     */
+    Azure::Nullable<std::vector<Models::Acl>> Acls;
+
+    /**
+     * Proposed LeaseId.
+     */
+    Azure::Nullable<std::string> LeaseId;
+
+    /**
+     * Specifies the duration of the lease, in seconds, or InfiniteLeaseDuration for a lease that
+     * never expires. A non-infinite lease can be between 15 and 60 seconds. A lease duration cannot
+     * be changed using renew or change. Does not apply to directories.
+     */
+    Azure::Nullable<std::chrono::seconds> LeaseDuration;
+
+    /**
+     * Optional parameters to schedule the file for deletion.
+     */
+    SchedulePathDeletionOptions ScheduleDeletionOptions;
+
     /**
      * Specify the access condition for the path.
      */
@@ -620,27 +739,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     } TransferOptions;
   };
 
-  using ScheduleFileExpiryOriginType = Blobs::Models::ScheduleBlobExpiryOriginType;
-
-  /**
-   * @brief Optional parameters for #Azure::Storage::Files::DataLake::FileClient::UploadFrom.
-   */
-  struct ScheduleFileDeletionOptions final
-  {
-    /**
-     * The expiry time from the specified origin. Only work if ExpiryOrigin is
-     * ScheduleFileExpiryOriginType::RelativeToCreation or
-     * ScheduleFileExpiryOriginType::RelativeToNow.
-     */
-    Azure::Nullable<std::chrono::milliseconds> TimeToExpire;
-
-    /**
-     * The expiry time in RFC1123 format. Only work if ExpiryOrigin is
-     * ScheduleFileExpiryOriginType::Absolute.
-     */
-    Azure::Nullable<DateTime> ExpiresOn;
-  };
-
   using AcquireLeaseOptions = Blobs::AcquireLeaseOptions;
   using BreakLeaseOptions = Blobs::BreakLeaseOptions;
   using RenewLeaseOptions = Blobs::RenewLeaseOptions;

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp

@@ -193,60 +193,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     using RehydratePriority = Blobs::Models::RehydratePriority;
     using ArchiveStatus = Blobs::Models::ArchiveStatus;
 
-    /**
-     * @brief An access control object.
-     */
-    struct Acl final
-    {
-      /**
-       * The scope of the ACL.
-       */
-      std::string Scope;
-
-      /**
-       * The type of the ACL.
-       */
-      std::string Type;
-
-      /**
-       * The ID of the ACL.
-       */
-      std::string Id;
-
-      /**
-       * The permissions of the ACL.
-       */
-      std::string Permissions;
-
-      /**
-       * @brief Creates an Acl based on acl input string.
-       * @param aclString the string to be parsed to Acl.
-       * @return Acl
-       */
-      static Acl FromString(const std::string& aclString);
-
-      /**
-       * @brief Creates a string from an Acl.
-       * @param acl the acl object to be serialized to a string.
-       * @return std::string
-       */
-      static std::string ToString(const Acl& acl);
-
-      /**
-       * @brief Creates a vector of Acl from a string that indicates multiple acls.
-       * @param aclsString the string that contains multiple acls.
-       * @return std::vector<Acl>
-       */
-      static std::vector<Acl> DeserializeAcls(const std::string& aclsString);
-
-      /**
-       * @brief Creates a string that contains several Acls.
-       * @param aclsArray the acls to be serialized into a string.
-       * @return std::string
-       */
-      static std::string SerializeAcls(const std::vector<Acl>& aclsArray);
-    };
-
     /**
      * @brief The properties of the path.
      */

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp

@@ -96,6 +96,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        * The name of the encryption scope under which the blob is encrypted.
        */
       Nullable<std::string> EncryptionScope;
+      Nullable<DateTime> CreatedOn;
+      Nullable<DateTime> ExpiresOn;
       std::string ETag;
     };
     namespace _detail {
@@ -380,6 +382,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
         Nullable<std::string> Acl;
         Nullable<std::string> ProposedLeaseId;
         Nullable<int64_t> LeaseDuration;
+        Nullable<std::string> ExpiryOptions;
         Nullable<std::string> ExpiresOn;
       };
       static Response<Models::CreatePathResult> Create(

sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp

@@ -81,6 +81,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
       }
     }
     protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
+    protocolLayerOptions.Flush = options.Flush;
     if (m_customerProvidedKey.HasValue())
     {
       protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;

sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp

@@ -276,6 +276,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
           protocolLayerOptionsCopy,
           _internal::WithReplicaStatus(context));
 
+      for (auto& path : response.Value.Paths)
+      {
+        if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() == DateTime(1601))
+        {
+          path.ExpiresOn.Reset();
+        }
+      }
+
       ListPathsPagedResponse pagedResponse;
       pagedResponse.Paths = std::move(response.Value.Paths);
       pagedResponse.m_onNextPageFunc = func;

sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp

@@ -205,6 +205,35 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     protocolLayerOptions.Properties = _detail::SerializeMetadata(options.Metadata);
     protocolLayerOptions.Umask = options.Umask;
     protocolLayerOptions.Permissions = options.Permissions;
+    protocolLayerOptions.Owner = options.Owner;
+    protocolLayerOptions.Group = options.Group;
+    protocolLayerOptions.ProposedLeaseId = options.LeaseId;
+    if (options.Acls.HasValue())
+    {
+      protocolLayerOptions.Acl = Models::Acl::SerializeAcls(options.Acls.Value());
+    }
+    if (options.LeaseDuration.HasValue())
+    {
+      protocolLayerOptions.LeaseDuration = static_cast<int64_t>(options.LeaseDuration->count());
+    }
+    AZURE_ASSERT_MSG(
+        !(options.ScheduleDeletionOptions.ExpiresOn.HasValue()
+          && options.ScheduleDeletionOptions.TimeToExpire.HasValue()),
+        "ExpiresOn and TimeToExpire are mutually exclusive.");
+    if (options.ScheduleDeletionOptions.ExpiresOn.HasValue())
+    {
+      protocolLayerOptions.ExpiryOptions
+          = Files::DataLake::ScheduleFileExpiryOriginType::Absolute.ToString();
+      protocolLayerOptions.ExpiresOn = options.ScheduleDeletionOptions.ExpiresOn.Value().ToString(
+          Azure::DateTime::DateFormat::Rfc1123);
+    }
+    else if (options.ScheduleDeletionOptions.TimeToExpire.HasValue())
+    {
+      protocolLayerOptions.ExpiryOptions
+          = Files::DataLake::ScheduleFileExpiryOriginType::RelativeToNow.ToString();
+      protocolLayerOptions.ExpiresOn
+          = std::to_string(options.ScheduleDeletionOptions.TimeToExpire.Value().count());
+    }
     if (m_customerProvidedKey.HasValue())
     {
       protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;

sdk/storage/azure-storage-files-datalake/src/rest_client.cpp

@@ -114,6 +114,18 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
           {
             vectorElement2.EncryptionScope = var0["EncryptionScope"].get<std::string>();
           }
+          if (var0.count("creationTime") != 0)
+          {
+            vectorElement2.CreatedOn
+                = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime(
+                    std::stoll(var0["creationTime"].get<std::string>()));
+          }
+          if (var0.count("expiryTime") != 0)
+          {
+            vectorElement2.ExpiresOn
+                = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime(
+                    std::stoll(var0["expiryTime"].get<std::string>()));
+          }
           if (var0.count("etag") != 0)
           {
             vectorElement2.ETag = var0["etag"].get<std::string>();
@@ -277,6 +289,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
       {
         request.SetHeader("x-ms-lease-duration", std::to_string(options.LeaseDuration.Value()));
       }
+      if (options.ExpiryOptions.HasValue() && !options.ExpiryOptions.Value().empty())
+      {
+        request.SetHeader("x-ms-expiry-option", options.ExpiryOptions.Value());
+      }
       if (options.ExpiresOn.HasValue() && !options.ExpiresOn.Value().empty())
       {
         request.SetHeader("x-ms-expiry-time", options.ExpiresOn.Value());

sdk/storage/azure-storage-files-datalake/swagger/README.md

@@ -233,8 +233,8 @@ directive:
       $.Path.properties["contentLength"]["x-ms-client-name"] = "FileSize";
       $.Path.properties["isDirectory"]["x-ms-client-default"] = false;
       $.Path.properties["EncryptionScope"]["x-nullable"] = true;
-      delete $.Path.properties["creationTime"];
-      delete $.Path.properties["expiryTime"];
+      $.Path.properties["creationTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "CreatedOn", "x-nullable": true};
+      $.Path.properties["expiryTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "ExpiresOn", "x-nullable": true};
       $.Path.properties["etag"] = {"type": "string", "x-ms-format": "string", "x-ms-client-default": "", "x-ms-client-name": "ETag"};
       delete $.Path.properties["eTag"];
       $.PathList["x-namespace"] = "_detail";
@@ -252,10 +252,6 @@ directive:
 
 ```yaml
 directive:
-  - from: swagger-document
-    where: $["x-ms-paths"]["/{filesystem}/{path}"].put.parameters
-    transform: >
-      $ = $.filter(p => !(p["$ref"] && (p["$ref"].endsWith("#/parameters/PathExpiryOptionsOptional"))));
   - from: swagger-document
     where: $["x-ms-paths"]["/{filesystem}/{path}"].put.responses
     transform: >
@@ -283,7 +279,9 @@ directive:
     where: $.parameters
     transform: >
       delete $["PathExpiryOptions"];
-      delete $["PathExpiryOptionsOptional"];
+      delete $.PathExpiryOptionsOptional["enum"];
+      delete $.PathExpiryOptionsOptional["x-ms-enum"];
+
 ```
 
 ### DeletePath