DLPX-81988 Pool creation hangs after EMFILE errors in agent (openzfs#522

)

cmd/zfs_object_agent/zettaobject/src/object_access/blob.rs

@@ -550,6 +550,13 @@ impl ObjectAccessTrait for BlobObjectAccess {
     }
 
     async fn delete_objects(&self, stream: &mut (dyn Stream<Item = String> + Send + Unpin)) {
+        // azure-sdk-for-rust does not yet support batched deletion of objects. So we issue
+        // object deletion requests in parallel. This means we open many TCP connections in
+        // parallel. Experimentally, we see that we get good performance with about half the
+        // default batch size that we use with S3 which is 1000. This also avoids running out
+        // of file desciptors when the ulimit is low.
+        let batch_size = *OBJECT_DELETION_BATCH_SIZE / 2;
+
         stream
             .map(|key| async move {
                 let op = self.access_stats.begin(ObjectAccessOpType::ObjectDelete);
@@ -584,7 +591,7 @@ impl ObjectAccessTrait for BlobObjectAccess {
                 .unwrap();
                 op.end(0);
             })
-            .buffered(*OBJECT_DELETION_BATCH_SIZE)
+            .buffered(batch_size)
             .count()
             .await;
     }

etc/systemd/system/zfs-object-agent.service.in

@@ -16,6 +16,7 @@ Environment="AWS_ACCESS_KEY_ID="
 EnvironmentFile=-/etc/zfs/zoa.conf
 ExecStart=/usr/bin/start_zoa
 Restart=always
+LimitNOFILE=infinity
 
 [Install]
 WantedBy=zfs-import.target