Support XFS filesystems in storage clusters

- Allow mdadm-based RAID-0 arrays to expand (experimental)
- Greatly expand remote fs guide with configuration/usage explanations
- Fix blocking bug with fs commands
- Resolves #219

convoy/batch.py

@@ -2980,6 +2980,7 @@ def egress_service_logs(
                 ('waiting for {} log files to be uploaded; this may take '
                  'some time, please be patient').format(
                      resp.number_of_files_uploaded))
+            count = 0
             while True:
                 blobs = blob_client.list_blobs(
                     cont, prefix=resp.virtual_directory_name,
@@ -2992,6 +2993,10 @@ def egress_service_logs(
                              resp.virtual_directory_name,
                              storage_settings.account))
                     break
+                count += 1
+                if count > 150:
+                    logger.error('exceeded wait timeout for log egress')
+                    return
                 time.sleep(2)
         if generate_sas:
             sas = storage.create_saskey(

convoy/remotefs.py

@@ -526,8 +526,8 @@ def create_storage_cluster(
                     not settings.is_premium_storage_vm_size(
                         rfs.storage_cluster.vm_size)):
                 raise RuntimeError(
-                    ('Premium storage requires a DS, DS_V2, FS, GS or LS '
-                     'series vm_size instead of {}'.format(
+                    ('Premium storage requires premium storage capable '
+                     'vm_size instead of {}'.format(
                          rfs.storage_cluster.vm_size)))
     # confirm before proceeding
     if not util.confirm_action(
@@ -1034,13 +1034,12 @@ def expand_storage_cluster(
     # check vms
     vms = {}
     new_disk_count = 0
+    mdadm_expand = False
     for i in range(rfs.storage_cluster.vm_count):
         # check if this vm filesystem supports expanding
         if (rfs.storage_cluster.vm_disk_map[i].filesystem != 'btrfs' and
                 rfs.storage_cluster.vm_disk_map[i].raid_level == 0):
-            raise RuntimeError(
-                'Cannot expand mdadm-based RAID-0 volumes. Please re-create '
-                'your storage cluster with btrfs using new disks.')
+            mdadm_expand = True
         vm_name = settings.generate_virtual_machine_name(
             rfs.storage_cluster, i)
         try:
@@ -1098,6 +1097,13 @@ def expand_storage_cluster(
         logger.error(
             'no new disks detected for storage cluster {}'.format(sc_id))
         return False
+    if mdadm_expand:
+        logger.warning(
+            '**WARNING** cluster expansion is being performed on mdadm-based '
+            'RAID arrays. This feature is experimental and can take an '
+            'extremely long time. Any interruption or unrecoverable '
+            'failure can result in data loss.')
+    del mdadm_expand
     # confirm before proceeding
     if not util.confirm_action(
             config, 'expand storage cluster {}'.format(sc_id)):
@@ -1150,7 +1156,7 @@ def expand_storage_cluster(
         return False
     logger.debug(
         'waiting for disks to attach to virtual machines and expanding '
-        'the gluster volume, this may take a while')
+        'the volume; please be patient as this can take a very long time')
     for offset in async_ops:
         premium, op = async_ops[offset]
         vm = op.result()

convoy/settings.py

@@ -4121,7 +4121,7 @@ def remotefs_settings(config, sc_id=None):
                 ('All disks {} for vm {} are not specified in '
                  'managed_disks:disk_names ({})').format(
                      disk_array, vmkey, _disk_set))
-        raid_level = _kv_read(vmd_conf[vmkey], 'raid_level', -1)
+        raid_level = _kv_read(vmd_conf[vmkey], 'raid_level', default=-1)
         if len(disk_array) == 1 and raid_level != -1:
             raise ValueError(
                 'Cannot specify a RAID-level with 1 disk in array')
@@ -4131,10 +4131,6 @@ def remotefs_settings(config, sc_id=None):
             if raid_level != 0:
                 raise ValueError('Unsupported RAID level {}'.format(
                     raid_level))
-        filesystem = vmd_conf[vmkey]['filesystem']
-        if filesystem != 'btrfs' and not filesystem.startswith('ext'):
-            raise ValueError('Unsupported filesystem type {}'.format(
-                filesystem))
         disk_map[int(vmkey)] = MappedVmDiskSettings(
             disk_array=disk_array,
             filesystem=vmd_conf[vmkey]['filesystem'],

docs/14-batch-shipyard-configuration-jobs.md

@@ -481,7 +481,9 @@ directory for the container execution is not explicitly set. The default is
 directory, you can pass the appropriate working directory parameter to the
 container runtime through either `additional_docker_run_options` or
 `additional_singularity_options`. A working directory option specified within
-that property takes precedence over this option.
+that property takes precedence over this option. Note that this option does
+not work in `native` mode currently; `native` mode will always override this
+option to `batch`.
 
 The required `tasks` property is an array of tasks to add to the job:
 

docs/15-batch-shipyard-configuration-fs.md

@@ -365,16 +365,16 @@ The number of entries in this map must match the `vm_count`.
           attach to this instance. These disks must be provisioned before
           creating the storage cluster.
         * (required) `filesystem` is the filesystem to use. Valid values are
-          `btrfs`, `ext4`, `ext3` and `ext2`. `btrfs` is generally stable for
-          RAID-0, with better features and data integrity protection. `btrfs`
-          also allows for RAID-0 expansion and is the only filesystem
-          compatible with the `fs cluster expand` command.
+          `btrfs`, `ext4`, `ext3`, `ext2`, and `xfs`. `btrfs` is generally
+          stable for RAID-0, with many features, data integrity
+          protection and also allows for direct RAID-0 expansion. `xfs`
+          is a high performance, stable and proven option.
         * (optional for single disk, required for multiple disks) `raid_level`
           is the RAID level to apply to the disks in the `disk_array`. The
-          only valid value for multiple disks is `0`. Note that if you wish
-          to expand the number of disks in the array in the future, you must
-          use `btrfs` as the filesystem. At least two disks per virtual
-          machine are required for RAID-0.
+          only valid value for multiple disks is `0`. At least two disks per
+          virtual machine are required for RAID-0. Note that if you wish
+          to expand the number of disks in the array in the future, it is
+          strongly recommended to use `btrfs` as the filesystem.
 * (optional) `prometheus` properties are to control if collectors for metrics
 to export to [Prometheus](https://prometheus.io/) monitoring are enabled.
 Note that all exporters do not have their ports exposed to the internet by
@@ -392,7 +392,7 @@ cluster VMs.
         * (optional) `options` is a list of options to pass to the
           node exporter instance running on all nodes. The following
           collectors are force disabled, in addition to others disabled by
-          default: textfile, wifi, xfs, zfs. The nfs collector is enabled if
+          default: textfile, wifi, zfs. The nfs collector is enabled if
           the file server is NFS, automatically.
 
 ## Remote Filesystems with Batch Shipyard Guide