Merge pull request #6009 from nextcloud/enh/4751/allow-to-exclude-dat…

…adir

borg: allow to exclude datadir and preview folder from backup via `.noaiobackup` file

Containers/borgbackup/backupscript.sh

@@ -185,13 +185,27 @@ if [ "$BORG_MODE" = backup ]; then
     # Borg options
     # auto,zstd compression seems to has the best ratio based on:
     # https://forum.level1techs.com/t/optimal-compression-for-borg-backups/145870/6
-    BORG_OPTS=(-v --stats --compression "auto,zstd" --exclude-caches)
+    BORG_OPTS=(-v --stats --compression "auto,zstd")
     if [ "$NEW_REPOSITORY" = 1 ]; then
         BORG_OPTS+=(--progress)
     fi
 
     # Exclude the nextcloud log and audit log for GDPR reasons
     BORG_EXCLUDE=(--exclude "/nextcloud_aio_volumes/nextcloud_aio_nextcloud/data/nextcloud.log*" --exclude "/nextcloud_aio_volumes/nextcloud_aio_nextcloud/data/audit.log")
+    BORG_INCLUDE=()
+
+    # Exclude datadir if .noaiobackup file was found
+    # shellcheck disable=SC2144
+    if [ -f "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/.noaiobackup" ]; then
+        BORG_EXCLUDE+=(--exclude "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/")
+        BORG_INCLUDE+=(--pattern="+/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/.noaiobackup")
+        echo "⚠️⚠️⚠️ '.noaiobackup' file was found in Nextclouds data directory. Excluding the data directory from backup!"
+    # Exclude preview folder if .noaiobackup file was found
+    elif [ -f /nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/appdata_*/preview/.noaiobackup ]; then
+        BORG_EXCLUDE+=(--exclude "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/appdata_*/preview/")
+        BORG_INCLUDE+=(--pattern="+/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/appdata_*/preview/.noaiobackup")
+        echo "⚠️⚠️⚠️ '.noaiobackup' file was found in the preview directory. Excluding the preview directory from backup!"
+    fi
 
     # Make sure that there is always a borg.config file before creating a new backup
     if ! [ -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config" ]; then
@@ -203,7 +217,7 @@ if [ "$BORG_MODE" = backup ]; then
     # Create the backup
     echo "Starting the backup..."
     get_start_time
-    if ! borg create "${BORG_OPTS[@]}" "${BORG_EXCLUDE[@]}" "::$CURRENT_DATE-nextcloud-aio" "/nextcloud_aio_volumes/" --exclude-from /borg_excludes; then
+    if ! borg create "${BORG_OPTS[@]}" "${BORG_INCLUDE[@]}" "${BORG_EXCLUDE[@]}" "::$CURRENT_DATE-nextcloud-aio" "/nextcloud_aio_volumes/" --exclude-from /borg_excludes; then
         echo "Deleting the failed backup archive..."
         borg delete --stats "::$CURRENT_DATE-nextcloud-aio"
         echo "Backup failed!"
@@ -320,16 +334,30 @@ if [ "$BORG_MODE" = restore ]; then
     fi
     echo "Restoring '$SELECTED_ARCHIVE'..."
 
-    # Exclude previews from restore if selected to speed up process
     ADDITIONAL_RSYNC_EXCLUDES=()
     ADDITIONAL_BORG_EXCLUDES=()
     ADDITIONAL_FIND_EXCLUDES=()
-    if [ -n "$RESTORE_EXCLUDE_PREVIEWS" ]; then
+    # Exclude datadir if .noaiobackup file was found
+    # shellcheck disable=SC2144
+    if [ -f "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/.noaiobackup" ]; then
+        # Keep these 3 in sync. Beware, the pattern syntax and the paths differ
+        ADDITIONAL_RSYNC_EXCLUDES=(--exclude "nextcloud_aio_nextcloud_data/**")
+        ADDITIONAL_BORG_EXCLUDES=(--exclude "sh:nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/**")
+        ADDITIONAL_FIND_EXCLUDES=(-o -regex 'nextcloud_aio_volumes/nextcloud_aio_nextcloud_data\(/.*\)?')
+        echo "⚠️⚠️⚠️ '.noaiobackup' file was found in Nextclouds data directory. Excluding the data directory from restore!"
+        echo "You might run into problems due to this afterwards as potentially this makes the directory go out of sync with the database."
+        echo "You might be able to fix this by running 'occ files:scan --all' and 'occ maintenance:repair' and 'occ files:scan-app-data' after the restore."
+        echo "See https://github.com/nextcloud/all-in-one#how-to-run-occ-commands"
+    # Exclude previews from restore if selected to speed up process or exclude preview folder if .noaiobackup file was found
+    elif [ -n "$RESTORE_EXCLUDE_PREVIEWS" ] || [ -f /nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/appdata_*/preview/.noaiobackup ]; then
         # Keep these 3 in sync. Beware, the pattern syntax and the paths differ
         ADDITIONAL_RSYNC_EXCLUDES=(--exclude "nextcloud_aio_nextcloud_data/appdata_*/preview/**")
         ADDITIONAL_BORG_EXCLUDES=(--exclude "sh:nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/appdata_*/preview/**")
         ADDITIONAL_FIND_EXCLUDES=(-o -regex 'nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/appdata_[^/]*/preview\(/.*\)?')
-        echo "Excluding previews from restore"
+        echo "⚠️⚠️⚠️ Excluding previews from restore!"
+        echo "You might run into problems due to this afterwards as potentially this makes the directory go out of sync with the database."
+        echo "You might be able to fix this by running 'occ files:scan-app-data preview' after the restore."
+        echo "See https://github.com/nextcloud/all-in-one#how-to-run-occ-commands"
     fi
 
     # Save Additional Backup dirs

readme.md

@@ -190,6 +190,7 @@ If your firewall/router has port 80 and 8443 open/forwarded and you point a doma
     - [Pro-tip: Backup archives access](#pro-tip-backup-archives-access)
     - [Delete backup archives manually](#delete-backup-archives-manually)
     - [Sync local backups regularly to another drive](#sync-local-backups-regularly-to-another-drive)
+    - [How to exclude Nextcloud's data directory or the preview folder from backup?](#how-to-exclude-nextclouds-data-directory-or-the-preview-folder-from-backup)
 - [How to stop/start/update containers or trigger the daily backup from a script externally?](#how-to-stopstartupdate-containers-or-trigger-the-daily-backup-from-a-script-externally)
 - [How to disable the backup section?](#how-to-disable-the-backup-section)
 - [How to change the default location of Nextcloud's Datadir?](#how-to-change-the-default-location-of-nextclouds-datadir)
@@ -751,6 +752,19 @@ Afterwards apply the correct permissions with `sudo chown root:root /root/backup
 1. Add the following new line to the crontab if not already present: `0 20 * * 7 /root/backup-script.sh` which will run the script at 20:00 on Sundays each week. 
 1. save and close the crontab (when using nano are the shortcuts for this `Ctrl + o` -> `Enter` and close the editor with `Ctrl + x`).
 
+#### How to exclude Nextcloud's data directory or the preview folder from backup?
+In order to speed up the backups and to keep the backup archives small, you might want to exclude Nextcloud's data directory or its preview folder from backup. 
+
+> [!WARNING]
+> However please note that you will run into problems if the database and the data directory or preview folder get out of sync. **So please only read further, if you have an additional external backup of the data directory!** See [this guide](#how-to-enable-automatic-updates-without-creating-a-backup-beforehand) for example.
+
+> [!TIP]
+> A better option is to use the external storage app inside Nextcloud as the data connected via the external storage app is not backed up by AIO's backup solution. See [this documentation](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/external_storage_configuration_gui.html) on how to configure the app.
+
+If you still want to proceed, you can exclude the data directory by simply creating a `.noaiobackup` file in the root directory of the specified `NEXTCLOUD_DATADIR` target. The same logic is implemented for the preview folder that is located inside the data directory, inside the `appdata_*/preview` folder. So simply create a `.noaiobackup` file in there if you want to exclude the preview folder.
+
+After doing a restore via the AIO interface, you might run into problems due to the data directory and database being out of sync. You might be able to fix this by running `occ files:scan --all` and `occ maintenance:repair` and `occ files:scan-app-data`. See https://github.com/nextcloud/all-in-one#how-to-run-occ-commands. If only the preview folder is excluded, the command `occ files:scan-app-data preview` should be used.
+
 ### How to stop/start/update containers or trigger the daily backup from a script externally?
 > [!WARNING]  
 > The below script will only work after the initial setup of AIO. So you will always need to first visit the AIO interface, type in your domain and start the containers the first time or restore an older AIO instance from its borg backup before you can use the script.