Merge pull request #5480 from owncloud/chunking-ng-cleanup

Chunking ng cleanup

src/libsync/propagateuploadng.cpp

@@ -98,6 +98,12 @@ void PropagateUploadFileNG::doStartUpload()
                 this, SLOT(slotPropfindIterate(QString,QMap<QString,QString>)));
         job->start();
         return;
+    } else if (progressInfo._valid) {
+        // The upload info is stale. remove the stale chunks on the server
+        _transferId = progressInfo._transferid;
+        // Fire and forget. Any error will be ignored.
+        (new DeleteJob(propagator()->account(), chunkUrl(), this))->start();
+        // startNewUpload will reset the _transferId and the UploadInfo in the db.
     }
 
     startNewUpload();

src/libsync/syncengine.cpp

@@ -23,6 +23,7 @@
 #include "syncfilestatus.h"
 #include "csync_private.h"
 #include "filesystem.h"
+#include "propagateremotedelete.h"
 
 #ifdef Q_OS_WIN
 #include <windows.h>
@@ -301,7 +302,16 @@ void SyncEngine::deleteStaleUploadInfos()
     }
 
     // Delete from journal.
-    _journal->deleteStaleUploadInfos(upload_file_paths);
+    auto ids = _journal->deleteStaleUploadInfos(upload_file_paths);
+
+    // Delete the stales chunk on the server.
+    if (account()->capabilities().chunkingNg()) {
+        foreach (uint transferId, ids) {
+            QUrl url = Utility::concatUrlPath(account()->url(), QLatin1String("remote.php/dav/uploads/")
+                + account()->davUser() + QLatin1Char('/') + QString::number(transferId));
+            (new DeleteJob(account(), url, this))->start();
+        }
+    }
 }
 
 void SyncEngine::deleteStaleErrorBlacklistEntries()

src/libsync/syncjournaldb.cpp

@@ -1370,21 +1370,22 @@ void SyncJournalDb::setUploadInfo(const QString& file, const SyncJournalDb::Uplo
     }
 }
 
-bool SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
+QVector<uint> SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
 {
     QMutexLocker locker(&_mutex);
+    QVector<uint> ids;
 
     if (!checkConnect()) {
-        return false;
+        return ids;
     }
 
     SqlQuery query(_db);
-    query.prepare("SELECT path FROM uploadinfo");
+    query.prepare("SELECT path,transferid FROM uploadinfo");
 
     if (!query.exec()) {
         QString err = query.error();
         qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;
-        return false;
+        return ids;
     }
 
     QStringList superfluousPaths;
@@ -1393,10 +1394,12 @@ bool SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
         const QString file = query.stringValue(0);
         if (!keep.contains(file)) {
             superfluousPaths.append(file);
+            ids.append(query.intValue(1));
         }
     }
 
-    return deleteBatch(*_deleteUploadInfoQuery, superfluousPaths, "uploadinfo");
+    deleteBatch(*_deleteUploadInfoQuery, superfluousPaths, "uploadinfo");
+    return ids;
 }
 
 SyncJournalErrorBlacklistRecord SyncJournalDb::errorBlacklistEntry( const QString& file )

src/libsync/syncjournaldb.h

@@ -105,7 +105,8 @@ class OWNCLOUDSYNC_EXPORT SyncJournalDb : public QObject
 
     UploadInfo getUploadInfo(const QString &file);
     void setUploadInfo(const QString &file, const UploadInfo &i);
-    bool deleteStaleUploadInfos(const QSet<QString>& keep);
+    // Return the list of transfer ids that were removed.
+    QVector<uint> deleteStaleUploadInfos(const QSet<QString>& keep);
 
     SyncJournalErrorBlacklistRecord errorBlacklistEntry( const QString& );
     bool deleteStaleErrorBlacklistEntries(const QSet<QString>& keep);

test/syncenginetestutils.h

@@ -315,7 +315,10 @@ class FakePropfindReply : public QNetworkReply
         QString fileName = getFilePathFromUrl(request.url());
         Q_ASSERT(!fileName.isNull()); // for root, it should be empty
         const FileInfo *fileInfo = remoteRootFileInfo.find(fileName);
-        Q_ASSERT(fileInfo);
+        if (!fileInfo) {
+            QMetaObject::invokeMethod(this, "respond404", Qt::QueuedConnection);
+            return;
+        }
         QString prefix = request.url().path().left(request.url().path().size() - fileName.size());
 
         // Don't care about the request and just return a full propfind
@@ -375,6 +378,13 @@ class FakePropfindReply : public QNetworkReply
         emit finished();
     }
 
+    Q_INVOKABLE void respond404() {
+        setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 404);
+        setError(InternalServerError, "Not Found");
+        emit metaDataChanged();
+        emit finished();
+    }
+
     void abort() override { }
 
     qint64 bytesAvailable() const override { return payload.size() + QIODevice::bytesAvailable(); }
@@ -524,7 +534,8 @@ class FakeGetReply : public QNetworkReply
     Q_OBJECT
 public:
     const FileInfo *fileInfo;
-    QByteArray payload;
+    char payload;
+    int size;
 
     FakeGetReply(FileInfo &remoteRootFileInfo, QNetworkAccessManager::Operation op, const QNetworkRequest &request, QObject *parent)
     : QNetworkReply{parent} {
@@ -540,8 +551,9 @@ class FakeGetReply : public QNetworkReply
     }
 
     Q_INVOKABLE void respond() {
-        payload.fill(fileInfo->contentChar, fileInfo->size);
-        setHeader(QNetworkRequest::ContentLengthHeader, payload.size());
+        payload = fileInfo->contentChar;
+        size = fileInfo->size;
+        setHeader(QNetworkRequest::ContentLengthHeader, size);
         setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 200);
         setRawHeader("OC-ETag", fileInfo->etag.toLatin1());
         setRawHeader("ETag", fileInfo->etag.toLatin1());
@@ -553,12 +565,12 @@ class FakeGetReply : public QNetworkReply
     }
 
     void abort() override { }
-    qint64 bytesAvailable() const override { return payload.size() + QIODevice::bytesAvailable(); }
+    qint64 bytesAvailable() const override { return size + QIODevice::bytesAvailable(); }
 
     qint64 readData(char *data, qint64 maxlen) override {
-        qint64 len = std::min(qint64{payload.size()}, maxlen);
-        strncpy(data, payload.constData(), len);
-        payload.remove(0, len);
+        qint64 len = std::min(qint64{size}, maxlen);
+        std::fill_n(data, len, payload);
+        size -= len;
         return len;
     }
 };
@@ -586,7 +598,7 @@ class FakeChunkMoveReply : public QNetworkReply
         Q_ASSERT(sourceFolder->isDir);
         int count = 0;
         int size = 0;
-        char payload = '*';
+        char payload = '\0';
 
         do {
             QString chunkName = QString::number(count).rightJustified(8, '0');
@@ -596,6 +608,7 @@ class FakeChunkMoveReply : public QNetworkReply
             Q_ASSERT(!x.isDir);
             Q_ASSERT(x.size > 0); // There should not be empty chunks
             size += x.size;
+            Q_ASSERT(!payload || payload == x.contentChar);
             payload = x.contentChar;
             ++count;
         } while(true);
@@ -607,7 +620,12 @@ class FakeChunkMoveReply : public QNetworkReply
         Q_ASSERT(!fileName.isEmpty());
 
         if ((fileInfo = remoteRootFileInfo.find(fileName))) {
-            QCOMPARE(request.rawHeader("If"), QByteArray("<" + request.rawHeader("Destination") + "> ([\"" + fileInfo->etag.toLatin1() + "\"])"));
+            QVERIFY(request.hasRawHeader("If")); // The client should put this header
+            if (request.rawHeader("If") != QByteArray("<" + request.rawHeader("Destination") +
+                                                "> ([\"" + fileInfo->etag.toLatin1() + "\"])")) {
+                QMetaObject::invokeMethod(this, "respondPreconditionFailed", Qt::QueuedConnection);
+                return;
+            }
             fileInfo->size = size;
             fileInfo->contentChar = payload;
         } else {
@@ -632,6 +650,13 @@ class FakeChunkMoveReply : public QNetworkReply
         emit finished();
     }
 
+    Q_INVOKABLE void respondPreconditionFailed() {
+        setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 412);
+        setError(InternalServerError, "Precondition Failed");
+        emit metaDataChanged();
+        emit finished();
+    }
+
     void abort() override { }
     qint64 readData(char *, qint64) override { return 0; }
 };