Merge pull request #4100 from nextcloud/feature/ask-server-to-recalc-…

…checksum

Ask server to recalculate checksum(hash)

src/common/checksums.cpp

@@ -337,7 +337,7 @@ ComputeChecksum *ValidateChecksumHeader::prepareStart(const QByteArray &checksum
 
     if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) {
         qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader;
-        emit validationFailed(tr("The checksum header is malformed."));
+        emit validationFailed(tr("The checksum header is malformed."), _calculatedChecksumType, _calculatedChecksum, ChecksumHeaderMalformed);
         return nullptr;
     }
 
@@ -360,15 +360,30 @@ void ValidateChecksumHeader::start(std::unique_ptr<QIODevice> device, const QByt
         calculator->start(std::move(device));
 }
 
+QByteArray ValidateChecksumHeader::calculatedChecksumType() const
+{
+    return _calculatedChecksumType;
+}
+
+QByteArray ValidateChecksumHeader::calculatedChecksum() const
+{
+    return _calculatedChecksum;
+}
+
 void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumType,
     const QByteArray &checksum)
 {
+    _calculatedChecksumType = checksumType;
+    _calculatedChecksum = checksum;
+
     if (checksumType != _expectedChecksumType) {
-        emit validationFailed(tr("The checksum header contained an unknown checksum type \"%1\"").arg(QString::fromLatin1(_expectedChecksumType)));
+        emit validationFailed(tr("The checksum header contained an unknown checksum type \"%1\"").arg(QString::fromLatin1(_expectedChecksumType)),
+            _calculatedChecksumType, _calculatedChecksum, ChecksumTypeUnknown);
         return;
     }
     if (checksum != _expectedChecksum) {
-        emit validationFailed(tr(R"(The downloaded file does not match the checksum, it will be resumed. "%1" != "%2")").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum)));
+        emit validationFailed(tr(R"(The downloaded file does not match the checksum, it will be resumed. "%1" != "%2")").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum)),
+            _calculatedChecksumType, _calculatedChecksum, ChecksumMismatch);
         return;
     }
     emit validated(checksumType, checksum);

src/common/checksums.h

@@ -140,6 +140,14 @@ class OCSYNC_EXPORT ValidateChecksumHeader : public QObject
 {
     Q_OBJECT
 public:
+    enum FailureReason {
+        Success,
+        ChecksumHeaderMalformed,
+        ChecksumTypeUnknown,
+        ChecksumMismatch,
+    };
+    Q_ENUM(FailureReason)
+
     explicit ValidateChecksumHeader(QObject *parent = nullptr);
 
     /**
@@ -161,9 +169,13 @@ class OCSYNC_EXPORT ValidateChecksumHeader : public QObject
      */
     void start(std::unique_ptr<QIODevice> device, const QByteArray &checksumHeader);
 
+    QByteArray calculatedChecksumType() const;
+    QByteArray calculatedChecksum() const;
+
 signals:
     void validated(const QByteArray &checksumType, const QByteArray &checksum);
-    void validationFailed(const QString &errMsg);
+    void validationFailed(const QString &errMsg, const QByteArray &calculatedChecksumType,
+        const QByteArray &calculatedChecksum, const ValidateChecksumHeader::FailureReason reason);
 
 private slots:
     void slotChecksumCalculated(const QByteArray &checksumType, const QByteArray &checksum);
@@ -173,6 +185,9 @@ private slots:
 
     QByteArray _expectedChecksumType;
     QByteArray _expectedChecksum;
+
+    QByteArray _calculatedChecksumType;
+    QByteArray _calculatedChecksum;
 };
 
 /**

src/libsync/account.cpp

@@ -57,7 +57,8 @@ using namespace QKeychain;
 
 namespace {
 constexpr int pushNotificationsReconnectInterval = 1000 * 60 * 2;
-constexpr int usernamePrefillServerVersinMinSupportedMajor = 24;
+constexpr int usernamePrefillServerVersionMinSupportedMajor = 24;
+constexpr int checksumRecalculateRequestServerVersionMinSupportedMajor = 24;
 }
 
 namespace OCC {
@@ -632,7 +633,17 @@ bool Account::serverVersionUnsupported() const
 
 bool Account::isUsernamePrefillSupported() const
 {
-    return serverVersionInt() >= makeServerVersion(usernamePrefillServerVersinMinSupportedMajor, 0, 0);
+    return serverVersionInt() >= makeServerVersion(usernamePrefillServerVersionMinSupportedMajor, 0, 0);
+}
+
+bool Account::isChecksumRecalculateRequestSupported() const
+{
+    return serverVersionInt() >= makeServerVersion(checksumRecalculateRequestServerVersionMinSupportedMajor, 0, 0);
+}
+
+int Account::checksumRecalculateServerVersionMinSupportedMajor() const
+{
+    return checksumRecalculateRequestServerVersionMinSupportedMajor;
 }
 
 void Account::setServerVersion(const QString &version)

src/libsync/account.h

@@ -232,6 +232,10 @@ class OWNCLOUDSYNC_EXPORT Account : public QObject
 
     bool isUsernamePrefillSupported() const;
 
+    bool isChecksumRecalculateRequestSupported() const;
+
+    int checksumRecalculateServerVersionMinSupportedMajor() const;
+
     /** True when the server connection is using HTTP2  */
     bool isHttp2Supported() { return _http2Supported; }
     void setHttp2Supported(bool value) { _http2Supported = value; }

src/libsync/deletejob.cpp

@@ -21,12 +21,12 @@ namespace OCC {
 Q_LOGGING_CATEGORY(lcDeleteJob, "nextcloud.sync.networkjob.delete", QtInfoMsg)
 
 DeleteJob::DeleteJob(AccountPtr account, const QString &path, QObject *parent)
-    : AbstractNetworkJob(account, path, parent)
+    : SimpleFileJob(account, path, parent)
 {
 }
 
 DeleteJob::DeleteJob(AccountPtr account, const QUrl &url, QObject *parent)
-    : AbstractNetworkJob(account, QString(), parent)
+    : SimpleFileJob(account, QString(), parent)
     , _url(url)
 {
 }
@@ -39,24 +39,10 @@ void DeleteJob::start()
     }
 
     if (_url.isValid()) {
-        sendRequest("DELETE", _url, req);
+        startRequest("DELETE", _url, req);
     } else {
-        sendRequest("DELETE", makeDavUrl(path()), req);
+        startRequest("DELETE", req);
     }
-
-    if (reply()->error() != QNetworkReply::NoError) {
-        qCWarning(lcDeleteJob) << " Network error: " << reply()->errorString();
-    }
-    AbstractNetworkJob::start();
-}
-
-bool DeleteJob::finished()
-{
-    qCInfo(lcDeleteJob) << "DELETE of" << reply()->request().url() << "FINISHED WITH STATUS"
-                       << replyStatusString();
-
-    emit finishedSignal();
-    return true;
 }
 
 QByteArray DeleteJob::folderToken() const

src/libsync/deletejob.h

@@ -23,22 +23,18 @@ namespace OCC {
  * @brief The DeleteJob class
  * @ingroup libsync
  */
-class DeleteJob : public AbstractNetworkJob
+class DeleteJob : public SimpleFileJob
 {
     Q_OBJECT
 public:
     explicit DeleteJob(AccountPtr account, const QString &path, QObject *parent = nullptr);
     explicit DeleteJob(AccountPtr account, const QUrl &url, QObject *parent = nullptr);
 
     void start() override;
-    bool finished() override;
 
     QByteArray folderToken() const;
     void setFolderToken(const QByteArray &folderToken);
 
-signals:
-    void finishedSignal();
-
 private:
     QUrl _url; // Only used if the constructor taking a url is taken.
     QByteArray _folderToken;

src/libsync/networkjobs.cpp

@@ -55,6 +55,7 @@ Q_LOGGING_CATEGORY(lcMkColJob, "nextcloud.sync.networkjob.mkcol", QtInfoMsg)
 Q_LOGGING_CATEGORY(lcProppatchJob, "nextcloud.sync.networkjob.proppatch", QtInfoMsg)
 Q_LOGGING_CATEGORY(lcJsonApiJob, "nextcloud.sync.networkjob.jsonapi", QtInfoMsg)
 Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "nextcloud.sync.networkjob.determineauthtype", QtInfoMsg)
+Q_LOGGING_CATEGORY(lcSimpleFileJob, "nextcloud.sync.networkjob.simplefilejob", QtInfoMsg)
 const int notModifiedStatusCode = 304;
 
 QByteArray parseEtag(const char *header)
@@ -1084,9 +1085,39 @@ bool SimpleNetworkJob::finished()
     return true;
 }
 
+SimpleFileJob::SimpleFileJob(AccountPtr account, const QString &filePath, QObject *parent)
+    : AbstractNetworkJob(account, filePath, parent)
+{
+}
+
+QNetworkReply *SimpleFileJob::startRequest(
+    const QByteArray &verb, const QNetworkRequest req, QIODevice *requestBody)
+{
+    return startRequest(verb, makeDavUrl(path()), req, requestBody);
+}
+
+QNetworkReply *SimpleFileJob::startRequest(
+    const QByteArray &verb, const QUrl &url, const QNetworkRequest req, QIODevice *requestBody)
+{
+    _verb = verb;
+    const auto reply = sendRequest(verb, url, req, requestBody);
+
+    if (reply->error() != QNetworkReply::NoError) {
+        qCWarning(lcSimpleFileJob) << verb << " Network error: " << reply->errorString();
+    }
+    AbstractNetworkJob::start();
+    return reply;
+}
+
+bool SimpleFileJob::finished()
+{
+    qCInfo(lcSimpleFileJob) << _verb << "for" << reply()->request().url() << "FINISHED WITH STATUS" << replyStatusString();
+    emit finishedSignal(reply());
+    return true;
+}
 
 DeleteApiJob::DeleteApiJob(AccountPtr account, const QString &path, QObject *parent)
-    : AbstractNetworkJob(account, path, parent)
+    : SimpleFileJob(account, path, parent)
 {
 
 }
@@ -1095,14 +1126,13 @@ void DeleteApiJob::start()
 {
     QNetworkRequest req;
     req.setRawHeader("OCS-APIREQUEST", "true");
-    QUrl url = Utility::concatUrlPath(account()->url(), path());
-    sendRequest("DELETE", url, req);
-    AbstractNetworkJob::start();
+
+    startRequest("DELETE", req);
 }
 
 bool DeleteApiJob::finished()
 {
-    qCInfo(lcJsonApiJob) << "JsonApiJob of" << reply()->request().url() << "FINISHED WITH STATUS"
+    qCInfo(lcJsonApiJob) << "DeleteApiJob of" << reply()->request().url() << "FINISHED WITH STATUS"
                          << reply()->error()
                          << (reply()->error() == QNetworkReply::NoError ? QLatin1String("") : errorString());
 
@@ -1118,7 +1148,7 @@ bool DeleteApiJob::finished()
     const auto replyData = QString::fromUtf8(reply()->readAll());
     qCInfo(lcJsonApiJob()) << "TMX Delete Job" << replyData;
     emit result(httpStatus);
-    return true;
+    return SimpleFileJob::finished();
 }
 
 void fetchPrivateLinkUrl(AccountPtr account, const QString &remotePath,

src/libsync/networkjobs.h

@@ -60,14 +60,39 @@ private slots:
     bool finished() override;
 };
 
+/**
+ * @brief A basic file manipulation job
+ * @ingroup libsync
+ */
+class OWNCLOUDSYNC_EXPORT SimpleFileJob : public AbstractNetworkJob
+{
+    Q_OBJECT
+public:
+    explicit SimpleFileJob(AccountPtr account, const QString &filePath, QObject *parent = nullptr);
+
+    QNetworkReply *startRequest(
+        const QByteArray &verb, const QNetworkRequest req = QNetworkRequest(), QIODevice *requestBody = nullptr);
+
+    QNetworkReply *startRequest(const QByteArray &verb, const QUrl &url, const QNetworkRequest req = QNetworkRequest(),
+        QIODevice *requestBody = nullptr);
+
+signals:
+    void finishedSignal(QNetworkReply *reply);
+protected slots:
+    bool finished() override;
+
+private:
+    QByteArray _verb;
+};
+
 /**
  * @brief sends a DELETE http request to a url.
  *
  * See Nextcloud API usage for the possible DELETE requests.
  *
  * This does *not* delete files, it does a http request.
  */
-class OWNCLOUDSYNC_EXPORT DeleteApiJob : public AbstractNetworkJob
+class OWNCLOUDSYNC_EXPORT DeleteApiJob : public SimpleFileJob
 {
     Q_OBJECT
 public:

src/libsync/propagatedownload.cpp

@@ -22,7 +22,6 @@
 #include "common/utility.h"
 #include "filesystem.h"
 #include "propagatorjobs.h"
-#include <common/checksums.h>
 #include <common/asserts.h>
 #include <common/constants.h>
 #include "clientsideencryptionjobs.h"
@@ -923,8 +922,53 @@ void PropagateDownloadFile::slotGetFinished()
     validator->start(_tmpFile.fileName(), checksumHeader);
 }
 
-void PropagateDownloadFile::slotChecksumFail(const QString &errMsg)
-{ 
+void PropagateDownloadFile::slotChecksumFail(const QString &errMsg,
+    const QByteArray &calculatedChecksumType, const QByteArray &calculatedChecksum, const ValidateChecksumHeader::FailureReason reason)
+{
+    if (reason == ValidateChecksumHeader::FailureReason::ChecksumMismatch && propagator()->account()->isChecksumRecalculateRequestSupported()) {
+            const QByteArray calculatedChecksumHeader(calculatedChecksumType + ':' + calculatedChecksum);
+            const QString fullRemotePathForFile(propagator()->fullRemotePath(_isEncrypted ? _item->_encryptedFileName : _item->_file));
+            auto *job = new SimpleFileJob(propagator()->account(), fullRemotePathForFile);
+            QObject::connect(job, &SimpleFileJob::finishedSignal, this,
+                [this, calculatedChecksumHeader, errMsg](const QNetworkReply *reply) { processChecksumRecalculate(reply, calculatedChecksumHeader, errMsg);
+            });
+
+            qCWarning(lcPropagateDownload) << "Checksum validation has failed for file:" << fullRemotePathForFile
+                                           << " Requesting checksum recalculation on the server...";
+            QNetworkRequest req;
+            req.setRawHeader(checksumRecalculateOnServerHeaderC, calculatedChecksumType);
+            job->startRequest(QByteArrayLiteral("PATCH"), req);
+            return;
+    }
+
+    checksumValidateFailedAbortDownload(errMsg);
+}
+
+void PropagateDownloadFile::processChecksumRecalculate(const QNetworkReply *reply, const QByteArray &originalChecksumHeader, const QString &errorMessage)
+{
+    if (reply->error() != QNetworkReply::NoError) {
+        qCCritical(lcPropagateDownload) << "Checksum recalculation has failed for file:" << reply->url()
+                                        << " Recalculation request has finished with error:" << reply->errorString();
+        checksumValidateFailedAbortDownload(errorMessage);
+        return;
+    }
+
+    const auto newChecksumHeaderFromServer = reply->rawHeader(checkSumHeaderC);
+    if (newChecksumHeaderFromServer == originalChecksumHeader) {
+        const auto newChecksumHeaderFromServerSplit = newChecksumHeaderFromServer.split(':');
+        if (newChecksumHeaderFromServerSplit.size() > 1) {
+            transmissionChecksumValidated(newChecksumHeaderFromServerSplit.first(), newChecksumHeaderFromServerSplit.last());
+            return;
+        }
+    }
+
+    qCCritical(lcPropagateDownload) << "Checksum recalculation has failed for file:" << reply->url() << " "
+                                    << checkSumHeaderC << " received is:" << newChecksumHeaderFromServer;
+    checksumValidateFailedAbortDownload(errorMessage);
+}
+
+void PropagateDownloadFile::checksumValidateFailedAbortDownload(const QString &errMsg)
+{
     FileSystem::remove(_tmpFile.fileName());
     propagator()->_anotherSyncNeeded = true;
     done(SyncFileItem::SoftError, errMsg); // tr("The file downloaded with a broken checksum, will be redownloaded."));

src/libsync/propagatedownload.h

@@ -17,6 +17,7 @@
 #include "owncloudpropagator.h"
 #include "networkjobs.h"
 #include "clientsideencryption.h"
+#include <common/checksums.h>
 
 #include <QBuffer>
 #include <QFile>
@@ -235,7 +236,10 @@ private slots:
 
     void abort(PropagatorJob::AbortType abortType) override;
     void slotDownloadProgress(qint64, qint64);
-    void slotChecksumFail(const QString &errMsg);
+    void slotChecksumFail(const QString &errMsg, const QByteArray &calculatedChecksumType,
+        const QByteArray &calculatedChecksum, const ValidateChecksumHeader::FailureReason reason);
+    void processChecksumRecalculate(const QNetworkReply *reply, const QByteArray &originalChecksumHeader, const QString &errorMessage);
+    void checksumValidateFailedAbortDownload(const QString &errMsg);
 
 private:
     void startAfterIsEncryptedIsChecked();