Deal with ranges -> message_ranges column rename (years ago... oops).…

… Update CMakeLists (newer c++std if available, prepare for needing DBUS). clearData on failure-exit in dumpavatars. Prep work for avatar extensions (#232). Properly import stickers in quotes on importfromdesktop. Refactor.

BUILDSCRIPT_MULTIPROC.bash44

@@ -16,7 +16,7 @@ else
 fi
 
 CXX="${CXX:-g++}"
-CXXFLAGS="${CXXFLAGS:--Wall -Wextra -Wshadow -Wold-style-cast -Woverloaded-virtual -pedantic -O3 -flto}"
+CXXFLAGS="${CXXFLAGS:--Wall -Wextra -Wshadow -Woverloaded-virtual -pedantic -O3 -flto}"
 CXXARCH="${CXXARCH:--march=native}"
 CXXSTD="${CXXSTD:--std=c++2a}"
 CXXFLAGSEXTRA="${CXXFLAGSEXTRA:-}"
@@ -26,7 +26,6 @@ BIN="${BIN:-signalbackup-tools}"
 
 # CONFIG: brew
 if [ "$CONFIG" = "brew" ] ; then
-  CXXFLAGS="-Wall -Wextra -Wshadow -Woverloaded-virtual -pedantic -O3 -flto"
   LDFLAGS="-Wall -Wextra -O3 -flto=auto"
 fi
 
@@ -37,14 +36,14 @@ fi
 
 # CONFIG: nixpkgs-darwin
 if [ "$CONFIG" = "nixpkgs-darwin" ] ; then
-  CXXFLAGS="-Wall -Wextra -Wshadow -Wold-style-cast -Woverloaded-virtual -pedantic -O3"
+  CXXFLAGS="-Wall -Wextra -Wshadow -Woverloaded-virtual -pedantic -O3"
   LDFLAGS="-Wall -Wextra -Wl,-framework,Security -Wl,-framework,CoreFoundation -O3"
 fi
 
 # CONFIG: windows
 if [ "$CONFIG" = "windows" ] ; then
   CXX="x86_64-w64-mingw32-g++"
-  CXXFLAGS="-Wall -Wextra -Wshadow -Wold-style-cast -Woverloaded-virtual -pedantic -D_WIN32_WINNT=0x600 -I/usr/x86_64-w64-mingw32/include/ -O3 -flto"
+  CXXFLAGS="-Wall -Wextra -Wshadow -Woverloaded-virtual -pedantic -D_WIN32_WINNT=0x600 -I/usr/x86_64-w64-mingw32/include/ -O3 -flto"
   CXXARCH=""
   LDFLAGS="-Wall -Wextra -Wl,--as-needed -static-libgcc -static-libstdc++ -static -L/usr/x86_64-w64-mingw32/lib/ -O3 -s -flto=auto"
   LDLIBS="-lcrypto -lsqlite3 -lssp -luser32 -lcrypt32 -ladvapi32 -lgdi32 -lws2_32"
@@ -82,7 +81,6 @@ SRC=("keyvalueframe/statics.cc"
      "signalbackup/listrecipients.cc"
      "signalbackup/importwachat.cc"
      "signalbackup/dumpmedia.cc"
-     "signalbackup/getattachmentmetadata.cc"
      "signalbackup/makeidsunique.cc"
      "signalbackup/dtimportstickerpacks.cc"
      "signalbackup/getallthreadrecipients.cc"
@@ -218,6 +216,7 @@ SRC=("keyvalueframe/statics.cc"
      "desktopdatabase/getsecrets_linux.cc"
      "reactionlist/setauthor.cc"
      "jsondatabase/jsondatabase.cc"
+     "attachmentmetadata/getattachmentmetadata.cc"
      "fileencryptor/init.cc"
      "fileencryptor/encryptframe.cc"
      "fileencryptor/fileencryptor.cc"
@@ -263,7 +262,6 @@ OBJ=("keyvalueframe/o/statics.o"
      "signalbackup/o/listrecipients.o"
      "signalbackup/o/importwachat.o"
      "signalbackup/o/dumpmedia.o"
-     "signalbackup/o/getattachmentmetadata.o"
      "signalbackup/o/makeidsunique.o"
      "signalbackup/o/dtimportstickerpacks.o"
      "signalbackup/o/getallthreadrecipients.o"
@@ -399,6 +397,7 @@ OBJ=("keyvalueframe/o/statics.o"
      "desktopdatabase/o/getsecrets_linux.o"
      "reactionlist/o/setauthor.o"
      "jsondatabase/o/jsondatabase.o"
+     "attachmentmetadata/o/getattachmentmetadata.o"
      "fileencryptor/o/init.o"
      "fileencryptor/o/encryptframe.o"
      "fileencryptor/o/fileencryptor.o"

CMakeLists.txt

@@ -1,11 +1,5 @@
 cmake_minimum_required(VERSION 3.14)
 
-set(CMAKE_CXX_EXTENSIONS off)
-set(CMAKE_CXX_STANDARD 20)
-#if version > 3.30 && CMAKE_CXX_STANDARD_LATEST is set
-#set standard to CMAKE_CXX_STANDARD_LATEST (if set)
-set(CMAKE_CXX_STANDARD_REQUIRED on)
-
 if (APPLE)
   foreach (HOMEBREW_PKG openssl sqlite)
     execute_process(COMMAND brew --prefix ${HOMEBREW_PKG} OUTPUT_VARIABLE HOMEBREW_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -18,6 +12,27 @@ project(signalbackup-tools)
 find_package(OpenSSL REQUIRED)
 find_package(SQLite3 REQUIRED)
 
+set(CMAKE_CXX_EXTENSIONS off)
+if (CMAKE_VERSION VERSION_LESS "3.30")  # the CMAKE_CXX_STANDARD_LATEST variable was only introduced in 3.30
+  set(CMAKE_CXX_STANDARD 20)
+elseif(CMAKE_CXX_STANDARD_LATEST) # if cmake version is ok and the variable is set, use it
+  set(CMAKE_CXX_STANDARD ${CMAKE_CXX_STANDARD_LATEST})
+endif()
+set(CMAKE_CXX_STANDARD_REQUIRED on)
+
+# find and set DBUS include and library paths
+#if (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+#  find_package(PkgConfig REQUIRED)
+#  pkg_check_modules(DBUS REQUIRED dbus-1)
+#  include_directories(${DBUS_INCLUDE_DIRS})
+#  set(DBUS_LIBS_ABSOLUTE)
+#  foreach(lib ${DBUS_LIBRARIES})
+#    set(tmp DBUS_${lib}_ABS)
+#    find_library(${tmp} ${lib} ${DBUS_LIBRARY_DIR})
+#    list(APPEND DBUS_LIBS_ABSOLUTE ${${tmp}})
+#  endforeach()
+#endif()
+
 if (APPLE)
   find_library(SECLIB Security)
   if (NOT SECLIB)
@@ -33,3 +48,4 @@ file(GLOB_RECURSE SOURCES CONFIGURE_DEPENDS *.cc *.h)
 add_executable(signalbackup-tools ${SOURCES})
 
 target_link_libraries(signalbackup-tools OpenSSL::Crypto SQLite::SQLite3 ${SECLIB} ${CFLIB})
+#target_link_libraries(signalbackup-tools OpenSSL::Crypto SQLite::SQLite3 ${SECLIB} ${CFLIB} ${DBUS_LIBS_ABSOLUTE})

README.md

@@ -42,7 +42,7 @@ To compile this project, current stable released versions of the following are n
 
 To compile the program, just running `g++ -std=c++20 */*.cc *.cc -lcrypto -lsqlite3` should generally do the trick. Add any compiler flags you feel useful, I personally use at least `-O3 -Wall -Wextra`. When compiling with an old compiler version (gcc 8.x or clang <= 7), also add the `-lstdc++fs` flag and replace `-std=c++20` with `-std=c++17`.
 
-If you have `cmake` available on your system, running `cmake -B build && cmake --build build -j $(nproc)` inside the project directory should also produce a working binary (found in the directory 'build') and make use of multiple processors if available.
+If you have `cmake` available on your system, running `cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j $(nproc)` inside the project directory should also produce a working binary (found in the directory 'build') and make use of multiple processors if available.
 
 For people not comfortable compiling source code, a script is provided that should compile the binary on Arch and Fedora (and probably many other distributions). Assuming the needed [requirements](#requirements) are installed, a simple `sh BUILDSCRIPT` should build the program (or, when using bash on a multiprocessor system, use `bash BUILDSCRIPT_MULTIPROC.bash44` for a faster build, and let me know if it works).
 

attachmentmetadata/attachmentmetadata.h

@@ -0,0 +1,43 @@
+/*
+  Copyright (C) 2024  Selwin van Dijk
+
+  This file is part of signalbackup-tools.
+
+  signalbackup-tools is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  signalbackup-tools is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with signalbackup-tools.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+#ifndef ATTACHMENTMETADATA_H_
+#define ATTACHMENTMETADATA_H_
+
+#include <string>
+
+struct AttachmentMetadata
+{
+  int width;
+  int height;
+  std::string filetype;
+  long long int filesize;
+  std::string hash;
+  std::string filename;
+  operator bool() const
+  {
+    return (width != -1 && height != -1 && !filetype.empty() && filesize != 0);
+  }
+
+  static AttachmentMetadata getAttachmentMetaData(std::string const &filename, bool skiphash = false);
+  static AttachmentMetadata getAttachmentMetaData(std::string const &filename, unsigned char *data,
+                                                  long long int data_size, bool skiphash = false);
+};
+
+#endif

signalbackup/getattachmentmetadata.cc → attachmentmetadata/getattachmentmetadata.cc

@@ -17,15 +17,13 @@
   along with signalbackup-tools.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-#include "signalbackup.ih"
+#include "attachmentmetadata.h"
 
-#ifndef USE_CRYPTOPP
 #include <openssl/sha.h>
-#endif
 
 #include "../base64/base64.h"
 
-SignalBackup::AttachmentMetadata SignalBackup::getAttachmentMetaData(std::string const &file, unsigned char *data, long long int data_size, bool skiphash) const
+AttachmentMetadata AttachmentMetadata::getAttachmentMetaData(std::string const &file, unsigned char *data, long long int data_size, bool skiphash) // static
 {
   //struct AttachmentMetadata
   //{
@@ -270,7 +268,7 @@ SignalBackup::AttachmentMetadata SignalBackup::getAttachmentMetaData(std::string
   return AttachmentMetadata{-1, -1, std::string(), data_size, hash, file};
 }
 
-SignalBackup::AttachmentMetadata SignalBackup::getAttachmentMetaData(std::string const &file, bool skiphash) const
+AttachmentMetadata AttachmentMetadata::getAttachmentMetaData(std::string const &file, bool skiphash) //static
 {
 
   //struct AttachmentMetadata

autoversion.h

@@ -20,6 +20,6 @@
 #ifndef VERSION_H_
 #define VERSION_H_
 
-#define VERSIONDATE "20240819.211304"
+#define VERSIONDATE "20240822.211310"
 
 #endif

avatarframe/avatarframe.h

@@ -23,6 +23,7 @@
 #include <cstring>
 
 #include "../framewithattachment/framewithattachment.h"
+#include "../attachmentmetadata/attachmentmetadata.h"
 
 class AvatarFrame : public FrameWithAttachment
 {
@@ -35,6 +36,7 @@ class AvatarFrame : public FrameWithAttachment
   };
 
   static Registrar s_registrar;
+  std::optional<std::string> d_mimetype;
  public:
   inline explicit AvatarFrame(uint64_t count = 0);
   inline AvatarFrame(unsigned char *bytes, size_t length, uint64_t count = 0);
@@ -53,6 +55,8 @@ class AvatarFrame : public FrameWithAttachment
   inline virtual bool validate() const override;
   inline std::string getHumanData() const override;
   inline unsigned int getField(std::string const &str) const;
+  inline std::optional<std::string> mimetype() const;
+  inline unsigned char *attachmentData(bool *badmac = nullptr, bool verbose = false);
  private:
   inline uint64_t dataSize() const override;
 };
@@ -272,4 +276,21 @@ inline unsigned int AvatarFrame::getField(std::string const &str) const
   return FIELD::INVALID;
 }
 
+inline std::optional<std::string> AvatarFrame::mimetype() const
+{
+  return d_mimetype;
+}
+
+inline unsigned char *AvatarFrame::attachmentData(bool *badmac, bool verbose)
+{
+  unsigned char *data = FrameWithAttachment::attachmentData(badmac, verbose);
+  if (data && !d_mimetype) // try to get mimetype
+  {
+    AttachmentMetadata amd = AttachmentMetadata::getAttachmentMetaData(std::string(), data, d_attachmentdata_size, true/*skiphash*/);
+    if (!amd.filetype.empty())
+      d_mimetype = amd.filetype;
+  }
+  return data;
+}
+
 #endif

signalbackup/deleteattachments.cc

@@ -19,6 +19,8 @@
 
 #include "signalbackup.ih"
 
+#include "../attachmentmetadata/attachmentmetadata.h"
+
 bool SignalBackup::deleteAttachments(std::vector<long long int> const &threadids,
                                      std::string const &before, std::string const &after,
                                      long long int filesize, std::vector<std::string> const &mimetypes,
@@ -139,7 +141,7 @@ bool SignalBackup::deleteAttachments(std::vector<long long int> const &threadids
         {
           // update message ranges if present:
           std::pair<std::shared_ptr<unsigned char []>, size_t> brdata =
-            d_database.getSingleResultAs<std::pair<std::shared_ptr<unsigned char []>, size_t>>("SELECT message_ranges FROM " + d_mms_table + " WHERE LENGTH(message_ranges) != 0 AND _id = ?",
+            d_database.getSingleResultAs<std::pair<std::shared_ptr<unsigned char []>, size_t>>("SELECT " + d_mms_ranges + " FROM " + d_mms_table + " WHERE LENGTH(" + d_mms_ranges + ") != 0 AND _id = ?",
                                                                                                res.getValueAs<long long int>(i, d_part_mid), {nullptr, 0});
           if (brdata.second)
           {
@@ -162,7 +164,8 @@ bool SignalBackup::deleteAttachments(std::vector<long long int> const &threadids
               new_bodyrange_vec.addField<1>(new_bodyrange);
             }
 
-            if (!d_database.exec("UPDATE " + d_mms_table + " SET message_ranges = ? WHERE _id = ?", {std::make_pair(new_bodyrange_vec.data(), static_cast<size_t>(new_bodyrange_vec.size())), res.getValueAs<long long int>(i, d_part_mid)}))
+            if (!d_database.exec("UPDATE " + d_mms_table + " SET " + d_mms_ranges + " = ? WHERE _id = ?",
+                                 {std::make_pair(new_bodyrange_vec.data(), static_cast<size_t>(new_bodyrange_vec.size())), res.getValueAs<long long int>(i, d_part_mid)}))
               return false;
           }
 
@@ -171,9 +174,11 @@ bool SignalBackup::deleteAttachments(std::vector<long long int> const &threadids
           if (d_verbose) [[unlikely]]
             Logger::message("Updated ", d_database.changed(), " mention to adjust for prependbody");
 
-          if (!d_database.exec("UPDATE " + d_mms_table + " SET body = ? || body WHERE _id = ? AND (body IS NOT NULL AND body != '')", {prepend + "\n\n", res.getValueAs<long long int>(i, d_part_mid)}))
+          if (!d_database.exec("UPDATE " + d_mms_table + " SET body = ? || body WHERE _id = ? AND (body IS NOT NULL AND body != '')",
+                               {prepend + "\n\n", res.getValueAs<long long int>(i, d_part_mid)}))
             return false;
-          if (!d_database.exec("UPDATE " + d_mms_table + " SET body = ? WHERE _id = ? AND (body IS NULL OR body == '')", {prepend, res.getValueAs<long long int>(i, d_part_mid)}))
+          if (!d_database.exec("UPDATE " + d_mms_table + " SET body = ? WHERE _id = ? AND (body IS NULL OR body == '')",
+                               {prepend, res.getValueAs<long long int>(i, d_part_mid)}))
             return false;
         }
       }
@@ -250,7 +255,7 @@ bool SignalBackup::deleteAttachments(std::vector<long long int> const &threadids
           continue;
         }
 
-        AttachmentMetadata amd = getAttachmentMetaData(replace[j].second);
+        AttachmentMetadata amd = AttachmentMetadata::getAttachmentMetaData(replace[j].second);
         if (!amd)
         {
           Logger::message("Failed to get metadata on new attachment: \"", replace[j].second, "\", skipping...");

signalbackup/dtinsertattachments.cc

@@ -19,11 +19,14 @@
 
 #include "signalbackup.ih"
 
+#include "../attachmentmetadata/attachmentmetadata.h"
+
 bool SignalBackup::dtInsertAttachments(long long int mms_id, long long int unique_id, int numattachments, long long int haspreview,
                                      long long int rowid, SqliteDB const &ddb, std::string const &where, std::string const &databasedir,
                                      bool isquote, bool issticker)
 {
   bool quoted_linkpreview = false;
+  bool quoted_sticker = false;
   if (numattachments == -1 && isquote) // quote attachments, number not known yet
   {
     SqliteDB::QueryResults res;
@@ -45,6 +48,16 @@ bool SignalBackup::dtInsertAttachments(long long int mms_id, long long int uniqu
         //std::cout << "GOT QUOTED LINK PREVIEW" << std::endl;
       }
     }
+
+    if (numattachments == 0)
+    {
+      if (ddb.exec("SELECT json_extract(json, '$.sticker.data.path') IS NOT NULL AS quoteissticker FROM messages " + where, &res) &&
+          res.rows() == 1 && res.getValueAs<long long int>(0, "quoteissticker") != 0)
+      {
+        quoted_sticker = true;
+        numattachments = 1;
+      }
+    }
   }
 
   if (numattachments == 0)
@@ -80,7 +93,7 @@ bool SignalBackup::dtInsertAttachments(long long int mms_id, long long int uniqu
                " FROM messages WHERE rowid = ?",
                rowid, &linkpreview_results);
     }
-    if (issticker)
+    if (issticker || quoted_sticker)
       jsonpath = "$.sticker.data";
 
     if (quoted_linkpreview)
@@ -239,7 +252,7 @@ bool SignalBackup::dtInsertAttachments(long long int mms_id, long long int uniqu
     }
 
 
-    // get attachment metadata !! NOTE RAW POINTER
+    // get attachment metadata
     AttachmentMetadata amd;
     if (version >= 2) [[likely]]
     {
@@ -248,7 +261,7 @@ bool SignalBackup::dtInsertAttachments(long long int mms_id, long long int uniqu
       std::unique_ptr<unsigned char[]> att_data;
       if (dar.getAttachmentData(std::out_ptr(att_data), d_verbose) != 0)
 #else
-      unsigned char *att_data = nullptr;
+      unsigned char *att_data = nullptr; // !! NOTE RAW POINTER
       if (dar.getAttachmentData(&att_data, d_verbose) != 0)
 #endif
       {
@@ -257,15 +270,15 @@ bool SignalBackup::dtInsertAttachments(long long int mms_id, long long int uniqu
       }
 
 #if __cpp_lib_out_ptr >= 202106L
-      amd = getAttachmentMetaData(fullpath, att_data.get(), size); // get metadata from heap
+      amd = AttachmentMetadata::getAttachmentMetaData(fullpath, att_data.get(), size); // get metadata from heap
 #else
-      amd = getAttachmentMetaData(fullpath, att_data, size);       // get metadata from heap
+      amd = AttachmentMetadata::getAttachmentMetaData(fullpath, att_data, size);       // get metadata from heap
       if (att_data)
         delete[] att_data;
 #endif
     }
     else
-      amd = getAttachmentMetaData(fullpath);                        // get from file
+      amd = AttachmentMetadata::getAttachmentMetaData(fullpath);                        // get from file
 
     if (amd.filename.empty() || (amd.filesize == 0 && results_attachment_data.valueAsInt(0, "size", 0) != 0))
     {
@@ -311,7 +324,7 @@ bool SignalBackup::dtInsertAttachments(long long int mms_id, long long int uniqu
     long long int new_part_id = std::any_cast<long long int>(retval);
     //std::cout << "Inserted part, new id: " << new_part_id << std::endl;
 
-    if (issticker)
+    if (issticker || quoted_sticker)
     {
       // get the data from $.sticker (instead of $.sticker.data)
       SqliteDB::QueryResults stickerdata;