Typo in usage() (fixes #283). Fix using long instead of uint64 in som…

…e msgtypes (windows uses 4 byte longs). Use SQL paramters in search data generation (fixed #282). Fix various URI escaping on generated HTML. Set self as sender/recipient in XML database depending on message direction. Print output when XML recipient is unexpectedly not present.

arg/usage.cc

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2022-2024  Selwin van Dijk
+  Copyright (C) 2022-2025  Selwin van Dijk
 
   This file is part of signalbackup-tools.
 
@@ -145,7 +145,7 @@ R"*(
                                          blocked contacts in the database.
    --includesettings                     Optional modifier for `--exporthtml'. Generate a page showing
                                          settings found in the backup file.
-   --includefullcontactslist             Optional modifier for `--exporthtml'. Generate a page showing all
+   --includefullcontactlist              Optional modifier for `--exporthtml'. Generate a page showing all
                                          contacts present in the backups database. These include hidden
                                          and blocked contacts, but also system contacts who might not use
                                          Signal at all.

autoversion.h

@@ -20,6 +20,6 @@
 #ifndef VERSION_H_
 #define VERSION_H_
 
-#define VERSIONDATE "20250206.160314"
+#define VERSIONDATE "20250207.154020"
 
 #endif

msgtypes/msgtypes.h

@@ -173,62 +173,62 @@ struct Types
     return type == MISSED_VIDEO_CALL_TYPE;
   }
 
-  inline static bool isJoined(long type)
+  inline static bool isJoined(uint64_t type)
   {
     return (type & BASE_TYPE_MASK) == JOINED_TYPE;
   }
 
-  inline static bool isNumberChange(long type)
+  inline static bool isNumberChange(uint64_t type)
   {
     return type == CHANGE_NUMBER_TYPE;
   }
 
-  inline static bool isDonationRequest(long type)
+  inline static bool isDonationRequest(uint64_t type)
   {
     return type == BOOST_REQUEST_TYPE;
   }
 
-  inline static bool isExpirationTimerUpdate(long type)
+  inline static bool isExpirationTimerUpdate(uint64_t type)
   {
     return (type & EXPIRATION_TIMER_UPDATE_BIT) != 0;
   }
 
-  inline static bool isIdentityUpdate(long type)
+  inline static bool isIdentityUpdate(uint64_t type)
   {
     return (type & KEY_EXCHANGE_MASK) == KEY_EXCHANGE_IDENTITY_UPDATE_BIT;
   }
 
-  inline static bool isIdentityVerified(long type)
+  inline static bool isIdentityVerified(uint64_t type)
   {
     return (type & KEY_EXCHANGE_MASK) == KEY_EXCHANGE_IDENTITY_VERIFIED_BIT;
   }
 
-  inline static bool isIdentityDefault(long type)
+  inline static bool isIdentityDefault(uint64_t type)
   {
     return (type & KEY_EXCHANGE_MASK) == KEY_EXCHANGE_IDENTITY_DEFAULT_BIT;
   }
 
-  inline static bool isSecureType(long type)
+  inline static bool isSecureType(uint64_t type)
   {
     return (type & SECURE_MESSAGE_BIT) != 0;
   }
 
-  inline static bool isEndSession(long type)
+  inline static bool isEndSession(uint64_t type)
   {
     return (type & END_SESSION_BIT) != 0;
   }
 
-  inline static bool isProfileChange(long type)
+  inline static bool isProfileChange(uint64_t type)
   {
     return (type & BASE_TYPE_MASK) == PROFILE_CHANGE_TYPE;
   }
 
-  inline static bool isMessageRequestAccepted(long type)
+  inline static bool isMessageRequestAccepted(uint64_t type)
   {
     return (type & SPECIAL_TYPES_MASK) == SPECIAL_TYPE_MESSAGE_REQUEST_ACCEPTED;
   }
 
-  inline static bool isStatusMessage(long type)
+  inline static bool isStatusMessage(uint64_t type)
   {
     return isCallType(type) || isGroupUpdate(type) || isGroupV2(type) ||
       isGroupQuit(type) || isIdentityUpdate(type) || isIdentityVerified(type) ||

signalbackup/decodestatusmessage.cc

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2019-2024  Selwin van Dijk
+  Copyright (C) 2019-2025  Selwin van Dijk
 
   This file is part of signalbackup-tools.
 

signalbackup/exporthtml.cc

@@ -626,24 +626,28 @@ bool SignalBackup::exportHtml(std::string const &directory, std::vector<long lon
 
         if (searchpage && (!Types::isStatusMessage(msg_info.type) && !msg_info.body.empty()))
         {
+          // all pages end in ".html", slice it off
+          std::string page(msg_info.threaddir + "/" + msg_info.filename, 0, msg_info.threaddir.size() + msg_info.filename.size() + 1 - 5);
+
           // because the body is already escaped for html at this point, we get it fresh from database (and have sqlite do the json formatting)
           if (!d_database.exec("SELECT json_object("
                                "'id', " + d_mms_table + "._id, "
                                "'b', " + d_mms_table + ".body, "
                                "'f', " + d_mms_table + "." + d_mms_recipient_id + ", "
                                "'tr', thread." + d_thread_recipient_id + ", "
                                "'o', (" + d_mms_table + "." + d_mms_type + " & 0x1F) IN (2,11,21,22,23,24,25,26), "
-                               "'d', (" + d_mms_table + ".date_received / 1000 - 1404165600), " // loose the last three digits (miliseconds, they are never displayed anyway).
-                                                                                                // subtract "2014-07-01". Signals initial release was 2014-07-29, negative numbers should work otherwise anyway.
-                               "'p', " + "SUBSTR(\"" + msg_info.threaddir + "/" + msg_info.filename + "\", 1, LENGTH(\"" + msg_info.threaddir + "/" + msg_info.filename + "\") - 5)" + ") AS line, " // all pages end in ".html", slice it off
+                               "'d', (" + d_mms_table + ".date_received / 1000 - 1404165600), " // lose the last three digits (miliseconds, they are never displayed anyway).
+                                                                                                // subtract "2014-07-01". Signals initial release was 2014-07-29, negative
+                                                                                                // numbers should work otherwise anyway.
+                               "'p', ?) AS line,"
                                + d_part_table + "._id AS rowid, " +
                                (d_database.tableContainsColumn(d_part_table, "unique_id") ?
                                 d_part_table + ".unique_id AS uniqueid" : "-1 AS uniqueid") +
                                " FROM " + d_mms_table + " "
                                "LEFT JOIN thread ON thread._id IS " + d_mms_table + ".thread_id "
                                "LEFT JOIN " + d_part_table + " ON " + d_part_table + "." + d_part_mid + " IS " + d_mms_table + "._id AND " + d_part_table + "." + d_part_ct + " = 'text/x-signal-plain' AND " + d_part_table + ".quote = 0 "
                                "WHERE " + d_mms_table + "._id = ?",
-                               msg_info.msg_id, &search_idx_results) ||
+                               {page, msg_info.msg_id}, &search_idx_results) ||
               search_idx_results.rows() < 1) [[unlikely]]
           {
             Logger::warning("Search_idx query failed or no results");

signalbackup/htmlwriteblockedlist.cc

@@ -300,8 +300,8 @@ bool SignalBackup::HTMLwriteBlockedlist(std::string const &dir, std::map<long lo
       else
         avatarpath += "/";
 
-      bepaald::replaceAll(&avatarpath, '\"', R"(\")");
       HTMLescapeUrl(&avatarpath);
+      bepaald::replaceAll(&avatarpath, '\"', R"(\")");
 
       outputfile
         << "      .avatar-" << rec_id << " {\n"

signalbackup/htmlwritecalllog.cc

@@ -333,6 +333,7 @@ void SignalBackup::HTMLwriteCallLog(std::vector<long long int> const &threads, s
         std::string raw_avatar_path(getRecipientInfoFromMap(recipientinfo, peer).display_name);
         WIN_LIMIT_FILENAME_LENGTH(raw_avatar_path);
         std::string avatar_path(sanitizeFilename(raw_avatar_path) + " (_id" + bepaald::toString(threadid) + ")");
+        HTMLescapeUrl(&avatar_path);
         bepaald::replaceAll(&avatar_path, '\"', R"(\")");
         std::string avatar_extension = getAvatarExtension(peer);
 

signalbackup/htmlwritefullcontacts.cc

@@ -294,7 +294,11 @@ bool SignalBackup::HTMLwriteFullContacts(std::string const &dir, std::map<long l
         WIN_LIMIT_FILENAME_LENGTH(raw_thread_dir);
         std::string thread_dir(sanitizeFilename(raw_thread_dir) + " (_id" + bepaald::toString(thread_id) + ")");
         if (bepaald::fileOrDirExists(dir + "/" + thread_dir + "/media/Avatar_" + bepaald::toString(rec_id) + "." + avatar_extension))
+        {
+          HTMLescapeUrl(&thread_dir);
+          bepaald::replaceAll(&thread_dir, '\"', R"(\")");
           avatarpath = std::move(thread_dir);
+        }
       }
 
       if (avatarpath.empty()) // avatar not already present anywhere, write out own...
@@ -309,9 +313,6 @@ bool SignalBackup::HTMLwriteFullContacts(std::string const &dir, std::map<long l
       else
         avatarpath += "/";
 
-      bepaald::replaceAll(&avatarpath, '\"', R"(\")");
-      HTMLescapeUrl(&avatarpath);
-
       outputfile
         << "      .avatar-" << rec_id << " {\n"
         << "        background-image: url(\"" << avatarpath << "media/Avatar_" << rec_id << "." << avatar_extension << "\");\n"

signalbackup/htmlwriteindex.cc

@@ -354,8 +354,8 @@ bool SignalBackup::HTMLwriteIndexImpl(std::vector<long long int> const &threads,
       WIN_LIMIT_FILENAME_LENGTH(raw_avatar_path);
       std::string avatar_path(sanitizeFilename(raw_avatar_path) + " (_id" + results(i, "_id") + ")");
       std::string avatar_extension = getAvatarExtension(rec_id);
-      bepaald::replaceAll(&avatar_path, '\"', R"(\")");
       HTMLescapeUrl(&avatar_path);
+      bepaald::replaceAll(&avatar_path, '\"', R"(\")");
 
       outputfile
         << "      .avatar-" << rec_id << " {\n"

signalbackup/importfromplaintextbackup.cc

@@ -52,22 +52,32 @@ bool SignalBackup::importFromPlaintextBackup(std::unique_ptr<SignalPlaintextBack
     return false;
   }
 
-  // mms messages that are 1-on-1 and outgoing, often only have a targetaddress, but no source addresses....
-  // the source is implied by the fact it's outgoing:
-  // same for incoming actually, and possibly also for non 1-on-1  messages...
+  // mms messages seem to usually omit the self-address, it is implied to be the sourceaddress when the message is outgoing,
+  // or (one of the) targetaddresses when the message is incoming.
   std::string selfe164(selfphone);
   if (selfe164.empty())
     selfe164 = d_database.getSingleResultAs<std::string>("SELECT " + d_recipient_e164 + " FROM recipient WHERE _id = ?", d_selfid, std::string());
   if (selfe164.empty())
     Logger::warning("Failed to get self phone");
   else
   {
-    // set source address
+
+    //ptdb->d_database.prettyPrint(false, "SELECT DISTINCT sourceaddress FROM smses WHERE ismms = 1 AND type = 2");
+    // set source address to self for outgoing
     ptdb->d_database.exec("UPDATE smses SET sourceaddress = ? WHERE "
-                          "sourceaddress IS NULL AND targetaddresses IS NOT NULL AND type = 2", selfe164);
+                          "NULLIF(sourceaddress, '') IS NULL AND ismms = 1 AND type = 2", selfe164);
+    //ptdb->d_database.prettyPrint(false, "SELECT DISTINCT sourceaddress FROM smses WHERE ismms = 1 AND type = 2");
 
+    //ptdb->d_database.prettyPrint(false, "SELECT DISTINCT targetaddresses FROM smses WHERE ismms = 1 AND type = 1");
     // set target address:
-    //...
+    ptdb->d_database.exec("UPDATE smses SET targetaddresses = "
+                          "CASE "
+                          "  WHEN targetaddresses IS NULL THEN json_array(?1)"
+                          "  ELSE json_insert(targetaddresses, '$[#]', ?1) "
+                          "END "
+                          "WHERE ismms = 1 AND type = 1", selfe164);
+    //ptdb->d_database.prettyPrint(false, "SELECT DISTINCT targetaddresses FROM smses WHERE ismms = 1 AND type = 1");
+
   }
 
   std::vector<std::pair<std::string, std::string>> dateranges;
@@ -258,6 +268,40 @@ bool SignalBackup::importFromPlaintextBackup(std::unique_ptr<SignalPlaintextBack
     if (rid == -1)
     {
       Logger::error("Failed to find source_recipient_id in contactmap (", makePrintable(pt_messages_sourceaddress), "). Should be present at this point. Skipping message");
+
+      Logger::error_indent("Extra info:");
+      Logger::error_indent("Thread recipient: ", trid);
+      std::string groupid = d_database.getSingleResultAs<std::string>("SELECT group_id FROM recipient WHERE _id = ?", trid, std::string());
+      if (!groupid.empty())
+      {
+        Logger::error_indent("Thread is group");
+
+        std::vector<long long int> members;
+        if (!getGroupMembersModern(&members, groupid))
+          Logger::error("Failed to get group members");
+
+        Logger::error_indent("Known group members (", members.size(), "): ");
+
+        for (auto m : members)
+        {
+          std::string phone = d_database.getSingleResultAs<std::string>("SELECT e164 FROM recipient WHERE _id = ?", m, std::string());
+          Logger::error_indent(" - ", m, " : ", phone.empty() ? "(empty)" : makePrintable(phone));
+        }
+      }
+      else
+        Logger::error_indent("Thread is 1-on-1");
+
+      Logger::error_indent("First message for this address was (probably):");
+      ptdb->d_database.prettyPrint(false,
+                                   "SELECT rowid, date, type, read, contact_name, address, numattachments, "
+                                   "sourceaddress, targetaddresses, ismms, skip FROM smses WHERE address = ? "
+                                   "ORDER BY date LIMIT 1", pt_messages_address);
+      Logger::error_indent("Current message for this address was:");
+      ptdb->d_database.prettyPrint(false,
+                                   "SELECT rowid, date, type, read, contact_name, address, numattachments, "
+                                   "sourceaddress, targetaddresses, ismms, skip FROM smses WHERE rowid = ? "
+                                   "ORDER BY date LIMIT 1", pt_messages.value(i, "rowid"));
+
       continue;
     }