Adds ios history migration and example

CHANGES_UNRELEASED.md

@@ -23,3 +23,7 @@ Use the template below to make assigning a version number during the release cut
 
 ### What's Changed
  - Added metrics for the `run_maintenance()` method.  This can be used by consumers to decide when to schedule the next `run_maintenance()` call and to check if calls are taking too much time.
+### What's new
+  - Exposed a function in Swift `migrateHistoryFromBrowserDb` to migrate history from `browser.db` to `places.db`, the function will migrate all the local visits in one go. ([#5077](https://github.com/mozilla/application-services/pull/5077)).
+    - The migration might take some time if a user had a lot of history, so make sure it is **not** run on a thread that shouldn't wait.
+    - The migration runs on a writer connection. This means that other writes to the `places.db` will be delayed until the migration is done.

components/places/ios/Places/Places.swift

@@ -933,4 +933,11 @@ public class PlacesWriteConnection: PlacesReadConnection {
             return try self.conn.applyObservation(visit: visitObservation)
         }
     }
+
+    open func migrateHistoryFromBrowserDb(path: String, lastSyncTimestamp: Int64) throws -> HistoryMigrationResult {
+        return try queue.sync {
+            try self.checkApi()
+            return try self.conn.placesHistoryImportFromIos(dbPath: path, lastSyncTimestamp: lastSyncTimestamp)
+        }
+    }
 }

components/places/src/ffi.rs

@@ -7,9 +7,12 @@
 use crate::api::matcher::{self, search_frecent, SearchParams};
 use crate::api::places_api::places_api_new;
 use crate::error::PlacesError;
-use crate::import::fennec::import_bookmarks;
-use crate::import::fennec::import_history;
+use crate::import::common::HistoryMigrationResult;
 use crate::import::fennec::import_pinned_sites;
+use crate::import::import_fennec_bookmarks;
+use crate::import::import_fennec_history;
+use crate::import::import_ios_bookmarks;
+use crate::import::import_ios_history;
 use crate::storage;
 use crate::storage::bookmarks;
 use crate::storage::bookmarks::BookmarkPosition;
@@ -218,17 +221,17 @@ impl PlacesApi {
     }
 
     fn places_history_import_from_fennec(&self, db_path: String) -> Result<String> {
-        let metrics = import_history(self, db_path.as_str())?;
+        let metrics = import_fennec_history(self, db_path.as_str())?;
         Ok(serde_json::to_string(&metrics)?)
     }
 
     fn places_bookmarks_import_from_fennec(&self, db_path: String) -> Result<String> {
-        let metrics = import_bookmarks(self, db_path.as_str())?;
+        let metrics = import_fennec_bookmarks(self, db_path.as_str())?;
         Ok(serde_json::to_string(&metrics)?)
     }
 
     fn places_bookmarks_import_from_ios(&self, db_path: String) -> Result<()> {
-        import_bookmarks(self, db_path.as_str())?;
+        import_ios_bookmarks(self, db_path.as_str())?;
         Ok(())
     }
 
@@ -578,6 +581,14 @@ impl PlacesConnection {
     fn bookmarks_update(&self, item: BookmarkUpdateInfo) -> Result<()> {
         self.with_conn(|conn| bookmarks::update_bookmark_from_info(conn, item))
     }
+
+    fn places_history_import_from_ios(
+        &self,
+        db_path: String,
+        last_sync_timestamp: i64,
+    ) -> Result<HistoryMigrationResult> {
+        self.with_conn(|conn| import_ios_history(conn, &db_path, last_sync_timestamp))
+    }
 }
 
 impl AsRef<SqlInterruptHandle> for PlacesConnection {

components/places/src/import/common.rs

@@ -4,7 +4,9 @@
 
 use crate::db::PlacesDb;
 use crate::error::*;
-use rusqlite::named_params;
+use rusqlite::{named_params, Connection};
+use serde::Serialize;
+use sql_support::ConnExt;
 use types::Timestamp;
 use url::Url;
 
@@ -27,24 +29,46 @@ pub mod sql_fns {
     use types::Timestamp;
     use url::Url;
 
-    #[inline(never)]
-    pub fn sanitize_timestamp(ctx: &Context<'_>) -> Result<Timestamp> {
+    fn sanitize_timestamp(ts: i64) -> Result<Timestamp> {
         let now = *NOW;
         let is_sane = |ts: Timestamp| -> bool { Timestamp::EARLIEST <= ts && ts <= now };
-        if let Ok(ts) = ctx.get::<i64>(0) {
-            let ts = Timestamp(u64::try_from(ts).unwrap_or(0));
-            if is_sane(ts) {
-                return Ok(ts);
-            }
-            // Maybe the timestamp was actually in μs?
-            let ts = Timestamp(ts.as_millis() / 1000);
-            if is_sane(ts) {
-                return Ok(ts);
-            }
+        let ts = Timestamp(u64::try_from(ts).unwrap_or(0));
+        if is_sane(ts) {
+            return Ok(ts);
+        }
+        // Maybe the timestamp was actually in μs?
+        let ts = Timestamp(ts.as_millis() / 1000);
+        if is_sane(ts) {
+            return Ok(ts);
         }
         Ok(now)
     }
 
+    // Unfortunately dates for history visits in old iOS databases
+    // have a type of `REAL` in their schema. This means they are represented
+    // as a float value and have to be read as f64s.
+    // This is unconventional, and you probably don't need to use
+    // this function otherwise.
+    #[inline(never)]
+    pub fn sanitize_float_timestamp(ctx: &Context<'_>) -> Result<Timestamp> {
+        let ts = ctx
+            .get::<f64>(0)
+            .map(|num| {
+                if num.is_normal() && num > 0.0 {
+                    num.round() as i64
+                } else {
+                    0
+                }
+            })
+            .unwrap_or(0);
+        sanitize_timestamp(ts)
+    }
+
+    #[inline(never)]
+    pub fn sanitize_integer_timestamp(ctx: &Context<'_>) -> Result<Timestamp> {
+        sanitize_timestamp(ctx.get::<i64>(0).unwrap_or(0))
+    }
+
     // Possibly better named as "normalize URL" - even in non-error cases, the
     // result string may not be the same href used passed as input.
     #[inline(never)]
@@ -128,3 +152,58 @@ impl Drop for ExecuteOnDrop<'_> {
         }
     }
 }
+
+pub fn select_count(conn: &PlacesDb, stmt: &str) -> Result<u32> {
+    let count: Result<Option<u32>> =
+        conn.try_query_row(stmt, [], |row| Ok(row.get::<_, u32>(0)?), false);
+    count.map(|op| op.unwrap_or(0))
+}
+
+#[derive(Serialize, PartialEq, Eq, Debug, Clone, Default)]
+pub struct HistoryMigrationResult {
+    pub num_total: u32,
+    pub num_succeeded: u32,
+    pub num_failed: u32,
+    pub total_duration: u64,
+}
+
+pub fn define_history_migration_functions(c: &Connection) -> Result<()> {
+    use rusqlite::functions::FunctionFlags;
+    c.create_scalar_function(
+        "validate_url",
+        1,
+        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
+        crate::import::common::sql_fns::validate_url,
+    )?;
+    c.create_scalar_function(
+        "sanitize_timestamp",
+        1,
+        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
+        crate::import::common::sql_fns::sanitize_integer_timestamp,
+    )?;
+    c.create_scalar_function(
+        "hash",
+        -1,
+        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
+        crate::db::db::sql_fns::hash,
+    )?;
+    c.create_scalar_function(
+        "generate_guid",
+        0,
+        FunctionFlags::SQLITE_UTF8,
+        crate::db::db::sql_fns::generate_guid,
+    )?;
+    c.create_scalar_function(
+        "sanitize_utf8",
+        1,
+        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
+        crate::import::common::sql_fns::sanitize_utf8,
+    )?;
+    c.create_scalar_function(
+        "sanitize_float_timestamp",
+        1,
+        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
+        crate::import::common::sql_fns::sanitize_float_timestamp,
+    )?;
+    Ok(())
+}

components/places/src/import/fennec/bookmarks.rs

@@ -437,7 +437,7 @@ fn bookmark_data_from_fennec_pinned(
 }
 
 mod sql_fns {
-    use crate::import::common::sql_fns::{sanitize_timestamp, sanitize_utf8, validate_url};
+    use crate::import::common::sql_fns::{sanitize_integer_timestamp, sanitize_utf8, validate_url};
     use rusqlite::{
         functions::{Context, FunctionFlags},
         Connection, Result,
@@ -460,7 +460,7 @@ mod sql_fns {
             "sanitize_timestamp",
             1,
             FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
-            sanitize_timestamp,
+            sanitize_integer_timestamp,
         )?;
         c.create_scalar_function(
             "sanitize_utf8",

components/places/src/import/fennec/history.rs

@@ -4,11 +4,10 @@
 
 use crate::api::places_api::PlacesApi;
 use crate::bookmark_sync::engine::update_frecencies;
-use crate::db::db::PlacesDb;
 use crate::error::*;
-use crate::import::common::attached_database;
-use rusqlite::Connection;
-use serde_derive::*;
+use crate::import::common::{
+    attached_database, define_history_migration_functions, select_count, HistoryMigrationResult,
+};
 use sql_support::ConnExt;
 use std::time::Instant;
 use url::Url;
@@ -17,14 +16,6 @@ use url::Url;
 // However, 36 was quite easy to obtain test databases for, and it shipped with quite an old ESR version (52).
 const FENNEC_DB_VERSION: i64 = 34;
 
-#[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone, Default)]
-pub struct HistoryMigrationResult {
-    pub num_total: u32,
-    pub num_succeeded: u32,
-    pub num_failed: u32,
-    pub total_duration: u128,
-}
-
 pub fn import(
     places_api: &PlacesApi,
     path: impl AsRef<std::path::Path>,
@@ -33,19 +24,13 @@ pub fn import(
     do_import(places_api, url)
 }
 
-pub fn select_count(conn: &PlacesDb, stmt: &str) -> u32 {
-    let count: Result<Option<u32>> =
-        conn.try_query_row(stmt, [], |row| Ok(row.get::<_, u32>(0)?), false);
-    count.unwrap().unwrap()
-}
-
 fn do_import(places_api: &PlacesApi, android_db_file_url: Url) -> Result<HistoryMigrationResult> {
     let conn_mutex = places_api.get_sync_connection()?;
     let conn = conn_mutex.lock();
 
     let scope = conn.begin_interrupt_scope()?;
 
-    define_sql_functions(&conn)?;
+    define_history_migration_functions(&conn)?;
 
     // Not sure why, but apparently beginning a transaction sometimes
     // fails if we open the DB as read-only. Hopefully we don't
@@ -64,7 +49,7 @@ fn do_import(places_api: &PlacesApi, android_db_file_url: Url) -> Result<History
     let tx = conn.begin_transaction()?;
 
     log::debug!("Counting Fennec history visits");
-    let num_total = select_count(&conn, &COUNT_FENNEC_HISTORY_VISITS);
+    let num_total = select_count(&conn, &COUNT_FENNEC_HISTORY_VISITS)?;
 
     log::debug!("Creating and populating staging table");
     conn.execute_batch(&CREATE_STAGING_TABLE)?;
@@ -89,7 +74,7 @@ fn do_import(places_api: &PlacesApi, android_db_file_url: Url) -> Result<History
     log::info!("Successfully imported history visits!");
 
     log::debug!("Counting Fenix history visits");
-    let num_succeeded = select_count(&conn, &COUNT_FENIX_HISTORY_VISITS);
+    let num_succeeded = select_count(&conn, &COUNT_FENIX_HISTORY_VISITS)?;
     let num_failed = num_total - num_succeeded;
 
     auto_detach.execute_now()?;
@@ -98,7 +83,7 @@ fn do_import(places_api: &PlacesApi, android_db_file_url: Url) -> Result<History
         num_total,
         num_succeeded,
         num_failed,
-        total_duration: import_start.elapsed().as_millis(),
+        total_duration: import_start.elapsed().as_millis() as u64,
     };
 
     Ok(metrics)
@@ -171,38 +156,3 @@ lazy_static::lazy_static! {
         "SELECT COUNT(*) FROM main.moz_historyvisits"
     ;
 }
-
-pub(super) fn define_sql_functions(c: &Connection) -> Result<()> {
-    use rusqlite::functions::FunctionFlags;
-    c.create_scalar_function(
-        "validate_url",
-        1,
-        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
-        crate::import::common::sql_fns::validate_url,
-    )?;
-    c.create_scalar_function(
-        "sanitize_timestamp",
-        1,
-        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
-        crate::import::common::sql_fns::sanitize_timestamp,
-    )?;
-    c.create_scalar_function(
-        "hash",
-        -1,
-        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
-        crate::db::db::sql_fns::hash,
-    )?;
-    c.create_scalar_function(
-        "generate_guid",
-        0,
-        FunctionFlags::SQLITE_UTF8,
-        crate::db::db::sql_fns::generate_guid,
-    )?;
-    c.create_scalar_function(
-        "sanitize_utf8",
-        1,
-        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
-        crate::import::common::sql_fns::sanitize_utf8,
-    )?;
-    Ok(())
-}

components/places/src/import/ios.rs

@@ -0,0 +1,8 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+pub mod bookmarks;
+pub mod history;
+pub use bookmarks::import as import_bookmarks;
+pub use history::import as import_history;