diff --git a/CHANGES_UNRELEASED.md b/CHANGES_UNRELEASED.md index e60d0edee2..fd5cd851e1 100644 --- a/CHANGES_UNRELEASED.md +++ b/CHANGES_UNRELEASED.md @@ -23,3 +23,7 @@ Use the template below to make assigning a version number during the release cut ### What's Changed - Added metrics for the `run_maintenance()` method. This can be used by consumers to decide when to schedule the next `run_maintenance()` call and to check if calls are taking too much time. +### What's new + - Exposed a function in Swift `migrateHistoryFromBrowserDb` to migrate history from `browser.db` to `places.db`, the function will migrate all the local visits in one go. ([#5077](https://github.com/mozilla/application-services/pull/5077)). + - The migration might take some time if a user had a lot of history, so make sure it is **not** run on a thread that shouldn't wait. + - The migration runs on a writer connection. This means that other writes to the `places.db` will be delayed until the migration is done. diff --git a/components/places/ios/Places/Places.swift b/components/places/ios/Places/Places.swift index 3f393f2a8d..019c7e0627 100644 --- a/components/places/ios/Places/Places.swift +++ b/components/places/ios/Places/Places.swift @@ -933,4 +933,11 @@ public class PlacesWriteConnection: PlacesReadConnection { return try self.conn.applyObservation(visit: visitObservation) } } + + open func migrateHistoryFromBrowserDb(path: String, lastSyncTimestamp: Int64) throws -> HistoryMigrationResult { + return try queue.sync { + try self.checkApi() + return try self.conn.placesHistoryImportFromIos(dbPath: path, lastSyncTimestamp: lastSyncTimestamp) + } + } } diff --git a/components/places/src/ffi.rs b/components/places/src/ffi.rs index 4450e90d5d..6a11331184 100644 --- a/components/places/src/ffi.rs +++ b/components/places/src/ffi.rs @@ -7,9 +7,12 @@ use crate::api::matcher::{self, search_frecent, SearchParams}; use crate::api::places_api::places_api_new; use crate::error::PlacesError; -use crate::import::fennec::import_bookmarks; -use crate::import::fennec::import_history; +use crate::import::common::HistoryMigrationResult; use crate::import::fennec::import_pinned_sites; +use crate::import::import_fennec_bookmarks; +use crate::import::import_fennec_history; +use crate::import::import_ios_bookmarks; +use crate::import::import_ios_history; use crate::storage; use crate::storage::bookmarks; use crate::storage::bookmarks::BookmarkPosition; @@ -218,17 +221,17 @@ impl PlacesApi { } fn places_history_import_from_fennec(&self, db_path: String) -> Result { - let metrics = import_history(self, db_path.as_str())?; + let metrics = import_fennec_history(self, db_path.as_str())?; Ok(serde_json::to_string(&metrics)?) } fn places_bookmarks_import_from_fennec(&self, db_path: String) -> Result { - let metrics = import_bookmarks(self, db_path.as_str())?; + let metrics = import_fennec_bookmarks(self, db_path.as_str())?; Ok(serde_json::to_string(&metrics)?) } fn places_bookmarks_import_from_ios(&self, db_path: String) -> Result<()> { - import_bookmarks(self, db_path.as_str())?; + import_ios_bookmarks(self, db_path.as_str())?; Ok(()) } @@ -578,6 +581,14 @@ impl PlacesConnection { fn bookmarks_update(&self, item: BookmarkUpdateInfo) -> Result<()> { self.with_conn(|conn| bookmarks::update_bookmark_from_info(conn, item)) } + + fn places_history_import_from_ios( + &self, + db_path: String, + last_sync_timestamp: i64, + ) -> Result { + self.with_conn(|conn| import_ios_history(conn, &db_path, last_sync_timestamp)) + } } impl AsRef for PlacesConnection { diff --git a/components/places/src/import/common.rs b/components/places/src/import/common.rs index f62dc8683d..77c032e363 100644 --- a/components/places/src/import/common.rs +++ b/components/places/src/import/common.rs @@ -4,7 +4,9 @@ use crate::db::PlacesDb; use crate::error::*; -use rusqlite::named_params; +use rusqlite::{named_params, Connection}; +use serde::Serialize; +use sql_support::ConnExt; use types::Timestamp; use url::Url; @@ -27,24 +29,46 @@ pub mod sql_fns { use types::Timestamp; use url::Url; - #[inline(never)] - pub fn sanitize_timestamp(ctx: &Context<'_>) -> Result { + fn sanitize_timestamp(ts: i64) -> Result { let now = *NOW; let is_sane = |ts: Timestamp| -> bool { Timestamp::EARLIEST <= ts && ts <= now }; - if let Ok(ts) = ctx.get::(0) { - let ts = Timestamp(u64::try_from(ts).unwrap_or(0)); - if is_sane(ts) { - return Ok(ts); - } - // Maybe the timestamp was actually in μs? - let ts = Timestamp(ts.as_millis() / 1000); - if is_sane(ts) { - return Ok(ts); - } + let ts = Timestamp(u64::try_from(ts).unwrap_or(0)); + if is_sane(ts) { + return Ok(ts); + } + // Maybe the timestamp was actually in μs? + let ts = Timestamp(ts.as_millis() / 1000); + if is_sane(ts) { + return Ok(ts); } Ok(now) } + // Unfortunately dates for history visits in old iOS databases + // have a type of `REAL` in their schema. This means they are represented + // as a float value and have to be read as f64s. + // This is unconventional, and you probably don't need to use + // this function otherwise. + #[inline(never)] + pub fn sanitize_float_timestamp(ctx: &Context<'_>) -> Result { + let ts = ctx + .get::(0) + .map(|num| { + if num.is_normal() && num > 0.0 { + num.round() as i64 + } else { + 0 + } + }) + .unwrap_or(0); + sanitize_timestamp(ts) + } + + #[inline(never)] + pub fn sanitize_integer_timestamp(ctx: &Context<'_>) -> Result { + sanitize_timestamp(ctx.get::(0).unwrap_or(0)) + } + // Possibly better named as "normalize URL" - even in non-error cases, the // result string may not be the same href used passed as input. #[inline(never)] @@ -128,3 +152,58 @@ impl Drop for ExecuteOnDrop<'_> { } } } + +pub fn select_count(conn: &PlacesDb, stmt: &str) -> Result { + let count: Result> = + conn.try_query_row(stmt, [], |row| Ok(row.get::<_, u32>(0)?), false); + count.map(|op| op.unwrap_or(0)) +} + +#[derive(Serialize, PartialEq, Eq, Debug, Clone, Default)] +pub struct HistoryMigrationResult { + pub num_total: u32, + pub num_succeeded: u32, + pub num_failed: u32, + pub total_duration: u64, +} + +pub fn define_history_migration_functions(c: &Connection) -> Result<()> { + use rusqlite::functions::FunctionFlags; + c.create_scalar_function( + "validate_url", + 1, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + crate::import::common::sql_fns::validate_url, + )?; + c.create_scalar_function( + "sanitize_timestamp", + 1, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + crate::import::common::sql_fns::sanitize_integer_timestamp, + )?; + c.create_scalar_function( + "hash", + -1, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + crate::db::db::sql_fns::hash, + )?; + c.create_scalar_function( + "generate_guid", + 0, + FunctionFlags::SQLITE_UTF8, + crate::db::db::sql_fns::generate_guid, + )?; + c.create_scalar_function( + "sanitize_utf8", + 1, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + crate::import::common::sql_fns::sanitize_utf8, + )?; + c.create_scalar_function( + "sanitize_float_timestamp", + 1, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + crate::import::common::sql_fns::sanitize_float_timestamp, + )?; + Ok(()) +} diff --git a/components/places/src/import/fennec/bookmarks.rs b/components/places/src/import/fennec/bookmarks.rs index ee9a1a31dd..44e4078005 100644 --- a/components/places/src/import/fennec/bookmarks.rs +++ b/components/places/src/import/fennec/bookmarks.rs @@ -437,7 +437,7 @@ fn bookmark_data_from_fennec_pinned( } mod sql_fns { - use crate::import::common::sql_fns::{sanitize_timestamp, sanitize_utf8, validate_url}; + use crate::import::common::sql_fns::{sanitize_integer_timestamp, sanitize_utf8, validate_url}; use rusqlite::{ functions::{Context, FunctionFlags}, Connection, Result, @@ -460,7 +460,7 @@ mod sql_fns { "sanitize_timestamp", 1, FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, - sanitize_timestamp, + sanitize_integer_timestamp, )?; c.create_scalar_function( "sanitize_utf8", diff --git a/components/places/src/import/fennec/history.rs b/components/places/src/import/fennec/history.rs index a74a00b140..1714ebfcbf 100644 --- a/components/places/src/import/fennec/history.rs +++ b/components/places/src/import/fennec/history.rs @@ -4,11 +4,10 @@ use crate::api::places_api::PlacesApi; use crate::bookmark_sync::engine::update_frecencies; -use crate::db::db::PlacesDb; use crate::error::*; -use crate::import::common::attached_database; -use rusqlite::Connection; -use serde_derive::*; +use crate::import::common::{ + attached_database, define_history_migration_functions, select_count, HistoryMigrationResult, +}; use sql_support::ConnExt; use std::time::Instant; use url::Url; @@ -17,14 +16,6 @@ use url::Url; // However, 36 was quite easy to obtain test databases for, and it shipped with quite an old ESR version (52). const FENNEC_DB_VERSION: i64 = 34; -#[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone, Default)] -pub struct HistoryMigrationResult { - pub num_total: u32, - pub num_succeeded: u32, - pub num_failed: u32, - pub total_duration: u128, -} - pub fn import( places_api: &PlacesApi, path: impl AsRef, @@ -33,19 +24,13 @@ pub fn import( do_import(places_api, url) } -pub fn select_count(conn: &PlacesDb, stmt: &str) -> u32 { - let count: Result> = - conn.try_query_row(stmt, [], |row| Ok(row.get::<_, u32>(0)?), false); - count.unwrap().unwrap() -} - fn do_import(places_api: &PlacesApi, android_db_file_url: Url) -> Result { let conn_mutex = places_api.get_sync_connection()?; let conn = conn_mutex.lock(); let scope = conn.begin_interrupt_scope()?; - define_sql_functions(&conn)?; + define_history_migration_functions(&conn)?; // Not sure why, but apparently beginning a transaction sometimes // fails if we open the DB as read-only. Hopefully we don't @@ -64,7 +49,7 @@ fn do_import(places_api: &PlacesApi, android_db_file_url: Url) -> Result Result Result Result<()> { - use rusqlite::functions::FunctionFlags; - c.create_scalar_function( - "validate_url", - 1, - FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, - crate::import::common::sql_fns::validate_url, - )?; - c.create_scalar_function( - "sanitize_timestamp", - 1, - FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, - crate::import::common::sql_fns::sanitize_timestamp, - )?; - c.create_scalar_function( - "hash", - -1, - FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, - crate::db::db::sql_fns::hash, - )?; - c.create_scalar_function( - "generate_guid", - 0, - FunctionFlags::SQLITE_UTF8, - crate::db::db::sql_fns::generate_guid, - )?; - c.create_scalar_function( - "sanitize_utf8", - 1, - FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, - crate::import::common::sql_fns::sanitize_utf8, - )?; - Ok(()) -} diff --git a/components/places/src/import/ios.rs b/components/places/src/import/ios.rs new file mode 100644 index 0000000000..b6dbe09489 --- /dev/null +++ b/components/places/src/import/ios.rs @@ -0,0 +1,8 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pub mod bookmarks; +pub mod history; +pub use bookmarks::import as import_bookmarks; +pub use history::import as import_history; diff --git a/components/places/src/import/ios_bookmarks.rs b/components/places/src/import/ios/bookmarks.rs similarity index 65% rename from components/places/src/import/ios_bookmarks.rs rename to components/places/src/import/ios/bookmarks.rs index ccdffbf5e5..fe07fd7e18 100644 --- a/components/places/src/import/ios_bookmarks.rs +++ b/components/places/src/import/ios/bookmarks.rs @@ -75,15 +75,12 @@ use url::Url; /// - Use iosBookmarksStaging to fixup the data that was actually inserted. /// - Update frecency for new items. /// - Cleanup (Delete mirror and mirror structure, detach iOS database, etc). -pub fn import_ios_bookmarks( - places_api: &PlacesApi, - path: impl AsRef, -) -> Result<()> { +pub fn import(places_api: &PlacesApi, path: impl AsRef) -> Result<()> { let url = crate::util::ensure_url_path(path)?; - do_import_ios_bookmarks(places_api, url) + do_import(places_api, url) } -fn do_import_ios_bookmarks(places_api: &PlacesApi, ios_db_file_url: Url) -> Result<()> { +fn do_import(places_api: &PlacesApi, ios_db_file_url: Url) -> Result<()> { let conn_mutex = places_api.get_sync_connection()?; let conn = conn_mutex.lock(); @@ -176,12 +173,12 @@ fn populate_mirror_tags(db: &crate::PlacesDb) -> Result<()> { { let mut stmt = db.prepare( "SELECT mirror.id, stage.tags - FROM main.moz_bookmarks_synced mirror - JOIN temp.iosBookmarksStaging stage USING(guid) - -- iOS tags are JSON arrays of strings (or null). - -- Both [] and null are allowed for 'no tags' - WHERE stage.tags IS NOT NULL - AND stage.tags != '[]'", + FROM main.moz_bookmarks_synced mirror + JOIN temp.iosBookmarksStaging stage USING(guid) + -- iOS tags are JSON arrays of strings (or null). + -- Both [] and null are allowed for 'no tags' + WHERE stage.tags IS NOT NULL + AND stage.tags != '[]'", )?; let mut rows = stmt.query([])?; @@ -228,9 +225,9 @@ fn populate_mirror_tags(db: &crate::PlacesDb) -> Result<()> { for item_id in tagged_items { log::trace!("tagging {} with {}", item_id, tag); db.execute_cached( - "INSERT INTO main.moz_bookmarks_synced_tag_relation(itemId, tagId) VALUES(:item_id, :tag_id)", - named_params! { ":tag_id": tag_id, ":item_id": item_id }, - )?; + "INSERT INTO main.moz_bookmarks_synced_tag_relation(itemId, tagId) VALUES(:item_id, :tag_id)", + named_params! { ":tag_id": tag_id, ":item_id": item_id }, + )?; } } log::debug!("Tagged {} items with {} tags", tagged_count, tag_count); @@ -258,11 +255,11 @@ lazy_static::lazy_static! { static ref WIPE_MIRROR: String = format!( // Is omitting the roots right? "DELETE FROM main.moz_bookmarks_synced - WHERE guid NOT IN {roots}; - DELETE FROM main.moz_bookmarks_synced_structure - WHERE guid NOT IN {roots}; - UPDATE main.moz_bookmarks_synced - SET needsMerge = 0;", + WHERE guid NOT IN {roots}; + DELETE FROM main.moz_bookmarks_synced_structure + WHERE guid NOT IN {roots}; + UPDATE main.moz_bookmarks_synced + SET needsMerge = 0;", roots = ROOTS, ); // We omit: @@ -279,55 +276,55 @@ lazy_static::lazy_static! { // Insert any missing entries into moz_places that we'll need for this. static ref FILL_MOZ_PLACES: String = format!( "INSERT OR IGNORE INTO main.moz_places(guid, url, url_hash, frecency) - SELECT IFNULL((SELECT p.guid FROM main.moz_places p - WHERE p.url_hash = hash(b.bmkUri) AND p.url = b.bmkUri), - generate_guid()), - b.bmkUri, - hash(b.bmkUri), - -1 - FROM temp.iosBookmarksStaging b - WHERE b.bmkUri IS NOT NULL - AND b.type = {bookmark_type}", + SELECT IFNULL((SELECT p.guid FROM main.moz_places p + WHERE p.url_hash = hash(b.bmkUri) AND p.url = b.bmkUri), + generate_guid()), + b.bmkUri, + hash(b.bmkUri), + -1 + FROM temp.iosBookmarksStaging b + WHERE b.bmkUri IS NOT NULL + AND b.type = {bookmark_type}", bookmark_type = IosBookmarkType::Bookmark as u8, ); static ref POPULATE_MIRROR: String = format!( "REPLACE INTO main.moz_bookmarks_synced( - guid, - parentGuid, - serverModified, - needsMerge, - validity, - isDeleted, - kind, - dateAdded, - title, - placeId, - keyword - ) - SELECT - b.guid, - b.parentid, - b.modified, - 1, -- needsMerge - 1, -- VALIDITY_VALID - 0, -- isDeleted - CASE b.type - WHEN {ios_bookmark_type} THEN {bookmark_kind} - WHEN {ios_folder_type} THEN {folder_kind} - WHEN {ios_separator_type} THEN {separator_kind} - -- We filter out anything else when inserting into the stage table - END, - b.date_added, - b.title, - -- placeId - CASE WHEN b.bmkUri IS NULL - THEN NULL - ELSE (SELECT id FROM main.moz_places p - WHERE p.url_hash = hash(b.bmkUri) AND p.url = b.bmkUri) - END, - b.keyword - FROM iosBookmarksStaging b", + guid, + parentGuid, + serverModified, + needsMerge, + validity, + isDeleted, + kind, + dateAdded, + title, + placeId, + keyword + ) + SELECT + b.guid, + b.parentid, + b.modified, + 1, -- needsMerge + 1, -- VALIDITY_VALID + 0, -- isDeleted + CASE b.type + WHEN {ios_bookmark_type} THEN {bookmark_kind} + WHEN {ios_folder_type} THEN {folder_kind} + WHEN {ios_separator_type} THEN {separator_kind} + -- We filter out anything else when inserting into the stage table + END, + b.date_added, + b.title, + -- placeId + CASE WHEN b.bmkUri IS NULL + THEN NULL + ELSE (SELECT id FROM main.moz_places p + WHERE p.url_hash = hash(b.bmkUri) AND p.url = b.bmkUri) + END, + b.keyword + FROM iosBookmarksStaging b", bookmark_kind = SyncedBookmarkKind::Bookmark as u8, folder_kind = SyncedBookmarkKind::Folder as u8, separator_kind = SyncedBookmarkKind::Separator as u8, @@ -340,115 +337,115 @@ lazy_static::lazy_static! { } const POPULATE_MIRROR_STRUCTURE: &str = " -REPLACE INTO main.moz_bookmarks_synced_structure(guid, parentGuid, position) - SELECT structure.child, structure.parent, structure.idx FROM ios.bookmarksBufferStructure structure - WHERE EXISTS( - SELECT 1 FROM iosBookmarksStaging stage - WHERE stage.isLocal = 0 - AND stage.guid = structure.child - ); -REPLACE INTO main.moz_bookmarks_synced_structure(guid, parentGuid, position) - SELECT structure.child, structure.parent, structure.idx FROM ios.bookmarksLocalStructure structure - WHERE EXISTS( - SELECT 1 FROM iosBookmarksStaging stage - WHERE stage.isLocal != 0 - AND stage.guid = structure.child - ); -"; + REPLACE INTO main.moz_bookmarks_synced_structure(guid, parentGuid, position) + SELECT structure.child, structure.parent, structure.idx FROM ios.bookmarksBufferStructure structure + WHERE EXISTS( + SELECT 1 FROM iosBookmarksStaging stage + WHERE stage.isLocal = 0 + AND stage.guid = structure.child + ); + REPLACE INTO main.moz_bookmarks_synced_structure(guid, parentGuid, position) + SELECT structure.child, structure.parent, structure.idx FROM ios.bookmarksLocalStructure structure + WHERE EXISTS( + SELECT 1 FROM iosBookmarksStaging stage + WHERE stage.isLocal != 0 + AND stage.guid = structure.child + ); + "; lazy_static::lazy_static! { static ref POPULATE_STAGING: String = format!( "INSERT OR IGNORE INTO temp.iosBookmarksStaging( - guid, - type, - parentid, - pos, - title, - bmkUri, - keyword, - tags, - date_added, - modified, - isLocal - ) - SELECT - b.guid, - b.type, - b.parentid, - b.pos, - b.title, - CASE - WHEN b.bmkUri IS NOT NULL - THEN validate_url(b.bmkUri) - ELSE NULL - END as uri, - b.keyword, - b.tags, - sanitize_timestamp(b.date_added), - sanitize_timestamp(b.server_modified), - 0 - FROM ios.bookmarksBuffer b - WHERE NOT b.is_deleted - -- Skip anything also in `local` (we can't use `replace`, - -- since we use `IGNORE` to avoid inserting bad records) - AND ( - (b.guid IN {roots}) - OR - (b.guid NOT IN (SELECT l.guid FROM ios.bookmarksLocal l)) - ) - AND (b.type != {ios_bookmark_type} OR uri IS NOT NULL) - ; - INSERT OR IGNORE INTO temp.iosBookmarksStaging( - guid, - type, - parentid, - pos, - title, - bmkUri, - keyword, - tags, - date_added, - modified, - isLocal - ) - SELECT - l.guid, - l.type, - l.parentid, - l.pos, - l.title, - validate_url(l.bmkUri) as uri, - l.keyword, - l.tags, - sanitize_timestamp(l.date_added), - sanitize_timestamp(l.local_modified), - 1 - FROM ios.bookmarksLocal l - WHERE NOT l.is_deleted - AND uri IS NOT NULL - ;", + guid, + type, + parentid, + pos, + title, + bmkUri, + keyword, + tags, + date_added, + modified, + isLocal + ) + SELECT + b.guid, + b.type, + b.parentid, + b.pos, + b.title, + CASE + WHEN b.bmkUri IS NOT NULL + THEN validate_url(b.bmkUri) + ELSE NULL + END as uri, + b.keyword, + b.tags, + sanitize_timestamp(b.date_added), + sanitize_timestamp(b.server_modified), + 0 + FROM ios.bookmarksBuffer b + WHERE NOT b.is_deleted + -- Skip anything also in `local` (we can't use `replace`, + -- since we use `IGNORE` to avoid inserting bad records) + AND ( + (b.guid IN {roots}) + OR + (b.guid NOT IN (SELECT l.guid FROM ios.bookmarksLocal l)) + ) + AND (b.type != {ios_bookmark_type} OR uri IS NOT NULL) + ; + INSERT OR IGNORE INTO temp.iosBookmarksStaging( + guid, + type, + parentid, + pos, + title, + bmkUri, + keyword, + tags, + date_added, + modified, + isLocal + ) + SELECT + l.guid, + l.type, + l.parentid, + l.pos, + l.title, + validate_url(l.bmkUri) as uri, + l.keyword, + l.tags, + sanitize_timestamp(l.date_added), + sanitize_timestamp(l.local_modified), + 1 + FROM ios.bookmarksLocal l + WHERE NOT l.is_deleted + AND uri IS NOT NULL + ;", roots = ROOTS, ios_bookmark_type = IosBookmarkType::Bookmark as u8, ); static ref CREATE_STAGING_TABLE: String = format!(" - CREATE TEMP TABLE temp.iosBookmarksStaging( - id INTEGER PRIMARY KEY, - guid TEXT NOT NULL UNIQUE, - type TINYINT NOT NULL - CHECK(type == {ios_bookmark_type} OR type == {ios_folder_type} OR type == {ios_separator_type}), - parentid TEXT, - pos INT, - title TEXT, - bmkUri TEXT - CHECK(type != {ios_bookmark_type} OR validate_url(bmkUri) == bmkUri), - keyword TEXT, - tags TEXT, - date_added INTEGER NOT NULL, - modified INTEGER NOT NULL, - isLocal TINYINT NOT NULL - )", + CREATE TEMP TABLE temp.iosBookmarksStaging( + id INTEGER PRIMARY KEY, + guid TEXT NOT NULL UNIQUE, + type TINYINT NOT NULL + CHECK(type == {ios_bookmark_type} OR type == {ios_folder_type} OR type == {ios_separator_type}), + parentid TEXT, + pos INT, + title TEXT, + bmkUri TEXT + CHECK(type != {ios_bookmark_type} OR validate_url(bmkUri) == bmkUri), + keyword TEXT, + tags TEXT, + date_added INTEGER NOT NULL, + modified INTEGER NOT NULL, + isLocal TINYINT NOT NULL + )", ios_bookmark_type = IosBookmarkType::Bookmark as u8, ios_folder_type = IosBookmarkType::Folder as u8, @@ -459,17 +456,17 @@ lazy_static::lazy_static! { static ref FIXUP_MOZ_BOOKMARKS: String = format!( // Is there anything else? "UPDATE main.moz_bookmarks SET - syncStatus = {unknown}, - syncChangeCounter = 1, - lastModified = IFNULL((SELECT stage.modified FROM temp.iosBookmarksStaging stage - WHERE stage.guid = main.moz_bookmarks.guid), - lastModified)", + syncStatus = {unknown}, + syncChangeCounter = 1, + lastModified = IFNULL((SELECT stage.modified FROM temp.iosBookmarksStaging stage + WHERE stage.guid = main.moz_bookmarks.guid), + lastModified)", unknown = SyncStatus::Unknown as u8 ); } mod sql_fns { - use crate::import::common::sql_fns::{sanitize_timestamp, validate_url}; + use crate::import::common::sql_fns::{sanitize_integer_timestamp, validate_url}; use rusqlite::{functions::FunctionFlags, Connection, Result}; pub(super) fn define_functions(c: &Connection) -> Result<()> { @@ -483,7 +480,7 @@ mod sql_fns { "sanitize_timestamp", 1, FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, - sanitize_timestamp, + sanitize_integer_timestamp, )?; Ok(()) } diff --git a/components/places/src/import/ios/history.rs b/components/places/src/import/ios/history.rs new file mode 100644 index 0000000000..303d7c46b9 --- /dev/null +++ b/components/places/src/import/ios/history.rs @@ -0,0 +1,171 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::time::Instant; + +use crate::bookmark_sync::engine::update_frecencies; +use crate::error::Result; +use crate::history_sync::engine::LAST_SYNC_META_KEY; +use crate::import::common::{ + attached_database, define_history_migration_functions, select_count, HistoryMigrationResult, +}; +use crate::storage::put_meta; +use crate::PlacesDb; +use url::Url; + +/// This import is used for iOS users migrating from `browser.db`-based +/// history storage to the new rust-places store. +/// +/// The goal of this import is to persist all local browser.db items into places database +/// +/// +/// ### Basic process +/// +/// - Attach the iOS database. +/// - Slurp records into a temp table "iOSHistoryStaging" from iOS database. +/// - This is mostly done for convenience, to punycode the URLs and some performance benefits over +/// using a view or reading things into Rust +/// - Add any entries to moz_places that are needed (in practice, most are +/// needed, users in practice don't have nearly as many bookmarks as history entries) +/// - Use iosHistoryStaging and the browser.db to migrate visits to the places visits table. +/// - Update frecency for new items. +/// - Cleanup (detach iOS database, etc). +pub fn import( + conn: &PlacesDb, + path: impl AsRef, + last_sync_timestamp: i64, +) -> Result { + let url = crate::util::ensure_url_path(path)?; + do_import(conn, url, last_sync_timestamp) +} + +fn do_import( + conn: &PlacesDb, + ios_db_file_url: Url, + last_sync_timestamp: i64, +) -> Result { + let scope = conn.begin_interrupt_scope()?; + define_history_migration_functions(conn)?; + // TODO: for some reason opening the db as read-only in **iOS** causes + // the migration to fail with an "attempting to write to a read-only database" + // when the migration is **not** writing to the BrowserDB database. + // this only happens in the simulator with artifacts built for iOS and not + // in unit tests. + + // ios_db_file_url.query_pairs_mut().append_pair("mode", "ro"); + let import_start = Instant::now(); + log::debug!("Attaching database {}", ios_db_file_url); + let auto_detach = attached_database(conn, &ios_db_file_url, "ios")?; + let tx = conn.begin_transaction()?; + let num_total = select_count(conn, &COUNT_IOS_HISTORY_VISITS)?; + log::debug!("The number of visits is: {:?}", num_total); + log::debug!("Creating and populating staging table"); + conn.execute_batch(&CREATE_STAGING_TABLE)?; + conn.execute_batch(&FILL_STAGING)?; + scope.err_if_interrupted()?; + + log::debug!("Populating missing entries in moz_places"); + conn.execute_batch(&FILL_MOZ_PLACES)?; + scope.err_if_interrupted()?; + + log::debug!("Inserting the history visits"); + conn.execute_batch(&INSERT_HISTORY_VISITS)?; + scope.err_if_interrupted()?; + + // Once the migration is done, we also migrate the sync timestamp if we have one + // this prevents us from having to do a **full** sync + put_meta(conn, LAST_SYNC_META_KEY, &last_sync_timestamp)?; + + tx.commit()?; + // Note: update_frecencies manages its own transaction, which is fine, + // since nothing that bad will happen if it is aborted. + log::debug!("Updating frecencies"); + update_frecencies(conn, &scope)?; + + log::info!("Successfully imported history visits!"); + + log::debug!("Counting Places history visits"); + let num_succeeded = select_count(conn, &COUNT_PLACES_HISTORY_VISITS)?; + let num_failed = num_total.saturating_sub(num_succeeded); + + auto_detach.execute_now()?; + + let metrics = HistoryMigrationResult { + num_total, + num_succeeded, + num_failed, + total_duration: import_start.elapsed().as_millis() as u64, + }; + + Ok(metrics) +} + +lazy_static::lazy_static! { + // Count IOS history visits + static ref COUNT_IOS_HISTORY_VISITS: &'static str = + "SELECT COUNT(*) FROM ios.visits v + LEFT JOIN ios.history h on v.siteID = h.id + WHERE h.is_deleted = 0" + ; + + // We use a staging table purely so that we can normalize URLs (and + // specifically, punycode them) + static ref CREATE_STAGING_TABLE: &'static str = " + CREATE TEMP TABLE temp.iOSHistoryStaging( + id INTEGER PRIMARY KEY, + url TEXT, + url_hash INTEGER NOT NULL, + title TEXT, + is_deleted TINYINT NOT NULL + ) WITHOUT ROWID;"; + + static ref FILL_STAGING: &'static str = " + INSERT OR IGNORE INTO temp.iOSHistoryStaging(id, url, url_hash, title, is_deleted) + SELECT + h.id, + validate_url(h.url), + hash(validate_url(h.url)), + sanitize_utf8(h.title), + h.is_deleted + FROM ios.history h + WHERE url IS NOT NULL" + ; + + // Insert any missing entries into moz_places that we'll need for this. + static ref FILL_MOZ_PLACES: &'static str = + "INSERT OR IGNORE INTO main.moz_places(guid, url, url_hash, title, frecency, sync_change_counter) + SELECT + IFNULL( + (SELECT p.guid FROM main.moz_places p WHERE p.url_hash = t.url_hash AND p.url = t.url), + generate_guid() + ), + t.url, + t.url_hash, + t.title, + -1, + 1 + FROM temp.iOSHistoryStaging t + WHERE t.is_deleted = 0" + ; + + // Insert history visits + static ref INSERT_HISTORY_VISITS: &'static str = + "INSERT OR IGNORE INTO main.moz_historyvisits(from_visit, place_id, visit_date, visit_type, is_local) + SELECT + NULL, -- iOS does not store enough information to rebuild redirect chains. + (SELECT p.id FROM main.moz_places p WHERE p.url_hash = t.url_hash AND p.url = t.url), + sanitize_float_timestamp(v.date), + v.type, -- iOS stores visit types that map 1:1 to ours. + v.is_local + FROM ios.visits v + LEFT JOIN temp.iOSHistoryStaging t on v.siteID = t.id + WHERE t.is_deleted = 0" + ; + + + // Count places history visits + static ref COUNT_PLACES_HISTORY_VISITS: &'static str = + "SELECT COUNT(*) FROM main.moz_historyvisits" + ; +} diff --git a/components/places/src/import/mod.rs b/components/places/src/import/mod.rs index cba1becc8a..084b15a5ce 100644 --- a/components/places/src/import/mod.rs +++ b/components/places/src/import/mod.rs @@ -7,5 +7,6 @@ pub mod fennec; pub use fennec::import_bookmarks as import_fennec_bookmarks; pub use fennec::import_history as import_fennec_history; pub use fennec::import_pinned_sites as import_fennec_pinned_sites; -pub mod ios_bookmarks; -pub use ios_bookmarks::import_ios_bookmarks; +pub mod ios; +pub use ios::import_bookmarks as import_ios_bookmarks; +pub use ios::import_history as import_ios_history; diff --git a/components/places/src/places.udl b/components/places/src/places.udl index 33218fc3b8..0b6aa24a0b 100644 --- a/components/places/src/places.udl +++ b/components/places/src/places.udl @@ -187,6 +187,9 @@ interface PlacesConnection { [Throws=PlacesError] Guid bookmarks_insert(InsertableBookmarkItem bookmark); + + [Throws=PlacesError] + HistoryMigrationResult places_history_import_from_ios(string db_path, i64 last_sync_timestamp); }; /** @@ -326,6 +329,13 @@ dictionary TopFrecentSiteInfo { string? title; }; +dictionary HistoryMigrationResult { + u32 num_total; + u32 num_succeeded; + u32 num_failed; + u64 total_duration; +}; + [Error] enum PlacesError { diff --git a/components/places/src/storage/history.rs b/components/places/src/storage/history.rs index a8e0d021e5..5a3dc0bc7d 100644 --- a/components/places/src/storage/history.rs +++ b/components/places/src/storage/history.rs @@ -899,7 +899,6 @@ pub mod history_sync { let visits = visits .iter() - .cloned() .filter(|v| Timestamp::from(v.date) > visit_ignored_mark) .collect::>(); diff --git a/components/support/types/src/lib.rs b/components/support/types/src/lib.rs index ef659c3878..fb919a0394 100644 --- a/components/support/types/src/lib.rs +++ b/components/support/types/src/lib.rs @@ -33,6 +33,11 @@ impl Timestamp { SystemTime::from(self).checked_sub(d).map(Timestamp::from) } + #[inline] + pub fn checked_add(self, d: Duration) -> Option { + SystemTime::from(self).checked_add(d).map(Timestamp::from) + } + pub fn as_millis(self) -> u64 { self.0 } diff --git a/examples/places-utils/src/places-utils.rs b/examples/places-utils/src/places-utils.rs index 7506c1d640..2b3e54e27d 100644 --- a/examples/places-utils/src/places-utils.rs +++ b/examples/places-utils/src/places-utils.rs @@ -130,13 +130,19 @@ fn run_desktop_import(db: &PlacesDb, filename: String) -> Result<()> { do_import(db, root) } -fn run_ios_import(api: &PlacesApi, filename: String) -> Result<()> { - println!("ios import from {}", filename); +fn run_ios_import_bookmarks(api: &PlacesApi, filename: String) -> Result<()> { + println!("ios import bookmarks from {}", filename); places::import::import_ios_bookmarks(api, filename)?; println!("Import finished!"); Ok(()) } +fn run_ios_import_history(conn: &PlacesDb, filename: String) -> Result<()> { + let res = places::import::import_ios_history(conn, filename, 0)?; + println!("Import finished!, results: {:?}", res); + Ok(()) +} + fn run_native_import(db: &PlacesDb, filename: String) -> Result<()> { println!("import from {}", filename); @@ -345,6 +351,14 @@ enum Command { input_file: String, }, + #[structopt(name = "import-ios-history")] + /// Import history from an iOS browser.db + ImportIosHistory { + #[structopt(name = "input-file", long, short = "i")] + /// The name of the file to read + input_file: String, + }, + #[structopt(name = "import-desktop-bookmarks")] /// Import bookmarks from JSON file exported by desktop Firefox ImportDesktopBookmarks { @@ -392,7 +406,8 @@ fn main() -> Result<()> { ), Command::ExportBookmarks { output_file } => run_native_export(&db, output_file), Command::ImportBookmarks { input_file } => run_native_import(&db, input_file), - Command::ImportIosBookmarks { input_file } => run_ios_import(&api, input_file), + Command::ImportIosBookmarks { input_file } => run_ios_import_bookmarks(&api, input_file), Command::ImportDesktopBookmarks { input_file } => run_desktop_import(&db, input_file), + Command::ImportIosHistory { input_file } => run_ios_import_history(&db, input_file), } } diff --git a/testing/separated/places-tests/src/fennec_history.rs b/testing/separated/places-tests/src/fennec_history.rs index c7713f8994..e60f4497df 100644 --- a/testing/separated/places-tests/src/fennec_history.rs +++ b/testing/separated/places-tests/src/fennec_history.rs @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use places::import::fennec::history::HistoryMigrationResult; +use places::import::common::HistoryMigrationResult; use places::{api::places_api::PlacesApi, types::VisitTransition, ErrorKind, Result}; use rusqlite::Connection; use std::path::Path; diff --git a/testing/separated/places-tests/src/ios_bookmarks.rs b/testing/separated/places-tests/src/ios_bookmarks.rs index 7524bd01e9..4d15a9828b 100644 --- a/testing/separated/places-tests/src/ios_bookmarks.rs +++ b/testing/separated/places-tests/src/ios_bookmarks.rs @@ -5,7 +5,7 @@ use dogear::Guid; use places::{ api::places_api::{ConnectionType, PlacesApi}, - import::ios_bookmarks::IosBookmarkType, + import::ios::bookmarks::IosBookmarkType, storage::bookmarks::{self, fetch::Item}, Result, }; diff --git a/testing/separated/places-tests/src/ios_history.rs b/testing/separated/places-tests/src/ios_history.rs new file mode 100644 index 0000000000..50a8251738 --- /dev/null +++ b/testing/separated/places-tests/src/ios_history.rs @@ -0,0 +1,201 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +use places::{ + api::places_api::{ConnectionType, PlacesApi}, + storage::history, + Result, VisitTransitionSet, +}; +use rusqlite::Connection; +use std::path::Path; +use std::{str::FromStr, time::Duration}; +use tempfile::tempdir; +use types::Timestamp; +use url::Url; + +fn empty_ios_db(path: &Path) -> Result { + let conn = Connection::open(path)?; + conn.execute_batch(include_str!("./ios_schema.sql"))?; + Ok(conn) +} + +#[derive(Default, Debug)] +struct IOSHistory { + id: u64, + guid: String, + url: Option, + title: String, + is_deleted: bool, + should_upload: bool, +} + +#[derive(Debug, Default, Clone)] + +struct IOSVisit { + id: u64, + site_id: u64, + date: i64, + type_: u64, + is_local: bool, +} + +#[derive(Debug, Default)] +struct HistoryTable(Vec); + +#[derive(Debug, Default)] +struct VisitTable(Vec); + +impl HistoryTable { + fn populate(&self, conn: &Connection) -> Result<()> { + let mut stmt = conn.prepare( + "INSERT INTO history( + id, + guid, + url, + title, + is_deleted, + should_upload + ) VALUES ( + :id, + :guid, + :url, + :title, + :is_deleted, + :should_upload + )", + )?; + for history_item in &self.0 { + stmt.execute(rusqlite::named_params! { + ":guid": history_item.id, + ":guid": history_item.guid, + ":url": history_item.url, + ":title": history_item.title, + ":is_deleted": history_item.is_deleted, + ":should_upload": history_item.should_upload, + })?; + } + Ok(()) + } +} + +impl VisitTable { + fn populate(&self, conn: &Connection) -> Result<()> { + let mut stmt = conn.prepare( + "INSERT INTO visits ( + id, + siteID, + date, + type, + is_local + ) VALUES ( + :id, + :siteID, + :date, + :type, + :is_local + )", + )?; + for visit in &self.0 { + stmt.execute(rusqlite::named_params! { + ":id": visit.id, + ":siteID": visit.site_id, + ":date": visit.date, + ":type": visit.type_, + ":is_local": visit.is_local, + })?; + } + Ok(()) + } +} + +#[test] +fn test_import_empty() -> Result<()> { + let tmpdir = tempdir().unwrap(); + let history = HistoryTable::default(); + let visits = VisitTable::default(); + let ios_path = tmpdir.path().join("browser.db"); + let ios_db = empty_ios_db(&ios_path)?; + + history.populate(&ios_db)?; + visits.populate(&ios_db)?; + let places_api = PlacesApi::new(tmpdir.path().join("places.sqlite"))?; + let conn = places_api.open_connection(ConnectionType::ReadWrite)?; + places::import::import_ios_history(&conn, ios_path, 0)?; + + Ok(()) +} + +#[test] +fn test_import_basic() -> Result<()> { + let tmpdir = tempdir().unwrap(); + let ios_path = tmpdir.path().join("browser.db"); + let ios_db = empty_ios_db(&ios_path)?; + let history_entry = IOSHistory { + id: 1, + guid: "EXAMPLE GUID".to_string(), + url: Some("https://example.com".to_string()), + title: "Example(dot)com".to_string(), + is_deleted: false, + should_upload: false, + }; + + // We subtract a bit because our sanitization logic is smart and rejects + // visits that have a future timestamp, + let before_first_visit_ts = Timestamp::now() + .checked_sub(Duration::from_secs(10000)) + .unwrap(); + let first_visit_ts = before_first_visit_ts + .checked_add(Duration::from_secs(100)) + .unwrap(); + let visit = IOSVisit { + id: 1, + site_id: 1, + // Dates in iOS are represented as μs + // we make sure that they get converted properly. + // when we compare them later we will compare against + // milliseconds + date: first_visit_ts.as_millis_i64() * 1000, + type_: 1, + ..Default::default() + }; + + let second_visit_ts = first_visit_ts + .checked_add(Duration::from_secs(100)) + .unwrap(); + + let other_visit = IOSVisit { + id: 2, + site_id: 1, + // Dates in iOS are represented as μs + date: second_visit_ts.as_millis_i64() * 1000, + type_: 1, + ..Default::default() + }; + + let history_table = HistoryTable(vec![history_entry]); + let visit_table = VisitTable(vec![visit, other_visit]); + history_table.populate(&ios_db)?; + visit_table.populate(&ios_db)?; + + let places_api = PlacesApi::new(tmpdir.path().join("places.sqlite"))?; + let conn = places_api.open_connection(ConnectionType::ReadWrite)?; + places::import::import_ios_history(&conn, ios_path, 0)?; + + let places_db = places_api.open_connection(ConnectionType::ReadOnly)?; + let visit_count = history::get_visit_count(&places_db, VisitTransitionSet::empty())?; + assert_eq!(visit_count, 2); + let url = Url::from_str("https://example.com").unwrap(); + let visited = history::get_visited(&places_db, vec![url]).unwrap(); + assert!(visited[0]); + let visit_infos = history::get_visit_infos( + &places_db, + before_first_visit_ts, + Timestamp::now(), + VisitTransitionSet::empty(), + )?; + assert_eq!(visit_infos[0].title, Some("Example(dot)com".to_owned())); + assert_eq!(visit_infos[1].title, Some("Example(dot)com".to_owned())); + assert_eq!(visit_infos[0].timestamp, first_visit_ts); + assert_eq!(visit_infos[1].timestamp, second_visit_ts); + Ok(()) +} diff --git a/testing/separated/places-tests/src/ios_schema.sql b/testing/separated/places-tests/src/ios_schema.sql index 849f7cce83..98b0b41bda 100644 --- a/testing/separated/places-tests/src/ios_schema.sql +++ b/testing/separated/places-tests/src/ios_schema.sql @@ -100,3 +100,35 @@ CREATE INDEX idx_bookmarksMirrorStructure_parent_idx ON bookmarksMirrorStructure CREATE INDEX idx_bookmarksBuffer_keyword ON bookmarksBuffer (keyword); CREATE INDEX idx_bookmarksLocal_keyword ON bookmarksLocal (keyword); CREATE INDEX idx_bookmarksMirror_keyword ON bookmarksMirror (keyword); + + +-- History entries +CREATE TABLE IF NOT EXISTS history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + -- Not null, but the value might be replaced by the server's. + guid TEXT NOT NULL UNIQUE, + -- May only be null for deleted records. + url TEXT UNIQUE, + title TEXT NOT NULL, + -- Can be null. Integer milliseconds. + server_modified INTEGER, + -- Can be null. Client clock. In extremis only. + local_modified INTEGER, + -- Boolean. Locally deleted. + is_deleted TINYINT NOT NULL, + -- Boolean. Set when changed or visits added. + should_upload TINYINT NOT NULL, + -- domain_id INTEGER REFERENCES domains(id) ON DELETE CASCADE, + CONSTRAINT urlOrDeleted CHECK (url IS NOT NULL OR is_deleted = 1) +); + +CREATE TABLE IF NOT EXISTS visits ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + siteID INTEGER NOT NULL REFERENCES history(id) ON DELETE CASCADE, + -- Microseconds since epoch. + date REAL NOT NULL, + type INTEGER NOT NULL, + -- Some visits are local. Some are remote ('mirrored'). This boolean flag is the split. + is_local TINYINT NOT NULL, + UNIQUE (siteID, date, type) +); diff --git a/testing/separated/places-tests/src/tests.rs b/testing/separated/places-tests/src/tests.rs index 756110d9bf..fa8ff472f4 100644 --- a/testing/separated/places-tests/src/tests.rs +++ b/testing/separated/places-tests/src/tests.rs @@ -6,3 +6,4 @@ mod check_coop_tx; mod fennec_bookmarks; mod fennec_history; mod ios_bookmarks; +mod ios_history;