diff --git a/package.json b/package.json index 075311c..29cce56 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "twitter-web-exporter", "description": "Export tweets, bookmarks, lists and much more from Twitter(X) web app.", - "version": "1.1.0", + "version": "1.2.0-alpha.1", "author": "prin ", "license": "MIT", "homepage": "https://github.com/prinsss/twitter-web-exporter", @@ -23,6 +23,7 @@ "@tabler/icons-preact": "2.44.0", "@tanstack/table-core": "8.11.2", "dayjs": "1.11.10", + "dexie": "4.0.4", "file-saver": "2.0.5", "i18next": "23.11.1", "preact": "10.19.3" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f9527c1..f46c58d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ dependencies: dayjs: specifier: 1.11.10 version: 1.11.10 + dexie: + specifier: 4.0.4 + version: 4.0.4 file-saver: specifier: 2.0.5 version: 2.0.5 @@ -1717,6 +1720,10 @@ packages: engines: {node: '>=12'} dev: true + /dexie@4.0.4: + resolution: {integrity: sha512-wFzwWSUdi+MC3jiFeQcCp9nInR7EaX8edzYY+4wmiITkQAiSnHpe4Wo2o5Ce5tJZe2nqt7mLW91MsW4GYx3ziQ==} + dev: false + /didyoumean@1.2.2: resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==} dev: true diff --git a/src/core/storage/database.ts b/src/core/storage/database.ts index 84c1d2b..1b029b4 100644 --- a/src/core/storage/database.ts +++ b/src/core/storage/database.ts @@ -1,3 +1,250 @@ +import Dexie, { KeyPaths } from 'dexie'; + +import packageJson from '@/../package.json'; +import { Capture, Tweet, User } from '@/types'; +import { extractTweetMedia } from '@/utils/api'; +import { parseTwitterDateTime } from '@/utils/common'; +import logger from '@/utils/logger'; +import { ExtensionType } from '../extensions'; + +const DB_NAME = packageJson.name; +const DB_VERSION = 1; + export class DatabaseManager { - // TODO + private db: Dexie; + + constructor() { + this.db = new Dexie(DB_NAME); + this.init(); + } + + /* + |-------------------------------------------------------------------------- + | Type-Safe Table Accessors + |-------------------------------------------------------------------------- + */ + + private tweets() { + return this.db.table('tweets'); + } + + private users() { + return this.db.table('users'); + } + + private captures() { + return this.db.table('captures'); + } + + /* + |-------------------------------------------------------------------------- + | Read Methods for Extensions + |-------------------------------------------------------------------------- + */ + + async extGetCaptures(extName: string) { + return this.captures().where('extension').equals(extName).toArray().catch(this.logError); + } + + async extGetCaptureCount(extName: string) { + return this.captures().where('extension').equals(extName).count().catch(this.logError); + } + + async extGetCapturedTweets(extName: string) { + const captures = await this.extGetCaptures(extName); + if (!captures) { + return []; + } + const tweetIds = captures.map((capture) => capture.data_key); + return this.tweets().where('rest_id').anyOf(tweetIds).toArray().catch(this.logError); + } + + async extGetCapturedUsers(extName: string) { + const captures = await this.extGetCaptures(extName); + if (!captures) { + return []; + } + const userIds = captures.map((capture) => capture.data_key); + return this.users().where('rest_id').anyOf(userIds).toArray().catch(this.logError); + } + + /* + |-------------------------------------------------------------------------- + | Write Methods for Extensions + |-------------------------------------------------------------------------- + */ + + async extAddTweets(extName: string, tweets: Tweet[]) { + await this.upsertTweets(tweets); + await this.upsertCaptures( + tweets.map((tweet) => ({ + id: `${extName}-${tweet.rest_id}`, + extension: extName, + type: ExtensionType.TWEET, + data_key: tweet.rest_id, + created_at: Date.now(), + })), + ); + } + + async extAddUsers(extName: string, users: User[]) { + await this.upsertUsers(users); + await this.upsertCaptures( + users.map((user) => ({ + id: `${extName}-${user.rest_id}`, + extension: extName, + type: ExtensionType.USER, + data_key: user.rest_id, + created_at: Date.now(), + })), + ); + } + + /* + |-------------------------------------------------------------------------- + | Delete Methods for Extensions + |-------------------------------------------------------------------------- + */ + + async extClearCaptures(extName: string) { + const captures = await this.extGetCaptures(extName); + if (!captures) { + return; + } + return this.captures().bulkDelete(captures.map((capture) => capture.id)); + } + + /* + |-------------------------------------------------------------------------- + | Common Methods + |-------------------------------------------------------------------------- + */ + + async upsertTweets(tweets: Tweet[]) { + return this.db + .transaction('rw', this.tweets(), () => { + const data: Tweet[] = tweets.map((tweet) => ({ + ...tweet, + twe_private_fields: { + created_at: +parseTwitterDateTime(tweet.legacy.created_at), + updated_at: Date.now(), + media_count: extractTweetMedia(tweet).length, + }, + })); + + return this.tweets().bulkPut(data); + }) + .catch(this.logError); + } + + async upsertUsers(users: User[]) { + return this.db + .transaction('rw', this.users(), () => { + const data: User[] = users.map((user) => ({ + ...user, + twe_private_fields: { + created_at: +parseTwitterDateTime(user.legacy.created_at), + updated_at: Date.now(), + }, + })); + + return this.users().bulkPut(data); + }) + .catch(this.logError); + } + + async upsertCaptures(captures: Capture[]) { + return this.db + .transaction('rw', this.captures(), () => { + return this.captures().bulkPut(captures).catch(this.logError); + }) + .catch(this.logError); + } + + async deleteAllTweets() { + return this.tweets().clear().catch(this.logError); + } + + async deleteAllUsers() { + return this.users().clear().catch(this.logError); + } + + async deleteAllCaptures() { + return this.captures().clear().catch(this.logError); + } + + /* + |-------------------------------------------------------------------------- + | Migrations + |-------------------------------------------------------------------------- + */ + + async init() { + // Indexes for the "tweets" table. + const tweetIndexPaths: KeyPaths[] = [ + 'rest_id', + 'twe_private_fields.created_at', + 'twe_private_fields.updated_at', + 'twe_private_fields.media_count', + 'core.user_results.result.legacy.screen_name', + 'legacy.favorite_count', + 'legacy.retweet_count', + 'legacy.bookmark_count', + 'legacy.quote_count', + 'legacy.reply_count', + 'views.count', + 'legacy.favorited', + 'legacy.retweeted', + 'legacy.bookmarked', + ]; + + // Indexes for the "users" table. + const userIndexPaths: KeyPaths[] = [ + 'rest_id', + 'twe_private_fields.created_at', + 'twe_private_fields.updated_at', + 'legacy.screen_name', + 'legacy.followers_count', + 'legacy.statuses_count', + 'legacy.favourites_count', + 'legacy.listed_count', + 'legacy.verified_type', + 'is_blue_verified', + 'legacy.following', + 'legacy.followed_by', + ]; + + // Indexes for the "captures" table. + const captureIndexPaths: KeyPaths[] = ['id', 'extension', 'type', 'created_at']; + + // Take care of database schemas and versioning. + // See: https://dexie.org/docs/Tutorial/Design#database-versioning + try { + this.db + .version(DB_VERSION) + .stores({ + tweets: tweetIndexPaths.join(','), + users: userIndexPaths.join(','), + captures: captureIndexPaths.join(','), + }) + .upgrade(async () => { + logger.info('Database upgraded'); + }); + + await this.db.open(); + logger.info('Database connected'); + } catch (error) { + this.logError(error); + } + } + + /* + |-------------------------------------------------------------------------- + | Loggers + |-------------------------------------------------------------------------- + */ + + logError(error: unknown) { + logger.error(`Database Error: ${(error as Error).message}`, error); + } } diff --git a/src/types/index.ts b/src/types/index.ts index e584922..32c2439 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,5 @@ -import { TimelineTweet } from './tweet'; import { TimelineTwitterList } from './list'; +import { TimelineTweet } from './tweet'; import { TimelineUser } from './user'; export * from './list'; @@ -144,3 +144,19 @@ export interface TimelineTimelineModule { }; }; } + +/** + * Represents a piece of data captured by an extension. + */ +export interface Capture { + /** Unique identifier for the capture. */ + id: string; + /** Name of extension that captured the data. */ + extension: string; + /** Type of data captured. Possible values: `tweet`, `user`. */ + type: string; + /** The index of the captured item. Use this to query actual data from the database. */ + data_key: string; + /** Timestamp when the data was captured. */ + created_at: number; +} diff --git a/src/types/tweet.ts b/src/types/tweet.ts index b280043..4a1c990 100644 --- a/src/types/tweet.ts +++ b/src/types/tweet.ts @@ -131,6 +131,18 @@ export interface Tweet { result: TweetUnion; }; }; + /** + * Some extra properties added by the script when inserting to local database. + * These are not present in the original tweet object and are used for internal purposes only. + */ + twe_private_fields: { + /** The UNIX timestamp representation of `legacy.created_at` in milliseconds. */ + created_at: number; + /** The UNIX timestamp in ms when inserted or updated to local database. */ + updated_at: number; + /** The number of media items in the tweet. */ + media_count: number; + }; } export interface RichTextTag { diff --git a/src/types/user.ts b/src/types/user.ts index 4870b04..325297e 100644 --- a/src/types/user.ts +++ b/src/types/user.ts @@ -61,6 +61,16 @@ export interface User { icon_name: string; }[]; }; + /** + * Some extra properties added by the script when inserting to local database. + * These are not present in the original tweet object and are used for internal purposes only. + */ + twe_private_fields: { + /** The UNIX timestamp representation of `legacy.created_at` in milliseconds. */ + created_at: number; + /** The UNIX timestamp in ms when inserted or updated to local database. */ + updated_at: number; + }; } export interface UserEntities { diff --git a/vite.config.ts b/vite.config.ts index d3e44fa..6ff4f90 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -73,6 +73,7 @@ export default defineConfig({ 'https://github.com/prinsss/twitter-web-exporter/releases/latest/download/twitter-web-exporter.user.js', require: [ 'https://cdn.jsdelivr.net/npm/dayjs@1.11.10/dayjs.min.js', + 'https://cdn.jsdelivr.net/npm/dexie@4.0.4/dist/dexie.min.js', 'https://cdn.jsdelivr.net/npm/i18next@23.11.1/i18next.min.js', 'https://cdn.jsdelivr.net/npm/preact@10.19.3/dist/preact.min.js', 'https://cdn.jsdelivr.net/npm/preact@10.19.3/hooks/dist/hooks.umd.js', @@ -87,6 +88,7 @@ export default defineConfig({ build: { externalGlobals: { dayjs: 'dayjs', + dexie: 'Dexie', i18next: 'i18next', preact: 'preact', 'preact/hooks': 'preactHooks',