Skip to content

Commit

Permalink
feat: add local database
Browse files Browse the repository at this point in the history
  • Loading branch information
prinsss committed Jun 16, 2024
1 parent 07127f0 commit d357c4d
Show file tree
Hide file tree
Showing 7 changed files with 298 additions and 3 deletions.
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "twitter-web-exporter",
"description": "Export tweets, bookmarks, lists and much more from Twitter(X) web app.",
"version": "1.1.0",
"version": "1.2.0-alpha.1",
"author": "prin <hi@prin.studio>",
"license": "MIT",
"homepage": "https://github.com/prinsss/twitter-web-exporter",
Expand All @@ -23,6 +23,7 @@
"@tabler/icons-preact": "2.44.0",
"@tanstack/table-core": "8.11.2",
"dayjs": "1.11.10",
"dexie": "4.0.4",
"file-saver": "2.0.5",
"i18next": "23.11.1",
"preact": "10.19.3"
Expand Down
7 changes: 7 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

249 changes: 248 additions & 1 deletion src/core/storage/database.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,250 @@
import Dexie, { KeyPaths } from 'dexie';

import packageJson from '@/../package.json';
import { Capture, Tweet, User } from '@/types';
import { extractTweetMedia } from '@/utils/api';
import { parseTwitterDateTime } from '@/utils/common';
import logger from '@/utils/logger';
import { ExtensionType } from '../extensions';

const DB_NAME = packageJson.name;
const DB_VERSION = 1;

export class DatabaseManager {
// TODO
private db: Dexie;

constructor() {
this.db = new Dexie(DB_NAME);
this.init();
}

/*
|--------------------------------------------------------------------------
| Type-Safe Table Accessors
|--------------------------------------------------------------------------
*/

private tweets() {
return this.db.table<Tweet>('tweets');
}

private users() {
return this.db.table<User>('users');
}

private captures() {
return this.db.table<Capture>('captures');
}

/*
|--------------------------------------------------------------------------
| Read Methods for Extensions
|--------------------------------------------------------------------------
*/

async extGetCaptures(extName: string) {
return this.captures().where('extension').equals(extName).toArray().catch(this.logError);
}

async extGetCaptureCount(extName: string) {
return this.captures().where('extension').equals(extName).count().catch(this.logError);
}

async extGetCapturedTweets(extName: string) {
const captures = await this.extGetCaptures(extName);
if (!captures) {
return [];
}
const tweetIds = captures.map((capture) => capture.data_key);
return this.tweets().where('rest_id').anyOf(tweetIds).toArray().catch(this.logError);
}

async extGetCapturedUsers(extName: string) {
const captures = await this.extGetCaptures(extName);
if (!captures) {
return [];
}
const userIds = captures.map((capture) => capture.data_key);
return this.users().where('rest_id').anyOf(userIds).toArray().catch(this.logError);
}

/*
|--------------------------------------------------------------------------
| Write Methods for Extensions
|--------------------------------------------------------------------------
*/

async extAddTweets(extName: string, tweets: Tweet[]) {
await this.upsertTweets(tweets);
await this.upsertCaptures(
tweets.map((tweet) => ({
id: `${extName}-${tweet.rest_id}`,
extension: extName,
type: ExtensionType.TWEET,
data_key: tweet.rest_id,
created_at: Date.now(),
})),
);
}

async extAddUsers(extName: string, users: User[]) {
await this.upsertUsers(users);
await this.upsertCaptures(
users.map((user) => ({
id: `${extName}-${user.rest_id}`,
extension: extName,
type: ExtensionType.USER,
data_key: user.rest_id,
created_at: Date.now(),
})),
);
}

/*
|--------------------------------------------------------------------------
| Delete Methods for Extensions
|--------------------------------------------------------------------------
*/

async extClearCaptures(extName: string) {
const captures = await this.extGetCaptures(extName);
if (!captures) {
return;
}
return this.captures().bulkDelete(captures.map((capture) => capture.id));
}

/*
|--------------------------------------------------------------------------
| Common Methods
|--------------------------------------------------------------------------
*/

async upsertTweets(tweets: Tweet[]) {
return this.db
.transaction('rw', this.tweets(), () => {
const data: Tweet[] = tweets.map((tweet) => ({
...tweet,
twe_private_fields: {
created_at: +parseTwitterDateTime(tweet.legacy.created_at),
updated_at: Date.now(),
media_count: extractTweetMedia(tweet).length,
},
}));

return this.tweets().bulkPut(data);
})
.catch(this.logError);
}

async upsertUsers(users: User[]) {
return this.db
.transaction('rw', this.users(), () => {
const data: User[] = users.map((user) => ({
...user,
twe_private_fields: {
created_at: +parseTwitterDateTime(user.legacy.created_at),
updated_at: Date.now(),
},
}));

return this.users().bulkPut(data);
})
.catch(this.logError);
}

async upsertCaptures(captures: Capture[]) {
return this.db
.transaction('rw', this.captures(), () => {
return this.captures().bulkPut(captures).catch(this.logError);
})
.catch(this.logError);
}

async deleteAllTweets() {
return this.tweets().clear().catch(this.logError);
}

async deleteAllUsers() {
return this.users().clear().catch(this.logError);
}

async deleteAllCaptures() {
return this.captures().clear().catch(this.logError);
}

/*
|--------------------------------------------------------------------------
| Migrations
|--------------------------------------------------------------------------
*/

async init() {
// Indexes for the "tweets" table.
const tweetIndexPaths: KeyPaths<Tweet>[] = [
'rest_id',
'twe_private_fields.created_at',
'twe_private_fields.updated_at',
'twe_private_fields.media_count',
'core.user_results.result.legacy.screen_name',
'legacy.favorite_count',
'legacy.retweet_count',
'legacy.bookmark_count',
'legacy.quote_count',
'legacy.reply_count',
'views.count',
'legacy.favorited',
'legacy.retweeted',
'legacy.bookmarked',
];

// Indexes for the "users" table.
const userIndexPaths: KeyPaths<User>[] = [
'rest_id',
'twe_private_fields.created_at',
'twe_private_fields.updated_at',
'legacy.screen_name',
'legacy.followers_count',
'legacy.statuses_count',
'legacy.favourites_count',
'legacy.listed_count',
'legacy.verified_type',
'is_blue_verified',
'legacy.following',
'legacy.followed_by',
];

// Indexes for the "captures" table.
const captureIndexPaths: KeyPaths<Capture>[] = ['id', 'extension', 'type', 'created_at'];

// Take care of database schemas and versioning.
// See: https://dexie.org/docs/Tutorial/Design#database-versioning
try {
this.db
.version(DB_VERSION)
.stores({
tweets: tweetIndexPaths.join(','),
users: userIndexPaths.join(','),
captures: captureIndexPaths.join(','),
})
.upgrade(async () => {
logger.info('Database upgraded');
});

await this.db.open();
logger.info('Database connected');
} catch (error) {
this.logError(error);
}
}

/*
|--------------------------------------------------------------------------
| Loggers
|--------------------------------------------------------------------------
*/

logError(error: unknown) {
logger.error(`Database Error: ${(error as Error).message}`, error);
}
}
18 changes: 17 additions & 1 deletion src/types/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { TimelineTweet } from './tweet';
import { TimelineTwitterList } from './list';
import { TimelineTweet } from './tweet';
import { TimelineUser } from './user';

export * from './list';
Expand Down Expand Up @@ -144,3 +144,19 @@ export interface TimelineTimelineModule<T = ItemContentUnion> {
};
};
}

/**
* Represents a piece of data captured by an extension.
*/
export interface Capture {
/** Unique identifier for the capture. */
id: string;
/** Name of extension that captured the data. */
extension: string;
/** Type of data captured. Possible values: `tweet`, `user`. */
type: string;
/** The index of the captured item. Use this to query actual data from the database. */
data_key: string;
/** Timestamp when the data was captured. */
created_at: number;
}
12 changes: 12 additions & 0 deletions src/types/tweet.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,18 @@ export interface Tweet {
result: TweetUnion;
};
};
/**
* Some extra properties added by the script when inserting to local database.
* These are not present in the original tweet object and are used for internal purposes only.
*/
twe_private_fields: {
/** The UNIX timestamp representation of `legacy.created_at` in milliseconds. */
created_at: number;
/** The UNIX timestamp in ms when inserted or updated to local database. */
updated_at: number;
/** The number of media items in the tweet. */
media_count: number;
};
}

export interface RichTextTag {
Expand Down
10 changes: 10 additions & 0 deletions src/types/user.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,16 @@ export interface User {
icon_name: string;
}[];
};
/**
* Some extra properties added by the script when inserting to local database.
* These are not present in the original tweet object and are used for internal purposes only.
*/
twe_private_fields: {
/** The UNIX timestamp representation of `legacy.created_at` in milliseconds. */
created_at: number;
/** The UNIX timestamp in ms when inserted or updated to local database. */
updated_at: number;
};
}

export interface UserEntities {
Expand Down
2 changes: 2 additions & 0 deletions vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ export default defineConfig({
'https://github.com/prinsss/twitter-web-exporter/releases/latest/download/twitter-web-exporter.user.js',
require: [
'https://cdn.jsdelivr.net/npm/dayjs@1.11.10/dayjs.min.js',
'https://cdn.jsdelivr.net/npm/dexie@4.0.4/dist/dexie.min.js',
'https://cdn.jsdelivr.net/npm/i18next@23.11.1/i18next.min.js',
'https://cdn.jsdelivr.net/npm/preact@10.19.3/dist/preact.min.js',
'https://cdn.jsdelivr.net/npm/preact@10.19.3/hooks/dist/hooks.umd.js',
Expand All @@ -87,6 +88,7 @@ export default defineConfig({
build: {
externalGlobals: {
dayjs: 'dayjs',
dexie: 'Dexie',
i18next: 'i18next',
preact: 'preact',
'preact/hooks': 'preactHooks',
Expand Down

0 comments on commit d357c4d

Please sign in to comment.