diff --git a/src/tw-html-nodejs-sync/Config/SyncInterval.tid b/src/tw-html-nodejs-sync/Config/SyncInterval.tid new file mode 100644 index 0000000..c5b486d --- /dev/null +++ b/src/tw-html-nodejs-sync/Config/SyncInterval.tid @@ -0,0 +1,4 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/Config/SyncInterval +description: minutes between sync + +3 \ No newline at end of file diff --git a/src/tw-html-nodejs-sync/Config/TiddlersPrefixToNotSync.tid b/src/tw-html-nodejs-sync/Config/TiddlersPrefixToNotSync.tid new file mode 100644 index 0000000..0ff514b --- /dev/null +++ b/src/tw-html-nodejs-sync/Config/TiddlersPrefixToNotSync.tid @@ -0,0 +1,3 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/Config/TiddlersPrefixToNotSync + +$:/state $:/temp \ No newline at end of file diff --git a/src/tw-html-nodejs-sync/Config/TiddlersToNotSync.tid b/src/tw-html-nodejs-sync/Config/TiddlersToNotSync.tid new file mode 100644 index 0000000..ba5e597 --- /dev/null +++ b/src/tw-html-nodejs-sync/Config/TiddlersToNotSync.tid @@ -0,0 +1,3 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/Config/TiddlersToNotSync + +$:/HistoryList $:/StoryList $:/Import $:/language $:/layout \ No newline at end of file diff --git a/src/tw-html-nodejs-sync/browser-background-sync.ts b/src/tw-html-nodejs-sync/browser-background-sync.ts new file mode 100644 index 0000000..ead7455 --- /dev/null +++ b/src/tw-html-nodejs-sync/browser-background-sync.ts @@ -0,0 +1,344 @@ +/* eslint-disable unicorn/no-array-callback-reference */ +import cloneDeep from 'lodash/cloneDeep'; +import mapValues from 'lodash/mapValues'; +import type { IServerStatus, ITiddlerFieldsParam, Tiddler } from 'tiddlywiki'; +import { activeServerStateTiddlerTitle, clientStatusStateTiddlerTitle, getLoopInterval } from './data/constants'; +import { filterOutNotSyncedTiddlers } from './data/filterOutNotSyncedTiddlers'; +import { getServerChangeFilter, getServerListFilter } from './data/filters'; +import { getClientInfoPoint, getFullHtmlEndPoint, getStatusEndPoint, getSyncEndPoint } from './data/getEndPoint'; +import { getSyncedTiddlersText } from './getSyncedTiddlersText'; +import type { IClientInfo, ISyncEndPointRequest } from './types'; +import { ConnectionState } from './types'; + +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +exports.name = 'browser-background-sync'; +exports.platforms = ['browser']; +// modules listed in https://tiddlywiki.com/dev/#StartupMechanism +// not blocking rendering +exports.after = ['render']; +exports.synchronous = true; +/* eslint-enable @typescript-eslint/no-unsafe-member-access */ + +interface IServerInfoTiddler extends Tiddler { + fields: Tiddler['fields'] & { + ipAddress: string; + /** + * Last synced time, be undefined if never synced + */ + lastSync: string | undefined; + name: string; + port: number; + text: ConnectionState; + }; +} + +class BackgroundSyncManager { + loop: ReturnType | undefined; + loopInterval: number; + /** lock the sync for `this.syncWithServer`, while last sync is still on progress */ + lock = false; + + constructor() { + // TODO: get this from setting + this.loopInterval = getLoopInterval(); + this.setupListener(); + this.startCheckServerStatusLoop(); + } + + setupListener() { + $tw.rootWidget.addEventListener('tw-html-nodejs-sync-get-server-status', async (event) => { + await this.getServerStatus(); + }); + $tw.rootWidget.addEventListener('tw-html-nodejs-sync-set-active-server-and-sync', async (event) => { + const titleToActive = event.paramObject?.title as string | undefined; + await this.setActiveServerAndSync(titleToActive); + }); + /** handle events from src/ui/ServerItemViewTemplate.tid 's $:/plugins/linonetwo/tw-html-nodejs-sync/ui/ServerItemViewTemplate */ + $tw.rootWidget.addEventListener('tw-html-nodejs-sync-sync-start', async (event) => { + await this.start(); + }); + $tw.rootWidget.addEventListener('tw-html-nodejs-sync-download-full-html', async (event) => { + await this.downloadFullHtmlAndApplyToWiki(); + }); + } + + startCheckServerStatusLoop() { + // eslint-disable-next-line @typescript-eslint/no-misused-promises, @typescript-eslint/promise-function-async + setInterval(() => this.getServerStatus(), this.loopInterval); + } + + async start(skipStatusCheck?: boolean) { + if (this.loop !== undefined) { + clearInterval(this.loop); + this.lock = false; + } + await this.onSyncStart(skipStatusCheck); + // eslint-disable-next-line @typescript-eslint/no-misused-promises, @typescript-eslint/promise-function-async + this.loop = setInterval(() => this.onSyncStart(skipStatusCheck), this.loopInterval); + } + + async onSyncStart(skipStatusCheck?: boolean) { + void this.getConnectedClientStatus(); + if (this.lock) { + return; + } + this.lock = true; + try { + if (skipStatusCheck !== true) { + await this.getServerStatus(); + } + await this.syncWithServer(); + // Maybe should add lock to avoid infinite loop, if also sync after autosave. But we don't have sync after autosave yet, so no lock on this is ok. + $tw.rootWidget.dispatchEvent({ type: 'tm-auto-save-wiki' }); + } finally { + this.lock = false; + } + } + + async setActiveServerAndSync(titleToActive: string | undefined) { + try { + if (typeof titleToActive === 'string' && $tw.wiki.getTiddler(titleToActive) !== undefined) { + // update status first + await this.getServerStatus(); + // get latest tiddler + const serverToActive = $tw.wiki.getTiddler(titleToActive); + if (serverToActive !== undefined) { + const newStatus = [ConnectionState.onlineActive, ConnectionState.online].includes(serverToActive.fields.text as ConnectionState) + ? ConnectionState.onlineActive + : ConnectionState.offlineActive; + $tw.wiki.addTiddler({ ...serverToActive.fields, text: newStatus }); + this.setActiveServerTiddlerTitle(titleToActive, serverToActive.fields.lastSync); + await this.start(true); + } + } + } catch (error) { + console.error(error); + } + } + + getActiveServerTiddlerTitle() { + return $tw.wiki.getTiddlerText(activeServerStateTiddlerTitle); + } + + setActiveServerTiddlerTitle(title: string, lastSync: string | undefined) { + // update active server record in `activeServerStateTiddlerTitle`, this is a pointer tiddler point to actual server tiddler + $tw.wiki.addTiddler({ title: activeServerStateTiddlerTitle, text: title, lastSync }); + // update server's last sync + const serverToActive = $tw.wiki.getTiddler(title); + if (serverToActive !== undefined) { + $tw.wiki.addTiddler({ ...serverToActive.fields, lastSync }); + } + } + + /** On TidGi desktop, get connected client info */ + async getConnectedClientStatus() { + try { + const response: Record = await fetch(getClientInfoPoint()).then( + async (response) => (await response.json()) as Record, + ); + Object.values(response).forEach((clientInfo) => { + $tw.wiki.addTiddler({ + title: `${clientStatusStateTiddlerTitle}/${clientInfo.Origin}`, + ...clientInfo, + }); + }); + } catch (error) { + console.warn(`tw-html-nodejs-sync can't connect to tw nodejs side. Error: ${(error as Error).message}`); + } + } + + /** On Tiddloid mobile, get TidGi server status */ + async getServerStatus() { + const timeout = 3000; + const activeTiddlerTitle = this.getActiveServerTiddlerTitle(); + const serverListWithUpdatedStatus = await Promise.all( + this.serverList.map(async (serverInfoTiddler) => { + const active = serverInfoTiddler.fields.title === activeTiddlerTitle; + try { + const controller = new AbortController(); + const id = setTimeout(() => { + controller.abort(); + }, timeout); + const response: IServerStatus = await fetch(getStatusEndPoint(serverInfoTiddler.fields.ipAddress, serverInfoTiddler.fields.port), { + signal: controller.signal, + }).then(async (response) => (await response.json()) as IServerStatus); + clearTimeout(id); + if (typeof response.tiddlywiki_version === 'string') { + return { + ...serverInfoTiddler, + fields: { + ...serverInfoTiddler.fields, + text: active ? ConnectionState.onlineActive : ConnectionState.online, + }, + }; + } + } catch (error) { + if ((error as Error).message.includes('The operation was aborted')) { + $tw.wiki.addTiddler({ + title: '$:/state/notification/tw-html-nodejs-sync/notification', + text: `GetServerStatus Timeout after ${timeout / 1000}s`, + }); + } else { + console.error(`getServerStatus() ${(error as Error).message} ${serverInfoTiddler.fields.name} ${(error as Error).stack ?? ''}`); + $tw.wiki.addTiddler({ + title: '$:/state/notification/tw-html-nodejs-sync/notification', + text: `GetServerStatus Failed ${(error as Error).message}`, + }); + } + } + $tw.notifier.display('$:/state/notification/tw-html-nodejs-sync/notification'); + return { + ...serverInfoTiddler, + fields: { + ...serverInfoTiddler.fields, + text: active ? ConnectionState.offlineActive : ConnectionState.offline, + }, + }; + }), + ); + serverListWithUpdatedStatus.forEach((tiddler) => { + $tw.wiki.addTiddler(tiddler.fields); + }); + } + + async syncWithServer() { + const onlineActiveServer = this.onlineActiveServer; + + if (onlineActiveServer !== undefined) { + // fix multiple online active server + this.serverList.forEach((serverInfoTiddler) => { + if (serverInfoTiddler?.fields?.text === ConnectionState.onlineActive && serverInfoTiddler?.fields?.title !== onlineActiveServer.fields.title) { + $tw.wiki.addTiddler({ ...serverInfoTiddler.fields, text: ConnectionState.online }); + } + }); + try { + const changedTiddlersFromClient = filterOutNotSyncedTiddlers(this.currentModifiedTiddlers); + + const requestBody: ISyncEndPointRequest = { tiddlers: changedTiddlersFromClient, lastSync: onlineActiveServer.fields.lastSync }; + // TODO: handle conflict, find intersection of changedTiddlersFromServer and changedTiddlersFromClient, and write changes to each other + // send modified tiddlers to server + const changedTiddlersFromServer: ITiddlerFieldsParam[] = await fetch( + getSyncEndPoint(onlineActiveServer.fields.ipAddress, onlineActiveServer.fields.port), + { + method: 'POST', + mode: 'cors', + body: JSON.stringify(requestBody), + headers: { + 'X-Requested-With': 'TiddlyWiki', + 'Content-Type': 'application/json', + }, + // TODO: add auth token in header, after we can scan QR code to get token easily + }, + ).then(async (response) => filterOutNotSyncedTiddlers((await response.json()) as ITiddlerFieldsParam[])); + changedTiddlersFromServer.forEach((tiddler) => { + // TODO: handle conflict + $tw.wiki.addTiddler(tiddler); + }); + + $tw.wiki.addTiddler({ + title: '$:/state/notification/tw-html-nodejs-sync/notification', + text: `Sync Complete ${getSyncedTiddlersText(changedTiddlersFromClient, changedTiddlersFromServer, { client: [], server: [] })}`, + }); + this.setActiveServerTiddlerTitle(onlineActiveServer.fields.title, this.getLastSyncString()); + } catch (error) { + console.error(error); + $tw.wiki.addTiddler({ + title: '$:/state/notification/tw-html-nodejs-sync/notification', + text: `Sync Failed ${(error as Error).message}`, + }); + } + $tw.notifier.display('$:/state/notification/tw-html-nodejs-sync/notification'); + } + } + + async downloadFullHtmlAndApplyToWiki() { + const onlineActiveServer = this.onlineActiveServer; + + if (onlineActiveServer !== undefined) { + try { + const fullHtml = await fetch(getFullHtmlEndPoint(onlineActiveServer.fields.ipAddress, onlineActiveServer.fields.port), { + mode: 'cors', + headers: { + 'X-Requested-With': 'TiddlyWiki', + 'Content-Type': 'text/html', + }, + }).then(async (response) => await response.text()); + this.setActiveServerTiddlerTitle(onlineActiveServer.fields.title, this.getLastSyncString()); + // get all state tiddlers we need, before document is overwritten + const serverList = cloneDeep(this.serverList); + // overwrite + document.write(fullHtml); + document.close(); + this.#showNotification(`Full html applied, set server list back.`); + + // write back after html stabled + addEventListener('DOMContentLoaded', (event) => { + setTimeout(() => { + $tw.wiki.addTiddlers(serverList.map((tiddler) => tiddler.fields)); + }, 1000); + }); + } catch (error) { + console.error(error); + this.#showNotification(`Full html apply failed ${(error as Error).message}`); + } + } + } + + get onlineActiveServer() { + return this.serverList.find((serverInfoTiddler) => { + return serverInfoTiddler?.fields?.text === ConnectionState.onlineActive; + }); + } + + /** + * update last sync using <> + */ + getLastSyncString() { + return $tw.utils.stringifyDate(new Date()); + } + + get currentModifiedTiddlers(): ITiddlerFieldsParam[] { + const onlineActiveServer = this.onlineActiveServer; + + if (onlineActiveServer === undefined) { + return []; + } + const lastSync = onlineActiveServer.fields.lastSync; + const diffTiddlersFilter: string = getServerChangeFilter(lastSync); + const diffTiddlers: string[] = $tw.wiki.compileFilter(diffTiddlersFilter)() ?? []; + return diffTiddlers + .map($tw.wiki.getTiddler) + .filter((tiddler): tiddler is Tiddler => tiddler !== undefined) + .map( + (tiddler): ITiddlerFieldsParam => + mapValues(tiddler.fields, (value) => { + if (value instanceof Date) { + return $tw.utils.stringifyDate(value); + } + return value as string; + }), + ); + } + + get serverList() { + // get server list using filter + const serverList: string[] = $tw.wiki.compileFilter(getServerListFilter())() ?? []; + return serverList.map((serverInfoTiddlerTitle) => { + return $tw.wiki.getTiddler(serverInfoTiddlerTitle) as IServerInfoTiddler; + }); + } + + #showNotification(text: string) { + $tw.wiki.addTiddler({ + title: '$:/state/notification/tw-html-nodejs-sync/notification', + text, + }); + $tw.notifier.display('$:/state/notification/tw-html-nodejs-sync/notification'); + } +} + +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +exports.startup = () => { + const syncManager = new BackgroundSyncManager(); + void syncManager.start(); +}; diff --git a/src/tw-html-nodejs-sync/browser-background-sync.ts.meta b/src/tw-html-nodejs-sync/browser-background-sync.ts.meta new file mode 100644 index 0000000..67bb47d --- /dev/null +++ b/src/tw-html-nodejs-sync/browser-background-sync.ts.meta @@ -0,0 +1,4 @@ +creator: LinOnetwo +title: $:/plugins/linonetwo/tw-html-nodejs-sync/browser-background-sync.js +type: application/javascript +module-type: startup diff --git a/src/tw-html-nodejs-sync/data/clientInfoStore.ts b/src/tw-html-nodejs-sync/data/clientInfoStore.ts new file mode 100644 index 0000000..8d63103 --- /dev/null +++ b/src/tw-html-nodejs-sync/data/clientInfoStore.ts @@ -0,0 +1,4 @@ +import { ClientInfoStore } from './clientInfoStoreClass'; + +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +module.exports.store = new ClientInfoStore(); diff --git a/src/tw-html-nodejs-sync/data/clientInfoStore.ts.meta b/src/tw-html-nodejs-sync/data/clientInfoStore.ts.meta new file mode 100644 index 0000000..0ed8fd8 --- /dev/null +++ b/src/tw-html-nodejs-sync/data/clientInfoStore.ts.meta @@ -0,0 +1,3 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/clientInfoStore.js +type: application/javascript +module-type: library diff --git a/src/tw-html-nodejs-sync/data/clientInfoStoreClass.ts b/src/tw-html-nodejs-sync/data/clientInfoStoreClass.ts new file mode 100644 index 0000000..0588801 --- /dev/null +++ b/src/tw-html-nodejs-sync/data/clientInfoStoreClass.ts @@ -0,0 +1,48 @@ +/* eslint-disable @typescript-eslint/restrict-plus-operands */ +/* eslint-disable @typescript-eslint/strict-boolean-expressions */ +import structuredClone from '@ungap/structured-clone'; +import * as types from '../types'; +import { getLoopInterval } from './constants'; +import UAParser from 'ua-parser-js'; + +export class ClientInfoStore { + #clients: Record = {}; + loopHandel: NodeJS.Timer; + + constructor() { + const loopInterval = getLoopInterval(); + const keyOfflineTimeout = loopInterval * 2; + const keyDeleteTimeout = loopInterval * 10; + this.loopHandel = setInterval(() => { + Object.keys(this.#clients).forEach((key) => { + const timestamp = this.#clients[key].timestamp; + // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions + if (!timestamp || Date.now() - timestamp > keyDeleteTimeout) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete this.#clients[key]; + } else if (Date.now() - timestamp > keyOfflineTimeout) { + this.#clients[key].state = types.ConnectionState.offline; + } + }); + }, loopInterval); + } + + get allClient() { + return structuredClone(this.#clients); + } + + updateClient(key: string, value: Partial) { + this.#clients[key] = { ...this.#clients[key], ...value }; + const ua = this.#clients[key]['User-Agent']; + if (ua) { + const userAgentInfo = new UAParser(ua); + const model = userAgentInfo.getDevice().model; + const os = userAgentInfo.getOS().name; // 获取系统 + this.#clients[key].name = model ?? userAgentInfo.getBrowser().name ?? this.#clients[key].Origin; + this.#clients[key].model = model; + this.#clients[key].os = os; + } else { + this.#clients[key].name = this.#clients[key].Origin; + } + } +} diff --git a/src/tw-html-nodejs-sync/data/constants.ts b/src/tw-html-nodejs-sync/data/constants.ts new file mode 100644 index 0000000..b98041f --- /dev/null +++ b/src/tw-html-nodejs-sync/data/constants.ts @@ -0,0 +1,4 @@ +export const activeServerStateTiddlerTitle = `$:/state/tw-html-nodejs-sync/activeServer`; +export const clientStatusStateTiddlerTitle = '$:/state/tw-html-nodejs-sync/clientStatus'; +// eslint-disable-next-line @typescript-eslint/strict-boolean-expressions +export const getLoopInterval = () => (Number($tw.wiki.getTiddlerText('$:/plugins/linonetwo/tw-html-nodejs-sync/Config/SyncInterval')) || 3) * 60 * 1000; diff --git a/src/tw-html-nodejs-sync/data/filterOutNotSyncedTiddlers.ts b/src/tw-html-nodejs-sync/data/filterOutNotSyncedTiddlers.ts new file mode 100644 index 0000000..0fb3a88 --- /dev/null +++ b/src/tw-html-nodejs-sync/data/filterOutNotSyncedTiddlers.ts @@ -0,0 +1,15 @@ +import { ITiddlerFields, ITiddlerFieldsParam } from 'tiddlywiki'; + +let tiddlersToNotSync: Set | undefined; +let prefixToNotSync: string[] | undefined; +export const filterOutNotSyncedTiddlers = (tiddlers: T[]): T[] => { + if (tiddlersToNotSync === undefined || prefixToNotSync === undefined) { + tiddlersToNotSync = new Set( + $tw.utils.parseStringArray($tw.wiki.getTiddlerText('$:/plugins/linonetwo/tw-html-nodejs-sync/Config/TiddlersToNotSync') ?? ''), + ); + prefixToNotSync = $tw.utils.parseStringArray($tw.wiki.getTiddlerText('$:/plugins/linonetwo/tw-html-nodejs-sync/Config/TiddlersPrefixToNotSync') ?? ''); + } + return tiddlers + .filter((tiddler: T) => !prefixToNotSync!.some((prefix) => (tiddler.title as string).startsWith(prefix))) + .filter((tiddler: T) => !tiddlersToNotSync!.has(tiddler.title as string)); +}; diff --git a/src/tw-html-nodejs-sync/data/filters.ts b/src/tw-html-nodejs-sync/data/filters.ts new file mode 100644 index 0000000..3e49b2c --- /dev/null +++ b/src/tw-html-nodejs-sync/data/filters.ts @@ -0,0 +1,8 @@ +export function getServerChangeFilter(lastSync: string | undefined) { + return `[all[]] :filter[get[modified]compare:date:gt[${lastSync ?? ''}]]`; +} + +/** + * also in src/ui/ServerList.tid 's list widget + */ +export const getServerListFilter = () => $tw.wiki.getTiddlerText('$:/plugins/linonetwo/tw-html-nodejs-sync/ServerListFilter'); diff --git a/src/tw-html-nodejs-sync/data/getClientInfo.ts b/src/tw-html-nodejs-sync/data/getClientInfo.ts new file mode 100644 index 0000000..081383d --- /dev/null +++ b/src/tw-html-nodejs-sync/data/getClientInfo.ts @@ -0,0 +1,11 @@ +import type Http from 'http'; +import { ConnectionState, IClientInfo } from '../types'; + +export function getClientInfo(request: Http.ClientRequest & Http.InformationEvent, state = ConnectionState.online): Partial { + return { + Origin: request.rawHeaders[request.rawHeaders.indexOf('Origin') + 1] ?? request.rawHeaders[request.rawHeaders.indexOf('Referer') + 1], + 'User-Agent': request.rawHeaders[request.rawHeaders.indexOf('User-Agent') + 1], + timestamp: Date.now(), + state, + }; +} diff --git a/src/tw-html-nodejs-sync/data/getEndPoint.ts b/src/tw-html-nodejs-sync/data/getEndPoint.ts new file mode 100644 index 0000000..30bf6b1 --- /dev/null +++ b/src/tw-html-nodejs-sync/data/getEndPoint.ts @@ -0,0 +1,27 @@ +export const syncRoute = '/tw-html-nodejs-sync/html-node-sync'; +export const statusRoute = '/tw-html-nodejs-sync/status'; +export const clientInfoRoute = '/tw-html-nodejs-sync/client-info'; +export const fullHtmlRoute = '/tw-html-nodejs-sync/get-full-html'; + +/** + * Our custom endpoint that used to sync with the server + */ +export function getSyncEndPoint(ipAddress: string, port: number): string { + return `http://${ipAddress}:${port}${syncRoute}`; +} + +/** + * Official status endpoint + */ +export function getStatusEndPoint(ipAddress: string, port: number): string { + return `http://${ipAddress}:${port}${statusRoute}`; +} + +/** Used for NodeJS server's client page get info from the same origin */ +export function getClientInfoPoint(): string { + return `http://${location.host}${clientInfoRoute}`; +} + +export function getFullHtmlEndPoint(ipAddress: string, port: number): string { + return `http://${location.host}${fullHtmlRoute}`; +} diff --git a/src/tw-html-nodejs-sync/data/mergeTiddler.ts b/src/tw-html-nodejs-sync/data/mergeTiddler.ts new file mode 100644 index 0000000..2be685f --- /dev/null +++ b/src/tw-html-nodejs-sync/data/mergeTiddler.ts @@ -0,0 +1,47 @@ +/* eslint-disable @typescript-eslint/strict-boolean-expressions */ +import { applyPatches, makePatches } from '@sanity/diff-match-patch'; +import type { ITiddlerFields } from 'tiddlywiki'; + +/** + * + * @url https://neil.fraser.name/software/diff_match_patch/demos/patch.html + * @param a + * @param b + * @returns + */ +export function mergeTiddler(a: ITiddlerFields, b: ITiddlerFields): ITiddlerFields { + if (a.title !== b.title) { + throw new Error(`Cannot merge tiddlers with different titles: ${a.title} and ${b.title}`); + } + const newerOne = a.modified > b.modified ? a : b; + if (leftIsBinaryRightIsString(a, b)) { + // we choose binary, because we assume binary data is more important than pure text + return a; + } else if (leftIsBinaryRightIsString(b, a)) { + return b; + } else if (isBinaryTiddler(a) && isBinaryTiddler(b)) { + return newerOne; + } + // both is string tiddler, we can merge them using diff-match-patch algorithm + // FIXME: Currently not working, it needs `c` that is an older version from server to work (3-way-merge), otherwise it will just use `b.text` as the merged text + const patches = makePatches(a.text, b.text); + const [mergedText] = applyPatches(patches, a.text); + const fields: ITiddlerFields = { + ...newerOne, + text: mergedText, + }; + return fields; +} + +function leftIsBinaryRightIsString(a: ITiddlerFields, b: ITiddlerFields) { + // a is binary, b is string + if (isBinaryTiddler(a) && !isBinaryTiddler(b)) { + return true; + } + return false; +} + +export function isBinaryTiddler(tiddlerFields: ITiddlerFields): boolean { + const contentTypeInfo = $tw.config.contentTypeInfo[tiddlerFields.type || 'text/vnd.tiddlywiki']; + return !!contentTypeInfo && contentTypeInfo.encoding === 'base64'; +} diff --git a/src/tw-html-nodejs-sync/data/toTWUTCString.ts b/src/tw-html-nodejs-sync/data/toTWUTCString.ts new file mode 100644 index 0000000..8395391 --- /dev/null +++ b/src/tw-html-nodejs-sync/data/toTWUTCString.ts @@ -0,0 +1,17 @@ +export function pad(number: number) { + if (number < 10) { + return `0${number}`; + } + return String(number); +} +export function toTWUTCString(date: Date) { + return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}${ + pad( + date.getUTCHours(), + ) + }${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}${ + (date.getUTCMilliseconds() / 1000) + .toFixed(3) + .slice(2, 5) + }`; +} diff --git a/src/tw-html-nodejs-sync/develop.tid b/src/tw-html-nodejs-sync/develop.tid new file mode 100755 index 0000000..95f3e48 --- /dev/null +++ b/src/tw-html-nodejs-sync/develop.tid @@ -0,0 +1,7 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/develop +creator: LinOnetwo +type: text/vnd.tiddlywiki + +TidGi-Mobile use [[$:/plugins/linonetwo/tw-html-nodejs-sync/templates/skinny-tiddlywiki5.html]] as template. See [[$:/plugins/linonetwo/tw-html-nodejs-sync/templates/about-skinny-tiddlywiki5-html]] for how it differ from original lazy-all template. + +Use [[SIOC|https://github.com/taurenshaman/semantic-web/blob/cee9e421eed3f31a602f82375fb92fb109d638b7/data/sioc.rdf]] Ontology for server tiddler. diff --git a/src/tw-html-nodejs-sync/getSyncedTiddlersText.ts b/src/tw-html-nodejs-sync/getSyncedTiddlersText.ts new file mode 100644 index 0000000..cbccf61 --- /dev/null +++ b/src/tw-html-nodejs-sync/getSyncedTiddlersText.ts @@ -0,0 +1,34 @@ +/* eslint-disable @typescript-eslint/strict-boolean-expressions */ +import take from 'lodash/take'; +import { ITiddlerFields, ITiddlerFieldsParam } from 'tiddlywiki'; + +export function getSyncedTiddlersText( + changedTiddlersFromClient: Array, + changedTiddlersFromServer: Array, + deletion: { client: string[]; server: string[] }, + options?: { reverse?: boolean }, +) { + const changedTitleDisplayLimit = 5; + + const formatList = (list: string[]) => take(list, changedTitleDisplayLimit).join(' '); + const moreCountText = (list: string[]) => list.length > changedTitleDisplayLimit ? `And ${list.length - changedTitleDisplayLimit} more` : ''; + + const clientText = formatList(changedTiddlersFromClient.map(tiddler => tiddler.caption as string ?? (tiddler.title))); + const clientCount = moreCountText(changedTiddlersFromClient.map(item => item.title as string)); + const serverText = formatList(changedTiddlersFromServer.map(tiddler => tiddler.caption as string ?? (tiddler.title))); + const serverCount = moreCountText(changedTiddlersFromServer.map(item => item.title as string)); + + const deletionClientText = formatList(deletion.client); + const deletionClientCount = moreCountText(deletion.client); + const deletionServerText = formatList(deletion.server); + const deletionServerCount = moreCountText(deletion.server); + + const up = options?.reverse ? '↓' : '↑'; + const down = options?.reverse ? '↑' : '↓'; + + return `${up} ${changedTiddlersFromClient.length} ${down} ${changedTiddlersFromServer.length}${ + changedTiddlersFromClient.length > 0 ? `\n\n${up}: ${clientText} ${clientCount}` : '' + }${changedTiddlersFromServer.length > 0 ? `\n\n${down}: ${serverText} ${serverCount}` : ''}${ + deletion.client.length > 0 ? `\n\nDeleted on Client: ${deletionClientText} ${deletionClientCount}` : '' + }${deletion.server.length > 0 ? `\n\nDeleted on Server: ${deletionServerText} ${deletionServerCount}` : ''}`; +} diff --git a/src/tw-html-nodejs-sync/plugin.info b/src/tw-html-nodejs-sync/plugin.info new file mode 100644 index 0000000..d476aec --- /dev/null +++ b/src/tw-html-nodejs-sync/plugin.info @@ -0,0 +1,13 @@ +{ + "title": "$:/plugins/linonetwo/tw-html-nodejs-sync", + "author": "LinOnetwo", + "description": "Sync data between Mobile HTML (Tiddloid/Quine2) <-> Desktop NodeJS App (TidGi) ", + "core-version": ">=5.1.22", + "plugin-type": "plugin", + "version": "0.6.5", + "list": "readme ui/ServerList", + "dependents": [ + "$:/plugins/tiddlywiki/qrcode", + "$:/plugins/kixam/datepicker" + ] +} \ No newline at end of file diff --git a/src/tw-html-nodejs-sync/readme.tid b/src/tw-html-nodejs-sync/readme.tid new file mode 100755 index 0000000..83d97b0 --- /dev/null +++ b/src/tw-html-nodejs-sync/readme.tid @@ -0,0 +1,68 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/readme +creator: LinOnetwo +type: text/vnd.tiddlywiki + +\define image() + >/> +\end + +! Sync Between NodeJS and Mobile HTML 在桌面端（NodeJS）和移动端（HTML文件）之间同步 + +本插件可以让你在基于NodeJS技术的桌面应用（例如太记）和基于HTML文件的手机端（例如Tiddloid安卓应用）之间同步数据。 + +手机应用 ↔ 桌面应用 ↔ 云端 + +This plugin enables you sync date between NodeJS server App (e.g. TidGi App) and HTML file based mobile App (e.g. Tiddloid Android App). + +Mobile App ↔ Desktop App ↔ Cloud + +!! How to use + +首先在手机端扫码，打开当前这个知识库的网页，然后保存出 HTML 文件后（详见中文教程的[[如何保存|https://tw-cn.netlify.app/#TiddlyGit%E5%A4%AA%E8%AE%B0]]章节），在 Tiddloid 里打开保存的 HTML 文件（可能需要先将文件移出下载目录不然Tiddloid无权限访问，详见其[[说明书|https://github.com/donmor/Tiddloid]]）。 + +然后打开[[服务器列表|$:/plugins/linonetwo/tw-html-nodejs-sync/ui/ServerList]]，录入一下服务器地址： + +# 将上面扫码得到的 URL 复制一下，填入[[服务器列表|$:/plugins/linonetwo/tw-html-nodejs-sync/ui/ServerList]]里的「智能识别黏贴框」，然后点击「智能识别」按钮，会自动将 URL 解析后填入相应框内。你也可以点击「扫二维码」按钮开始扫码，扫码结果会自动填入「智能识别黏贴框」内 +# 此时，「服务器IP」和「服务器端口」框里应该要已经填好了 ip 和端口号 +# 你需要填写一下「服务器名」这个框 +# 点击「新增服务器」，会新建一个服务器信息条目，请确认创建此条目 +# 这时新的服务器应该就出现在服务器列表里了，你可以点击「启用同步并立即同步」按钮，这样就可以在桌面端和手机端同步数据了 +# 之后会每五分钟自动同步一次用户创建的条目（不包含插件），如果需要从桌面端同步插件到手机端，需要使用「[[拉取并覆盖|$:/plugins/linonetwo/tw-html-nodejs-sync/ui/DownloadFullHtml]]」功能 + +First scan the code on your cell phone, open the current knowledge base page, then save the HTML file (see the [[How to save|https://tw-cn.netlify.app/#TiddlyGit%E5%A4%AA%E8%AE%B0]] section of the Chinese tutorial for more details) and open the saved HTML file in Tiddloid (You may need to move the file out of the download directory first or Tiddloid will not have access to it, see its [[Instructions|https://github.com/donmor/Tiddloid]] for details). + +Then open the [[Server List|$:/plugins/linonetwo/tw-html-nodejs-sync/ui/ServerList]] and enter the following server address. + +# Copy the URL you got from the code above and fill in the "Smart Identify Sticky Box" in [[Server List|$:/plugins/linonetwo/tw-html-nodejs-sync/ui/ServerList]], then click the "Smart Identify" button, the URL will be automatically parsed and filled in the corresponding box. You can also click the "Scan QR Code" button to start scanning the code, and the result will be automatically filled into the "Smart Identify Sticky Box". +# At this point, the "Server IP" and "Server Port" boxes should already be filled with the ip and port numbers. +# You need to fill in the "server name" box +# Click on "Add Server", a new server information entry will be created, please confirm to create this entry +# The new server should now appear in the server list, you can click the "Enable sync and sync now" button to sync data between desktop and mobile. +# If you need to sync the plugin from desktop to mobile, you need to use the "[[pull and overwrite|$:/plugins/linonetwo/tw-html-nodejs-sync/ui/DownloadFullHtml]]" function. + +Translated with www.DeepL.com/Translator (free version) + +!! Server list and forms 服务器列表和新增表单 + +[[Server List|$:/plugins/linonetwo/tw-html-nodejs-sync/ui/ServerList]] contains syncable servers, and button to add new server. + +!! QrCode of current server 当前服务器的二维码 + +<$set name="content" value={{{ [{$:/info/url/host}addprefix[//]addprefix{$:/info/url/protocol}addsuffix[/tw-html-nodejs-sync/get-full-html]] }}}> + <> + <> + + +Scan to add this server 扫码以添加此服务器 ({{$:/info/url/full}}). + +（二维码：当前服务器的地址） + +Needs official QR plugin [[$:/plugins/tiddlywiki/qrcode]] to work. 需要安装官方二维码插件 [[$:/plugins/tiddlywiki/qrcode]] 才会有二维码出现在上面 + +!! FAQ 常见问题 + +!!! 在外面用了一天之后回家添加服务器，却无法同步到电脑端 When you come home to add a server after using it outside for a day, it won't sync to the computer side + +就是因为你新添加服务器后，它只会同步新建服务器之后新建的条目。这是又因为新建服务器时会根据当前时间添加一个 lastSync 字段，目前在点击同步按钮时，只会同步这个时间点之后的内容。你可以手动把服务器条目里这个字段的时间改早一天，这样就会同步你今天添加的内容了。未来我们将会添加更智能的同步方式，就不再需要这个字段了，就不会遇到这个问题了。 + +This is because when you add a new server, it only syncs the new entries created after the new server is created. This is again because when you create a new server you add a lastSync field based on the current time, and currently when you click the sync button it will only sync after this point in time. You can manually change the time of this field in the server entry to one day earlier, and it will sync the content you added today. In the future we will add a smarter way of syncing so that this field is no longer needed and you won't encounter this problem. diff --git a/src/tw-html-nodejs-sync/scan-qr-widget-style.tid b/src/tw-html-nodejs-sync/scan-qr-widget-style.tid new file mode 100644 index 0000000..bea4fb8 --- /dev/null +++ b/src/tw-html-nodejs-sync/scan-qr-widget-style.tid @@ -0,0 +1,29 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/scan-qr-widget-style.css +tags: $:/tags/Stylesheet +type: text/css + +#scan-qr-widget-loadingMessage { + text-align: center; + padding: 40px; + background-color: #eee; +} + +#scan-qr-widget-canvas { + width: 100%; +} + +#scan-qr-widget-output { + margin-top: 20px; + background: #eee; + padding: 10px; + padding-bottom: 0; +} + +#scan-qr-widget-output div { + padding-bottom: 10px; + word-wrap: break-word; +} + +#scan-qr-widget-noQRFound { + text-align: center; +} diff --git a/src/tw-html-nodejs-sync/scan-qr-widget.ts b/src/tw-html-nodejs-sync/scan-qr-widget.ts new file mode 100644 index 0000000..e1e74b8 --- /dev/null +++ b/src/tw-html-nodejs-sync/scan-qr-widget.ts @@ -0,0 +1,194 @@ +/* eslint-disable @typescript-eslint/strict-boolean-expressions */ +import type { Widget as IWidget, IChangedTiddlers } from 'tiddlywiki'; +import jsQR from 'jsqr-es6'; +import type { Point } from 'jsqr-es6/dist/locator'; + +const Widget = (require('$:/core/modules/widgets/widget.js') as { widget: typeof IWidget }).widget; + +class ScanQRWidget extends Widget { + refresh(_changedTiddlers: IChangedTiddlers) { + return false; + } + + /** id to prevent multiple loops */ + loopId = 0; + + /** + * Lifecycle method: Render this widget into the DOM + */ + render(parent: Node, _nextSibling: Node) { + this.parentDomNode = parent; + this.computeAttributes(); + this.execute(); + + /** tiddler to put the result */ + const outputTiddler = this.getAttribute('outputTiddler'); + /** + * tiddler contains the open state of this widget. For example: + * + * ```tw5 + * <$reveal type="match" state="$:/state/tw-html-nodejs-sync/server/new/scan-qr-widget-open" text="yes"> + <$ScanQRWidget outputTiddler="$:/state/tw-html-nodejs-sync/server/new" stopOnDetect="yes" stateTiddler="$:/state/tw-html-nodejs-sync/server/new/scan-qr-widget-open" /> + + * ``` + */ + const stateTiddler = this.getAttribute('stateTiddler'); + const outputField = this.getAttribute('field') || 'text'; + const stopOnDetect = this.getAttribute('stopOnDetect') === 'yes' || this.getAttribute('stopOnDetect') === 'true'; + + const containerElement = document.createElement('div'); + containerElement.innerHTML = ` +
+
🎥 Unable to access video stream (please make sure you have a webcam enabled)
+ + +
+ `; + this.domNodes.push(containerElement); + this.loopId += 1; + const loopId = this.loopId; + // wait till dom created + requestAnimationFrame(() => void this.jsqr(loopId, containerElement, { outputTiddler, stopOnDetect, stateTiddler, outputField })); + parent.appendChild(containerElement); + } + + async jsqr( + loopId: number, + containerElement: HTMLDivElement, + options: { outputField: string; outputTiddler?: string; stateTiddler?: string; stopOnDetect: boolean }, + ) { + const video = document.createElement('video'); + const canvasElement = document.querySelector('#scan-qr-widget-canvas'); + if (canvasElement === null) { + console.warn('ScanQRWidget: canvasElement is null'); + return; + } + const canvas = canvasElement.getContext('2d'); + const loadingMessage = document.querySelector('#scan-qr-widget-loadingMessage'); + const outputContainer = document.querySelector('#scan-qr-widget-output'); + const outputMessage = document.querySelector('#scan-qr-widget-outputMessage'); + const outputData = document.querySelector('#scan-qr-widget-outputData'); + if (canvas === null || outputData === null) { + console.warn('ScanQRWidget: canvas or outputData is null', { canvas, outputData }); + return; + } + + // Use facingMode: environment to attemt to get the front camera on phones + const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment' } }); + + function drawLine(begin: Point, end: Point, color: string | CanvasGradient | CanvasPattern) { + if (canvas === null) { + return; + } + canvas.beginPath(); + canvas.moveTo(begin.x, begin.y); + canvas.lineTo(end.x, end.y); + canvas.lineWidth = 4; + canvas.strokeStyle = color; + canvas.stroke(); + } + + let lastResult: string | undefined; + let hasDetectedResult = false; + + const tick = () => { + if ( + loadingMessage === null || + canvasElement === null || + outputContainer === null || + canvas === null || + outputMessage === null || + outputData === null || + outputData.parentElement === null + ) { + console.warn( + 'ScanQRWidget: !loadingMessage || !canvasElement || !outputContainer || !canvas || !outputMessage || !outputData || !outputData.parentElement, it is null', + { + loadingMessage, + canvasElement, + outputContainer, + canvas, + outputMessage, + outputData, + 'outputData.parentElement': outputData?.parentElement, + }, + ); + + return; + } + loadingMessage.textContent = '⌛ Loading video...'; + if (video.readyState === video.HAVE_ENOUGH_DATA) { + loadingMessage.hidden = true; + canvasElement.hidden = false; + outputContainer.hidden = false; + + canvasElement.height = video.videoHeight; + canvasElement.width = video.videoWidth; + canvas.drawImage(video, 0, 0, canvasElement.width, canvasElement.height); + const imageData = canvas.getImageData(0, 0, canvasElement.width, canvasElement.height); + const code = jsQR(imageData.data, imageData.width, imageData.height, { + inversionAttempts: 'dontInvert', + }); + outputMessage.hidden = true; + outputData.parentElement.hidden = false; + let result; + if (code === null) { + result = 'No code detected'; + } else { + drawLine(code.location.topLeftCorner, code.location.topRightCorner, '#FF3B58'); + drawLine(code.location.topRightCorner, code.location.bottomRightCorner, '#FF3B58'); + drawLine(code.location.bottomRightCorner, code.location.bottomLeftCorner, '#FF3B58'); + drawLine(code.location.bottomLeftCorner, code.location.topLeftCorner, '#FF3B58'); + result = code.data; + hasDetectedResult = true; + } + + if (result !== lastResult) { + outputData.textContent = outputData.textContent ?? ''; + outputData.textContent += `${result}\n`; + lastResult = result; + // fast check of ip address + if (options.outputTiddler) { + const textFieldTiddler = $tw.wiki.getTiddler(options.outputTiddler); + const newServerInfoTiddler = { + ...textFieldTiddler?.fields, + title: options.outputTiddler, + [options.outputField]: result, + }; + // create if not exists + $tw.wiki.addTiddler(newServerInfoTiddler); + } + } + } + const stopDueToHasResult = options.stopOnDetect && hasDetectedResult; + /** if new loop happened, this.loopId will > loopId, stop current loop */ + const canContinueCurrentLoop = this.loopId === loopId && containerElement.offsetParent !== null; + if (!canContinueCurrentLoop || stopDueToHasResult) { + video.pause(); + video.parentElement?.removeChild(video); + stream.getTracks().forEach(function (track) { + track.stop(); + }); + } + if (stopDueToHasResult && options?.stateTiddler) { + $tw.wiki.addTiddler({ title: options.stateTiddler, text: 'no' }); + } + if (canContinueCurrentLoop && !stopDueToHasResult) { + requestAnimationFrame(tick); + } + }; + + // initialize the first tick + video.srcObject = stream; + video.setAttribute('playsinline', 'true'); // required to tell iOS safari we don't want fullscreen + await video.play(); + requestAnimationFrame(tick); + } +} + +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +exports.widget = ScanQRWidget; +exports.ScanQRWidget = ScanQRWidget; diff --git a/src/tw-html-nodejs-sync/scan-qr-widget.ts.meta b/src/tw-html-nodejs-sync/scan-qr-widget.ts.meta new file mode 100644 index 0000000..0ab36c6 --- /dev/null +++ b/src/tw-html-nodejs-sync/scan-qr-widget.ts.meta @@ -0,0 +1,3 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/scan-qr-widget.js +type: application/javascript +module-type: widget diff --git a/src/tw-html-nodejs-sync/server/SaveTemplate/about-skinny-tiddlywiki5-html.tid b/src/tw-html-nodejs-sync/server/SaveTemplate/about-skinny-tiddlywiki5-html.tid new file mode 100644 index 0000000..f19640f --- /dev/null +++ b/src/tw-html-nodejs-sync/server/SaveTemplate/about-skinny-tiddlywiki5-html.tid @@ -0,0 +1,38 @@ +title: $:/plugins/linonetwo/tw-html-nodejs-sync/templates/about-skinny-tiddlywiki5-html + +!! Difference + +!!! $:/plugins/linonetwo/tw-html-nodejs-sync/templates/save/lazy-all + +* Add some plugin to saveTiddlerFilter ignore list at the end, and remove useless `+[sort[title]]` +* Replace `{{$:/core/templates/tiddlywiki5.html}}` with our `{{$:/plugins/linonetwo/tw-html-nodejs-sync/templates/skinny-tiddlywiki5.html}}` + +!!! $:/plugins/linonetwo/tw-html-nodejs-sync/templates/skinny-tiddlywiki5.html + +`