From 9c14fb99fa5034b22df82106786a9a573a713d37 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 24 Jan 2025 13:19:20 -0500 Subject: [PATCH 01/10] Create channels service and small fix for loading chain on app start --- 3rd-party/auth | 2 +- packages/backend/package.json | 1 - .../connections-manager.service.ts | 40 +- .../src/nest/storage/channels/channel.ts | 0 .../nest/storage/channels/channels.service.ts | 484 +++++++++++ .../src/nest/storage/storage.module.ts | 2 + .../src/nest/storage/storage.service.spec.ts | 802 +++++++++--------- .../src/nest/storage/storage.service.ts | 424 +-------- 8 files changed, 922 insertions(+), 833 deletions(-) create mode 100644 packages/backend/src/nest/storage/channels/channel.ts create mode 100644 packages/backend/src/nest/storage/channels/channels.service.ts diff --git a/3rd-party/auth b/3rd-party/auth index 56cf854389..fd7101145f 160000 --- a/3rd-party/auth +++ b/3rd-party/auth @@ -1 +1 @@ -Subproject commit 56cf85438990004975f34bf121bdef22d4b81068 +Subproject commit fd7101145fc15aeb14bda46578b7a4d6d84e4e5b diff --git a/packages/backend/package.json b/packages/backend/package.json index 0e223593c2..c792b3f713 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -82,7 +82,6 @@ "mock-fs": "^5.1.2", "pvutils": "^1.1.3", "tmp": "^0.2.1", - "pvutils": "^1.1.3", "ts-jest": "^29.0.3", "ts-loader": "9.4.2", "ts-node": "10.9.1", diff --git a/packages/backend/src/nest/connections-manager/connections-manager.service.ts b/packages/backend/src/nest/connections-manager/connections-manager.service.ts index d6a2d7c822..b0bdfcc75e 100644 --- a/packages/backend/src/nest/connections-manager/connections-manager.service.ts +++ b/packages/backend/src/nest/connections-manager/connections-manager.service.ts @@ -896,7 +896,7 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI this.serverIoProvider.io.emit(SocketActionTypes.CERTIFICATES_STORED, { certificates: await this.storageService?.loadAllCertificates(), }) - await this.storageService?.loadAllChannels() + await this.storageService?.channels.loadAllChannels() } }) this.socketService.on( @@ -994,7 +994,7 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI this.socketService.on( SocketActionTypes.CREATE_CHANNEL, async (args: CreateChannelPayload, callback: (response?: CreateChannelResponse) => void) => { - callback(await this.storageService?.subscribeToChannel(args.channel)) + callback(await this.storageService?.channels.subscribeToChannel(args.channel)) } ) this.socketService.on( @@ -1003,39 +1003,39 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI payload: { channelId: string; ownerPeerId: string }, callback: (response: DeleteChannelResponse) => void ) => { - callback(await this.storageService?.deleteChannel(payload)) + callback(await this.storageService?.channels.deleteChannel(payload)) } ) this.socketService.on( SocketActionTypes.DELETE_FILES_FROM_CHANNEL, async (payload: DeleteFilesFromChannelSocketPayload) => { this.logger.info(`socketService - ${SocketActionTypes.DELETE_FILES_FROM_CHANNEL}`) - await this.storageService?.deleteFilesFromChannel(payload) + await this.storageService?.channels.deleteFilesFromChannel(payload) // await this.deleteFilesFromTemporaryDir() //crashes on mobile, will be fixes in next versions } ) this.socketService.on(SocketActionTypes.SEND_MESSAGE, async (args: SendMessagePayload) => { - await this.storageService?.sendMessage(args.message) + await this.storageService?.channels.sendMessage(args.message) }) this.socketService.on( SocketActionTypes.GET_MESSAGES, async (payload: GetMessagesPayload, callback: (response?: MessagesLoadedPayload) => void) => { - callback(await this.storageService?.getMessages(payload.channelId, payload.ids)) + callback(await this.storageService?.channels.getMessages(payload.channelId, payload.ids)) } ) // Files this.socketService.on(SocketActionTypes.DOWNLOAD_FILE, async (metadata: FileMetadata) => { - await this.storageService?.downloadFile(metadata) + await this.storageService?.channels.downloadFile(metadata) }) this.socketService.on(SocketActionTypes.UPLOAD_FILE, async (metadata: FileMetadata) => { - await this.storageService?.uploadFile(metadata) + await this.storageService?.channels.uploadFile(metadata) }) this.socketService.on(SocketActionTypes.FILE_UPLOADED, async (args: FileMetadata) => { - await this.storageService?.uploadFile(args) + await this.storageService?.channels.uploadFile(args) }) this.socketService.on(SocketActionTypes.CANCEL_DOWNLOAD, mid => { - this.storageService?.cancelDownload(mid) + this.storageService?.channels.cancelDownload(mid) }) // System @@ -1058,37 +1058,37 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI this.logger.info(`Storage - ${StorageEvents.CERTIFICATES_STORED}`) this.serverIoProvider.io.emit(SocketActionTypes.CERTIFICATES_STORED, payload) }) - this.storageService.on(StorageEvents.CHANNELS_STORED, (payload: ChannelsReplicatedPayload) => { + this.storageService.channels.on(StorageEvents.CHANNELS_STORED, (payload: ChannelsReplicatedPayload) => { this.serverIoProvider.io.emit(SocketActionTypes.CHANNELS_STORED, payload) }) - this.storageService.on(StorageEvents.MESSAGES_STORED, (payload: MessagesLoadedPayload) => { + this.storageService.channels.on(StorageEvents.MESSAGES_STORED, (payload: MessagesLoadedPayload) => { this.serverIoProvider.io.emit(SocketActionTypes.MESSAGES_STORED, payload) }) - this.storageService.on(StorageEvents.MESSAGE_IDS_STORED, (payload: ChannelMessageIdsResponse) => { + this.storageService.channels.on(StorageEvents.MESSAGE_IDS_STORED, (payload: ChannelMessageIdsResponse) => { if (payload.ids.length === 0) { return } this.serverIoProvider.io.emit(SocketActionTypes.MESSAGE_IDS_STORED, payload) }) - this.storageService.on(StorageEvents.CHANNEL_SUBSCRIBED, (payload: ChannelSubscribedPayload) => { + this.storageService.channels.on(StorageEvents.CHANNEL_SUBSCRIBED, (payload: ChannelSubscribedPayload) => { this.serverIoProvider.io.emit(SocketActionTypes.CHANNEL_SUBSCRIBED, payload) }) - this.storageService.on(StorageEvents.REMOVE_DOWNLOAD_STATUS, (payload: RemoveDownloadStatus) => { + this.storageService.channels.on(StorageEvents.REMOVE_DOWNLOAD_STATUS, (payload: RemoveDownloadStatus) => { this.serverIoProvider.io.emit(SocketActionTypes.REMOVE_DOWNLOAD_STATUS, payload) }) - this.storageService.on(StorageEvents.FILE_UPLOADED, (payload: UploadFilePayload) => { + this.storageService.channels.on(StorageEvents.FILE_UPLOADED, (payload: UploadFilePayload) => { this.serverIoProvider.io.emit(SocketActionTypes.FILE_UPLOADED, payload) }) - this.storageService.on(StorageEvents.DOWNLOAD_PROGRESS, (payload: DownloadStatus) => { + this.storageService.channels.on(StorageEvents.DOWNLOAD_PROGRESS, (payload: DownloadStatus) => { this.serverIoProvider.io.emit(SocketActionTypes.DOWNLOAD_PROGRESS, payload) }) - this.storageService.on(StorageEvents.MESSAGE_MEDIA_UPDATED, (payload: FileMetadata) => { + this.storageService.channels.on(StorageEvents.MESSAGE_MEDIA_UPDATED, (payload: FileMetadata) => { this.serverIoProvider.io.emit(SocketActionTypes.MESSAGE_MEDIA_UPDATED, payload) }) - this.storageService.on(StorageEvents.COMMUNITY_UPDATED, (payload: Community) => { + this.storageService.channels.on(StorageEvents.COMMUNITY_UPDATED, (payload: Community) => { this.serverIoProvider.io.emit(SocketActionTypes.COMMUNITY_UPDATED, payload) }) - this.storageService.on(StorageEvents.SEND_PUSH_NOTIFICATION, (payload: PushNotificationPayload) => { + this.storageService.channels.on(StorageEvents.SEND_PUSH_NOTIFICATION, (payload: PushNotificationPayload) => { this.serverIoProvider.io.emit(SocketActionTypes.PUSH_NOTIFICATION, payload) }) this.storageService.on(StorageEvents.CSRS_STORED, async (payload: { csrs: string[] }) => { diff --git a/packages/backend/src/nest/storage/channels/channel.ts b/packages/backend/src/nest/storage/channels/channel.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/backend/src/nest/storage/channels/channels.service.ts b/packages/backend/src/nest/storage/channels/channels.service.ts new file mode 100644 index 0000000000..2d7acfecfe --- /dev/null +++ b/packages/backend/src/nest/storage/channels/channels.service.ts @@ -0,0 +1,484 @@ +import { Inject, Injectable } from '@nestjs/common' +import { keyObjectFromString, verifySignature } from '@quiet/identity' +import { type KeyValueType, type EventsType, IPFSAccessController, type LogEntry } from '@orbitdb/core' +import { EventEmitter } from 'events' +import { type PeerId } from '@libp2p/interface' +import { getCrypto } from 'pkijs' +import { stringToArrayBuffer } from 'pvutils' +import validate from '../../validation/validators' +import { + ChannelMessage, + ConnectionProcessInfo, + type CreateChannelResponse, + DeleteFilesFromChannelSocketPayload, + FileMetadata, + type MessagesLoadedPayload, + NoCryptoEngineError, + PublicChannel, + PushNotificationPayload, + SocketActionTypes, +} from '@quiet/types' +import fs from 'fs' +import { IpfsFileManagerService } from '../../ipfs-file-manager/ipfs-file-manager.service' +import { IPFS_REPO_PATCH, ORBIT_DB_DIR, QUIET_DIR } from '../../const' +import { IpfsFilesManagerEvents } from '../../ipfs-file-manager/ipfs-file-manager.types' +import { LocalDbService } from '../../local-db/local-db.service' +import { createLogger } from '../../common/logger' +import { PublicChannelsRepo } from '../../common/types' +import { DBOptions, StorageEvents } from '../storage.types' +import { CertificatesStore } from '../certificates/certificates.store' +import { OrbitDbService } from '../orbitDb/orbitDb.service' +import { KeyValueIndexedValidated } from '../orbitDb/keyValueIndexedValidated' +import { MessagesAccessController } from '../orbitDb/MessagesAccessController' +import { EventsWithStorage } from '../orbitDb/eventsWithStorage' + +@Injectable() +export class ChannelsService extends EventEmitter { + private peerId: PeerId | null = null + public publicChannelsRepos: Map = new Map() + private publicKeysMap: Map = new Map() + + private channels: KeyValueType | null + + private readonly logger = createLogger(ChannelsService.name) + + constructor( + @Inject(QUIET_DIR) public readonly quietDir: string, + @Inject(ORBIT_DB_DIR) public readonly orbitDbDir: string, + @Inject(IPFS_REPO_PATCH) public readonly ipfsRepoPath: string, + private readonly filesManager: IpfsFileManagerService, + private readonly localDbService: LocalDbService, + private readonly orbitDbService: OrbitDbService, + private readonly certificatesStore: CertificatesStore + ) { + super() + } + + // INITIALIZATION + + public async init(peerId: PeerId) { + this.logger.info(`Initializing ${ChannelsService.name}`) + this.peerId = peerId + + this.logger.info(`Starting file manager`) + this.attachFileManagerEvents() + await this.filesManager.init() + + this.logger.info(`Initializing Databases`) + await this.initChannels() + + this.logger.info(`Initialized ${ChannelsService.name}`) + } + + public async initChannels() { + this.logger.time(`Initializing channel databases`) + + this.attachFileManagerEvents() + await this.createDbForChannels() + await this.initAllChannels() + + this.logger.timeEnd('Initializing channel databases') + this.logger.info('Initialized databases') + } + + public async startSync() { + await this.channels?.sync.start() + for (const channel of this.publicChannelsRepos.values()) { + await channel.db.sync.start() + } + } + + public async setChannel(id: string, channel: PublicChannel) { + if (!this.channels) { + throw new Error('Channels have not been initialized!') + } + await this.channels.put(id, channel) + } + + public async getChannel(id: string) { + if (!this.channels) { + throw new Error('Channels have not been initialized!') + } + return await this.channels.get(id) + } + + public async getChannels(): Promise { + if (!this.channels) { + throw new Error('Channels have not been initialized!') + } + return (await this.channels.all()).map(x => x.value) + } + + public async loadAllChannels() { + this.logger.info('Getting all channels') + this.emit(StorageEvents.CHANNELS_STORED, { + channels: await this.getChannels(), + }) + } + + private async createDbForChannels() { + this.logger.info('Creating public-channels database') + this.channels = await this.orbitDbService.orbitDb.open>('public-channels', { + sync: false, + Database: KeyValueIndexedValidated(), + AccessController: IPFSAccessController({ write: ['*'] }), + }) + + this.channels.events.on('update', async (entry: LogEntry) => { + this.logger.info('public-channels database updated') + + this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.CHANNELS_STORED) + + const channels = await this.getChannels() + + this.emit(StorageEvents.CHANNELS_STORED, { channels }) + + channels.forEach(channel => this.subscribeToChannel(channel, { replicate: true })) + }) + + const channels = await this.getChannels() + this.logger.info('Channels count:', channels.length) + this.logger.info( + 'Channels names:', + channels.map(x => x.name) + ) + channels.forEach(channel => this.subscribeToChannel(channel)) + } + + async initAllChannels() { + this.emit(StorageEvents.CHANNELS_STORED, { + channels: await this.getChannels(), + }) + } + + async verifyMessage(message: ChannelMessage): Promise { + const crypto = getCrypto() + if (!crypto) throw new NoCryptoEngineError() + + const signature = stringToArrayBuffer(message.signature) + let cryptoKey = this.publicKeysMap.get(message.pubKey) + + if (!cryptoKey) { + cryptoKey = await keyObjectFromString(message.pubKey, crypto) + this.publicKeysMap.set(message.pubKey, cryptoKey) + } + + return await verifySignature(signature, message.message, cryptoKey) + } + + protected async getAllEventLogEntries(db: EventsType): Promise { + const res: T[] = [] + + for await (const x of db.iterator()) { + res.push(x.value) + } + + return res + } + + public async subscribeToChannel( + channelData: PublicChannel, + options = { replicate: false } + ): Promise { + let db: EventsType + // @ts-ignore + if (channelData.address) { + // @ts-ignore + channelData.id = channelData.address + } + let repo = this.publicChannelsRepos.get(channelData.id) + + if (repo) { + db = repo.db + } else { + try { + db = await this.createChannel(channelData, options) + } catch (e) { + this.logger.error(`Can't subscribe to channel ${channelData.id}`, e) + return + } + if (!db) { + this.logger.error(`Can't subscribe to channel ${channelData.id}, the DB isn't initialized!`) + return + } + repo = this.publicChannelsRepos.get(channelData.id) + } + + if (repo && !repo.eventsAttached) { + this.logger.info('Subscribing to channel ', channelData.id) + + db.events.on('update', async (entry: LogEntry) => { + this.logger.info(`${channelData.id} database updated`, entry.hash, entry.payload.value?.channelId) + + const message = entry.payload.value! + const verified = await this.verifyMessage(message) + + this.emit(StorageEvents.MESSAGES_STORED, { + messages: [message], + isVerified: verified, + }) + + const ids = (await this.getAllEventLogEntries(db)).map(msg => msg.id) + const community = await this.localDbService.getCurrentCommunity() + + if (community) { + this.emit(StorageEvents.MESSAGE_IDS_STORED, { + ids, + channelId: channelData.id, + communityId: community.id, + }) + } + + // FIXME: the 'update' event runs if we replicate entries and if we add + // entries ourselves. So we may want to check if the message is written + // by us. + // + // Display push notifications on mobile + if (process.env.BACKEND === 'mobile') { + if (!verified) return + + // Do not notify about old messages + if (message.createdAt < parseInt(process.env.CONNECTION_TIME || '')) return + + const username = await this.certificatesStore.getCertificateUsername(message.pubKey) + if (!username) { + this.logger.error(`Can't send push notification, no username found for public key '${message.pubKey}'`) + return + } + + const payload: PushNotificationPayload = { + message: JSON.stringify(message), + username: username, + } + + this.emit(StorageEvents.SEND_PUSH_NOTIFICATION, payload) + } + }) + + const ids = (await this.getAllEventLogEntries(db)).map(msg => msg.id) + const community = await this.localDbService.getCurrentCommunity() + + if (community) { + this.emit(StorageEvents.MESSAGE_IDS_STORED, { + ids, + channelId: channelData.id, + communityId: community.id, + }) + } + + repo.eventsAttached = true + } + + this.logger.info(`Subscribed to channel ${channelData.id}`) + this.emit(StorageEvents.CHANNEL_SUBSCRIBED, { + channelId: channelData.id, + }) + return { channel: channelData } + } + + public async getMessages(channelId: string, ids: string[]): Promise { + const repo = this.publicChannelsRepos.get(channelId) + if (!repo) return + + const messages = await this.getAllEventLogEntries(repo.db) + const filteredMessages: ChannelMessage[] = [] + + for (const id of ids) { + filteredMessages.push(...messages.filter(i => i.id === id)) + } + + return { + messages: filteredMessages, + isVerified: true, + } + } + + private async createChannel(channelData: PublicChannel, options: DBOptions): Promise> { + if (!validate.isChannel(channelData)) { + this.logger.error('Invalid channel format') + throw new Error('Create channel validation error') + } + + this.logger.info(`Creating channel ${channelData.id}`) + + const channelId = channelData.id + const db = await this.orbitDbService.orbitDb.open>(`channels.${channelId}`, { + type: 'events', + Database: EventsWithStorage(), + AccessController: MessagesAccessController({ write: ['*'] }), + }) + const channel = await this.getChannel(channelId) + + if (channel === undefined) { + await this.setChannel(channelId, channelData) + } else { + this.logger.info(`Channel ${channelId} already exists`) + } + + this.publicChannelsRepos.set(channelId, { db, eventsAttached: false }) + this.logger.info(`Set ${channelId} to local channels`) + this.logger.info(`Created channel ${channelId}`) + + return db + } + + public async deleteChannel(payload: { channelId: string; ownerPeerId: string }) { + this.logger.info('deleting channel storage', payload) + const { channelId, ownerPeerId } = payload + const channel = await this.getChannel(channelId) + if (!this.peerId) { + this.logger.error('deleteChannel - peerId is null') + throw new Error('deleteChannel - peerId is null') + } + const isOwner = ownerPeerId === this.peerId.toString() + if (channel && isOwner) { + if (!this.channels) { + throw new Error('Channels have not been initialized!') + } + await this.channels.del(channelId) + } + let repo = this.publicChannelsRepos.get(channelId) + if (!repo) { + const db = await this.orbitDbService.orbitDb.open>(`channels.${channelId}`, { + sync: false, + type: 'events', + Database: EventsWithStorage(), + AccessController: MessagesAccessController({ write: ['*'] }), + }) + repo = { + db, + eventsAttached: false, + } + } + await repo.db.sync.stop() + await repo.db.drop() + this.publicChannelsRepos.delete(channelId) + return { channelId: payload.channelId } + } + + public async deleteChannelFiles(files: FileMetadata[]) { + for (const file of files) { + await this.deleteFile(file) + } + } + + public async deleteFile(fileMetadata: FileMetadata) { + await this.filesManager.deleteBlocks(fileMetadata) + } + + public async sendMessage(message: ChannelMessage) { + if (!validate.isMessage(message)) { + this.logger.error('STORAGE: public channel message is invalid') + return + } + const repo = this.publicChannelsRepos.get(message.channelId) + if (!repo) { + this.logger.error(`Could not send message. No '${message.channelId}' channel in saved public channels`) + return + } + try { + this.logger.info('Sending message:', message.id) + await repo.db.add(message) + } catch (e) { + this.logger.error( + `STORAGE: Could not append message (entry not allowed to write to the log). Details: ${e.message}` + ) + } + } + + private attachFileManagerEvents = () => { + this.filesManager.on(IpfsFilesManagerEvents.DOWNLOAD_PROGRESS, status => { + this.emit(StorageEvents.DOWNLOAD_PROGRESS, status) + }) + this.filesManager.on(IpfsFilesManagerEvents.MESSAGE_MEDIA_UPDATED, messageMedia => { + this.emit(StorageEvents.MESSAGE_MEDIA_UPDATED, messageMedia) + }) + this.filesManager.on(StorageEvents.REMOVE_DOWNLOAD_STATUS, payload => { + this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, payload) + }) + this.filesManager.on(StorageEvents.FILE_UPLOADED, payload => { + this.emit(StorageEvents.FILE_UPLOADED, payload) + }) + this.filesManager.on(StorageEvents.DOWNLOAD_PROGRESS, payload => { + this.emit(StorageEvents.DOWNLOAD_PROGRESS, payload) + }) + this.filesManager.on(StorageEvents.MESSAGE_MEDIA_UPDATED, payload => { + this.emit(StorageEvents.MESSAGE_MEDIA_UPDATED, payload) + }) + } + + public async uploadFile(metadata: FileMetadata) { + this.filesManager.emit(IpfsFilesManagerEvents.UPLOAD_FILE, metadata) + } + + public async downloadFile(metadata: FileMetadata) { + this.filesManager.emit(IpfsFilesManagerEvents.DOWNLOAD_FILE, metadata) + } + + public cancelDownload(mid: string) { + this.filesManager.emit(IpfsFilesManagerEvents.CANCEL_DOWNLOAD, mid) + } + + public async deleteFilesFromChannel(payload: DeleteFilesFromChannelSocketPayload) { + const { messages } = payload + Object.keys(messages).map(async key => { + const message = messages[key] + if (message?.media?.path) { + const mediaPath = message.media.path + this.logger.info('deleteFilesFromChannel : mediaPath', mediaPath) + const isFileExist = await this.checkIfFileExist(mediaPath) + this.logger.info(`deleteFilesFromChannel : isFileExist- ${isFileExist}`) + if (isFileExist) { + fs.unlink(mediaPath, unlinkError => { + if (unlinkError) { + this.logger.error(`deleteFilesFromChannel : unlink error`, unlinkError) + } + }) + } else { + this.logger.error(`deleteFilesFromChannel : file does not exist`, mediaPath) + } + } + }) + } + + public async checkIfFileExist(filepath: string): Promise { + return await new Promise(resolve => { + fs.access(filepath, fs.constants.F_OK, error => { + resolve(!error) + }) + }) + } + + public async closeChannels(): Promise { + try { + this.logger.info('Closing channels DB') + await this.channels?.close() + this.logger.info('Closed channels DB') + } catch (e) { + this.logger.error('Error closing channels db', e) + } + } + + public async closeFileManager(): Promise { + try { + this.logger.info('Stopping IPFS files manager') + await this.filesManager.stop() + } catch (e) { + this.logger.error('Error stopping IPFS files manager', e) + } + } + + public async clean() { + this.peerId = null + + // @ts-ignore + this.channels = undefined + // @ts-ignore + this.messageThreads = undefined + // @ts-ignore + this.publicChannelsRepos = new Map() + this.publicKeysMap = new Map() + + this.channels = null + } +} diff --git a/packages/backend/src/nest/storage/storage.module.ts b/packages/backend/src/nest/storage/storage.module.ts index 78eb7cb09e..ed01e66bd5 100644 --- a/packages/backend/src/nest/storage/storage.module.ts +++ b/packages/backend/src/nest/storage/storage.module.ts @@ -8,6 +8,7 @@ import { CertificatesStore } from './certificates/certificates.store' import { CommunityMetadataStore } from './communityMetadata/communityMetadata.store' import { UserProfileStore } from './userProfile/userProfile.store' import { IpfsModule } from '../ipfs/ipfs.module' +import { ChannelsService } from './channels/channels.service' @Module({ imports: [LocalDbModule, IpfsModule, IpfsFileManagerModule], @@ -18,6 +19,7 @@ import { IpfsModule } from '../ipfs/ipfs.module' CommunityMetadataStore, CertificatesRequestsStore, UserProfileStore, + ChannelsService, ], exports: [StorageService], }) diff --git a/packages/backend/src/nest/storage/storage.service.spec.ts b/packages/backend/src/nest/storage/storage.service.spec.ts index 91622ea722..7da77a9707 100644 --- a/packages/backend/src/nest/storage/storage.service.spec.ts +++ b/packages/backend/src/nest/storage/storage.service.spec.ts @@ -1,401 +1,401 @@ -import { jest } from '@jest/globals' - -import { Test, TestingModule } from '@nestjs/testing' -import { keyFromCertificate, parseCertificate } from '@quiet/identity' -import { - prepareStore, - getFactory, - publicChannels, - generateMessageFactoryContentWithId, - Store, -} from '@quiet/state-manager' -import { - ChannelMessage, - Community, - FileMetadata, - Identity, - MessageType, - PublicChannel, - TestMessage, -} from '@quiet/types' - -import path from 'path' -import { type PeerId } from '@libp2p/interface' -import waitForExpect from 'wait-for-expect' -import { TestModule } from '../common/test.module' -import { createArbitraryFile, libp2pInstanceParams } from '../common/utils' -import { IpfsModule } from '../ipfs/ipfs.module' -import { IpfsService } from '../ipfs/ipfs.service' -import { Libp2pModule } from '../libp2p/libp2p.module' -import { Libp2pService } from '../libp2p/libp2p.service' -import { SocketModule } from '../socket/socket.module' -import { StorageModule } from './storage.module' -import { StorageService } from './storage.service' -import fs from 'fs' -import { type FactoryGirl } from 'factory-girl' -import { fileURLToPath } from 'url' -import { LocalDbModule } from '../local-db/local-db.module' -import { LocalDbService } from '../local-db/local-db.service' -import { ORBIT_DB_DIR } from '../const' -import { createLogger } from '../common/logger' -import { createUserCertificateTestHelper, createTestRootCA } from '@quiet/identity' - -const logger = createLogger('storageService:test') - -const filename = fileURLToPath(import.meta.url) -const dirname = path.dirname(filename) - -describe('StorageService', () => { - let module: TestingModule - let storageService: StorageService - let ipfsService: IpfsService - let libp2pService: Libp2pService - let localDbService: LocalDbService - let peerId: PeerId - - let store: Store - let factory: FactoryGirl - let community: Community - let channel: PublicChannel - let alice: Identity - let john: Identity - let message: ChannelMessage - let channelio: PublicChannel - let filePath: string - let utils: any - let orbitDbDir: string - - jest.setTimeout(50000) - - beforeAll(async () => { - store = prepareStore().store - factory = await getFactory(store) - - community = await factory.create('Community') - - channel = publicChannels.selectors.publicChannels(store.getState())[0] - - channelio = { - name: channel.name, - description: channel.description, - owner: channel.owner, - timestamp: channel.timestamp, - id: channel.id, - } - - alice = await factory.create('Identity', { id: community.id, nickname: 'alice' }) - - john = await factory.create('Identity', { id: community.id, nickname: 'john' }) - - message = ( - await factory.create('Message', { - identity: alice, - message: generateMessageFactoryContentWithId(channel.id), - }) - ).message - }) - - beforeEach(async () => { - jest.clearAllMocks() - utils = await import('../common/utils') - filePath = path.join(dirname, '/500kB-file.txt') - - module = await Test.createTestingModule({ - imports: [TestModule, StorageModule, IpfsModule, SocketModule, Libp2pModule, LocalDbModule], - }).compile() - - storageService = await module.resolve(StorageService) - localDbService = await module.resolve(LocalDbService) - libp2pService = await module.resolve(Libp2pService) - ipfsService = await module.resolve(IpfsService) - - orbitDbDir = await module.resolve(ORBIT_DB_DIR) - - const params = await libp2pInstanceParams() - peerId = params.peerId.peerId - - await libp2pService.createInstance(params) - expect(libp2pService.libp2pInstance).not.toBeNull() - - await localDbService.open() - expect(localDbService.getStatus()).toEqual('open') - - await localDbService.setCommunity(community) - await localDbService.setCurrentCommunityId(community.id) - }) - - afterEach(async () => { - await libp2pService.libp2pInstance?.stop() - await ipfsService.ipfsInstance?.stop() - await storageService.stop() - if (fs.existsSync(filePath)) { - fs.rmSync(filePath) - } - await module.close() - }) - - it('should be defined', async () => { - await storageService.init(peerId) - }) - - describe('Storage', () => { - it('should not create paths if createPaths is set to false', async () => { - const orgProcessPlatform = process.platform - Object.defineProperty(process, 'platform', { - value: 'android', - }) - expect(fs.existsSync(orbitDbDir)).toBe(false) - - // FIXME: throws TypeError: Cannot assign to read only property 'createPaths' of object '[object Module]' and I can't be bothered to figure out how to get it to work - // const createPathsSpy = jest.spyOn(utils, 'createPaths') - - await storageService.init(peerId) - - // FIXME: throws TypeError: Cannot assign to read only property 'createPaths' of object '[object Module]' and I can't be bothered to figure out how to get it to work - // expect(createPathsSpy).not.toHaveBeenCalled() - - Object.defineProperty(process, 'platform', { - value: orgProcessPlatform, - }) - }) - - it('db address should be the same on all platforms', () => { - const dbAddress = StorageService.dbAddress({ root: 'zdpuABCDefgh123', path: 'channels.general_abcd' }) - expect(dbAddress).toEqual(`/orbitdb/zdpuABCDefgh123/channels.general_abcd`) - }) - }) - - describe('Channels', () => { - it('deletes channel as owner', async () => { - await storageService.init(peerId) - await storageService.subscribeToChannel(channelio) - - const result = await storageService.deleteChannel({ channelId: channelio.id, ownerPeerId: peerId.toString() }) - expect(result).toEqual({ channelId: channelio.id }) - - const channelFromKeyValueStore = (await storageService.getChannels()).filter(x => x.id === channelio.id) - expect(channelFromKeyValueStore).toEqual([]) - }) - - it('delete channel as standard user', async () => { - await storageService.init(peerId) - await storageService.subscribeToChannel(channelio) - - const result = await storageService.deleteChannel({ channelId: channelio.id, ownerPeerId: 'random peer id' }) - expect(result).toEqual({ channelId: channelio.id }) - - const channelFromKeyValueStore = (await storageService.getChannels()).filter(x => x.id === channelio.id) - expect(channelFromKeyValueStore).toEqual([channelio]) - }) - }) - - describe('Message access controller', () => { - it('is saved to db if passed signature verification', async () => { - await storageService.init(peerId) - - await storageService.subscribeToChannel(channelio) - - const publicChannelRepo = storageService.publicChannelsRepos.get(message.channelId) - expect(publicChannelRepo).not.toBeUndefined() - // @ts-expect-error - const db = publicChannelRepo.db - const eventSpy = jest.spyOn(db, 'add') - - const messageCopy = { - ...message, - } - delete messageCopy.media - - await storageService.sendMessage(messageCopy) - - // Confirm message has passed orbitdb validator (check signature verification only) - expect(eventSpy).toHaveBeenCalled() - // @ts-expect-error - const savedMessages = await storageService.getAllEventLogEntries(db) - expect(savedMessages.length).toBe(1) - expect(savedMessages[0]).toEqual(messageCopy) - }) - - it('is not saved to db if did not pass signature verification', async () => { - const aliceMessage = await factory.create['payload']>( - 'Message', - { - identity: alice, - message: generateMessageFactoryContentWithId(channel.id), - } - ) - // @ts-expect-error userCertificate can be undefined - const johnCertificate: string = john.userCertificate - const johnPublicKey = keyFromCertificate(parseCertificate(johnCertificate)) - - const spoofedMessage = { - ...aliceMessage.message, - channelId: channelio.id, - pubKey: johnPublicKey, - } - delete spoofedMessage.media // Media 'undefined' is not accepted by db.add - - await storageService.init(peerId) - - await storageService.subscribeToChannel(channelio) - - const publicChannelRepo = storageService.publicChannelsRepos.get(message.channelId) - expect(publicChannelRepo).not.toBeUndefined() - // @ts-expect-error - const db = publicChannelRepo.db - const eventSpy = jest.spyOn(db, 'add') - - await storageService.sendMessage(spoofedMessage) - - // Confirm message has passed orbitdb validator (check signature verification only) - expect(eventSpy).toHaveBeenCalled() - // @ts-expect-error getAllEventLogEntries is protected - expect((await storageService.getAllEventLogEntries(db)).length).toBe(0) - }) - }) - - describe('Users', () => { - it('gets all users from db', async () => { - const expected = [ - { - onionAddress: 'zghidexs7qt24ivu3jobjqdtzzwtyau4lppnunx5pkif76pkpyp7qcid.onion', - peerId: '12D3KooWKCWstmqi5gaQvipT7xVneVGfWV7HYpCbmUu626R92hXx', - username: 'b', - }, - { - onionAddress: 'nhliujn66e346evvvxeaf4qma7pqshcgbu6t7wseoannc2fy4cnscryd.onion', - peerId: '12D3KooWCXzUw71ovvkDky6XkV57aCWUV9JhJoKhoqXa1gdhFNoL', - username: 'c', - }, - { - onionAddress: '6vu2bxki777it3cpayv6fq6vpl4ke3kzj7gxicfygm55dhhtphyfdvyd.onion', - peerId: '12D3KooWEHzmff5kZAvyU6Diq5uJG8QkWJxFNUcBLuWjxUGvxaqw', - username: 'o', - }, - { - onionAddress: 'y7yczmugl2tekami7sbdz5pfaemvx7bahwthrdvcbzw5vex2crsr26qd.onion', - peerId: '12D3KooWHgLdRMqkepNiYnrur21cyASUNk1f9NZ5tuGa9He8QXNa', - username: 'o', - }, - ] - - const certs: string[] = [] - const csrs: string[] = [] - const rootCA = await createTestRootCA() - for (const userData of expected) { - const { userCsr, userCert } = await createUserCertificateTestHelper( - { nickname: userData.username, commonName: userData.onionAddress, peerId: userData.peerId }, - rootCA - ) - if (['b', 'c'].includes(userData.username)) { - certs.push(userCert!.userCertString) - } - if (['c', 'o'].includes(userData.username)) { - csrs.push(userCsr.userCsr) - } - } - - // const certs = [ - // // b - // 'MIICITCCAcegAwIBAgIGAY8GkBEVMAoGCCqGSM49BAMCMAwxCjAIBgNVBAMTAWEwHhcNMjQwNDIyMTYwNzM1WhcNMzAwMjAxMDcwMDAwWjBJMUcwRQYDVQQDEz56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDG8SNnoS1BYoV72jcyQFVlsrwvd2Bb9/9L13Tc4SHJwitTUB3F+y/7pk0tAPrZi2qasU2PO9lTwUxXYcAfpCRSjgdcwgdQwCQYDVR0TBAIwADALBgNVHQ8EBAMCAIAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBEGCisGAQQBg4wbAgEEAxMBYjA9BgkrBgECAQ8DAQEEMBMuUW1lUGJCMjVoMWZYN1dBRk42ckZSNGFWRFdVRlFNU3RSSEdERFM0UlFaUTRZcTBJBgNVHREEQjBAgj56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiBkTZo6/D0YgNMPcDpuf7n+rDEQls6cMVxEVw/H8vxbhwIhAM+e6we9YP4JeNgOGgd0iZNEpq8N7dla4XO+YVWrh0YG', - - // // c - // 'MIICITCCAcegAwIBAgIGAY8Glf+pMAoGCCqGSM49BAMCMAwxCjAIBgNVBAMTAWEwHhcNMjQwNDIyMTYxNDA0WhcNMzAwMjAxMDcwMDAwWjBJMUcwRQYDVQQDEz5uaGxpdWpuNjZlMzQ2ZXZ2dnhlYWY0cW1hN3Bxc2hjZ2J1NnQ3d3Nlb2FubmMyZnk0Y25zY3J5ZC5vbmlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP1WBKQdMz5yMpv5hWj6j+auIsnfiJE8dtuxeeM4N03K1An61F0o47CWg04DydwmoPn5gwefEv8t9Cz9nv/VUGejgdcwgdQwCQYDVR0TBAIwADALBgNVHQ8EBAMCAIAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBEGCisGAQQBg4wbAgEEAxMBYzA9BgkrBgECAQ8DAQEEMBMuUW1WY1hRTXVmRWNZS0R0d3NFSlRIUGJzc3BCeU02U0hUYlJHR2VEdkVFdU1RQTBJBgNVHREEQjBAgj5uaGxpdWpuNjZlMzQ2ZXZ2dnhlYWY0cW1hN3Bxc2hjZ2J1NnQ3d3Nlb2FubmMyZnk0Y25zY3J5ZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiEAgMCBxF3oK4ituEWcAK6uawMCludZu4YujIpBIR+v2LICIBhMHXrBy1KWc70t6idB+5XkInsRZz5nw1vwgRJ4mw98', - // ] - - // const csrs = [ - // // c - // 'MIIB4TCCAYgCAQAwSTFHMEUGA1UEAxM+emdoaWRleHM3cXQyNGl2dTNqb2JqcWR0enp3dHlhdTRscHBudW54NXBraWY3NnBrcHlwN3FjaWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQxvEjZ6EtQWKFe9o3MkBVZbK8L3dgW/f/S9d03OEhycIrU1Adxfsv+6ZNLQD62YtqmrFNjzvZU8FMV2HAH6QkUoIHcMC4GCSqGSIb3DQEJDjEhMB8wHQYDVR0OBBYEFG1W6vJTK/uPuRK2LPaVZyebVVc+MA8GCSqGSIb3DQEJDDECBAAwEQYKKwYBBAGDjBsCATEDEwFiMD0GCSsGAQIBDwMBATEwEy5RbWVQYkIyNWgxZlg3V0FGTjZyRlI0YVZEV1VGUU1TdFJIR0REUzRSUVpRNFlxMEcGA1UdETFAEz56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjAKBggqhkjOPQQDAgNHADBEAiAjxneoJZtCzkd75HTT+pcj+objG3S04omjeMMw1N+B/wIgAaJRgifnWEnWFYm614UmPw9un2Uwk1gVhN2tSwJ65sM=', - - // // o - // 'MIIDHjCCAsMCAQAwSTFHMEUGA1UEAxM+NnZ1MmJ4a2k3NzdpdDNjcGF5djZmcTZ2cGw0a2Uza3pqN2d4aWNmeWdtNTVkaGh0cGh5ZmR2eWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATMpfp2hSfWFL26OZlZKZEWG9fyAM1ndlEzO0kLxT0pA/7/fs+a5X/s4TkzqCVVQSzhas/84q0WE99ScAcM1LQJoIICFjAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBR6VRzktP1pzZxsGUaJivNUrtgSrzCCAUcGCSqGSIb3DQEJDDGCATgEggE0KZq9s6HEViRfplVgYkulg6XV411ZRe4U1UjfXTf1pRaygfcenGbT6RRagPtZzjuq5hHdYhqDjRzZhnbn8ZASYTgBM7qcseUq5UpS1pE08DI2jePKqatp3Pzm6a/MGSziESnREx784JlKfwKMjJl33UA8lQm9nhSeAIHyBx3c4Lf8IXdW2n3rnhbVfjpBMAxwh6lt+e5agtGXy+q/xAESUeLPfUgRYWctlLgt8Op+WTpLyBkZsVFoBvJrMt2XdM0RI32YzTRr56GXFa4VyQmY5xXwlQSPgidAP7jPkVygNcoeXvAz2ZCk3IR1Cn3mX8nMko53MlDNaMYldUQA0ug28/S7BlSlaq2CDD4Ol3swTq7C4KGTxKrI36ruYUZx7NEaQDF5V7VvqPCZ0fZoTIJuSYTQ67gwEQYKKwYBBAGDjBsCATEDEwFvMD0GCSsGAQIBDwMBATEwEy5RbVhSWTRyaEF4OE11cThkTUdrcjlxa25KZEU2VUhaRGRHYURSVFFFYndGTjViMEcGA1UdETFAEz42dnUyYnhraTc3N2l0M2NwYXl2NmZxNnZwbDRrZTNremo3Z3hpY2Z5Z201NWRoaHRwaHlmZHZ5ZC5vbmlvbjAKBggqhkjOPQQDAgNJADBGAiEAt+f1u/bchg5AZHv6NTGNoXeejTRWUhX3ioGwW6TGg84CIQCHqKNzDh2JjS/hUHx5PApAmfNnQTSf19X6LnNHQweU1g==', - - // // o - // 'MIIDHTCCAsMCAQAwSTFHMEUGA1UEAxM+eTd5Y3ptdWdsMnRla2FtaTdzYmR6NXBmYWVtdng3YmFod3RocmR2Y2J6dzV2ZXgyY3JzcjI2cWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATMq0l4bCmjdb0grtzpwtDVLM9E1IQpL9vrB4+lD9OBZzlrx2365jV7shVu9utas8w8fxtKoBZSnT5+32ZMFTB4oIICFjAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBSoDQpTZdEvi1/Rr/muVXT1clyKRDCCAUcGCSqGSIb3DQEJDDGCATgEggE0BQvyvkiiXEf/PLKnsR1Ba9AhYsVO8o56bnftUnoVzBlRZgUzLJvOSroPk/EmbVz+okhMrcYNgCWHvxrAqHVVq0JRP6bi98BtCUotx6OPFHp5K5QCL60hod1uAnhKocyJG9tsoM9aS+krn/k+g4RCBjiPZ25cC7QG/UNr6wyIQ8elBho4MKm8iOp7EShSsZOV1f6xrnXYCC/zyUc85GEuycLzVImgAQvPATbdMzY4zSGnNLHxkvSUNxaR9LnEWf+i1jeqcOiXOvmdyU5Be3ZqhGKvvBg/5vyLQiCIfeapjZemnLqFHQBitglDm2xnKL6HzMyfZoAHPV7YcWYR4spU9Ju8Q8aqSeAryx7sx55eSR4GO5UQTo5DrQn6xtkwOZ/ytsOknFthF8jcA9uTAMDKA2TylCUwEQYKKwYBBAGDjBsCATEDEwFvMD0GCSsGAQIBDwMBATEwEy5RbVQxOFV2blVCa3NlTWMzU3FuZlB4cEh3TjhuekxySmVOU0xadGM4ckFGWGh6MEcGA1UdETFAEz55N3ljem11Z2wydGVrYW1pN3NiZHo1cGZhZW12eDdiYWh3dGhyZHZjYnp3NXZleDJjcnNyMjZxZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiEAoFrAglxmk7ciD6AHQOB1qEoLu0NARcxgwmIry8oeTHwCICyXp5NJQ9Z8vReIAQNng2H2+/XjHifZEWzhoN0VkcBx', - // ] - - await storageService.init(peerId) - // @ts-ignore - storageService.certificatesRequestsStore = { - getEntries: jest.fn(() => { - return csrs - }), - } - // @ts-ignore - storageService.certificatesStore = { - getEntries: jest.fn(() => { - return certs - }), - } - - const allUsers = await storageService.getAllUsers() - - expect(allUsers).toStrictEqual(expected) - }) - }) - - describe('Files deletion', () => { - let realFilePath: string - let messages: { - messages: Record - } - - beforeEach(async () => { - realFilePath = path.join(dirname, '/real-file.txt') - createArbitraryFile(realFilePath, 2147483) - await storageService.init(peerId) - - const metadata: FileMetadata = { - path: realFilePath, - name: 'test-large-file', - ext: '.txt', - cid: 'uploading_id', - message: { - id: 'id', - channelId: channel.id, - }, - } - - const aliceMessage = await factory.create['payload']>( - 'Message', - { - identity: alice, - message: generateMessageFactoryContentWithId(channel.id, MessageType.File, metadata), - } - ) - - messages = { - messages: { - [aliceMessage.message.id]: aliceMessage.message, - }, - } - }) - - afterEach(() => { - if (fs.existsSync(realFilePath)) { - fs.rmSync(realFilePath) - } - }) - - it('delete file correctly', async () => { - const isFileExist = await storageService.checkIfFileExist(realFilePath) - expect(isFileExist).toBeTruthy() - - await expect(storageService.deleteFilesFromChannel(messages)).resolves.not.toThrowError() - - await waitForExpect(async () => { - expect(await storageService.checkIfFileExist(realFilePath)).toBeFalsy() - }, 2000) - }) - - it('file dont exist - not throw error', async () => { - fs.rmSync(realFilePath) - - await waitForExpect(async () => { - expect(await storageService.checkIfFileExist(realFilePath)).toBeFalsy() - }, 2000) - - await expect(storageService.deleteFilesFromChannel(messages)).resolves.not.toThrowError() - }) - }) -}) +// import { jest } from '@jest/globals' + +// import { Test, TestingModule } from '@nestjs/testing' +// import { keyFromCertificate, parseCertificate } from '@quiet/identity' +// import { +// prepareStore, +// getFactory, +// publicChannels, +// generateMessageFactoryContentWithId, +// Store, +// } from '@quiet/state-manager' +// import { +// ChannelMessage, +// Community, +// FileMetadata, +// Identity, +// MessageType, +// PublicChannel, +// TestMessage, +// } from '@quiet/types' + +// import path from 'path' +// import { type PeerId } from '@libp2p/interface' +// import waitForExpect from 'wait-for-expect' +// import { TestModule } from '../common/test.module' +// import { createArbitraryFile, libp2pInstanceParams } from '../common/utils' +// import { IpfsModule } from '../ipfs/ipfs.module' +// import { IpfsService } from '../ipfs/ipfs.service' +// import { Libp2pModule } from '../libp2p/libp2p.module' +// import { Libp2pService } from '../libp2p/libp2p.service' +// import { SocketModule } from '../socket/socket.module' +// import { StorageModule } from './storage.module' +// import { StorageService } from './storage.service' +// import fs from 'fs' +// import { type FactoryGirl } from 'factory-girl' +// import { fileURLToPath } from 'url' +// import { LocalDbModule } from '../local-db/local-db.module' +// import { LocalDbService } from '../local-db/local-db.service' +// import { ORBIT_DB_DIR } from '../const' +// import { createLogger } from '../common/logger' +// import { createUserCertificateTestHelper, createTestRootCA } from '@quiet/identity' + +// const logger = createLogger('storageService:test') + +// const filename = fileURLToPath(import.meta.url) +// const dirname = path.dirname(filename) + +// describe('StorageService', () => { +// let module: TestingModule +// let storageService: StorageService +// let ipfsService: IpfsService +// let libp2pService: Libp2pService +// let localDbService: LocalDbService +// let peerId: PeerId + +// let store: Store +// let factory: FactoryGirl +// let community: Community +// let channel: PublicChannel +// let alice: Identity +// let john: Identity +// let message: ChannelMessage +// let channelio: PublicChannel +// let filePath: string +// let utils: any +// let orbitDbDir: string + +// jest.setTimeout(50000) + +// beforeAll(async () => { +// store = prepareStore().store +// factory = await getFactory(store) + +// community = await factory.create('Community') + +// channel = publicChannels.selectors.publicChannels(store.getState())[0] + +// channelio = { +// name: channel.name, +// description: channel.description, +// owner: channel.owner, +// timestamp: channel.timestamp, +// id: channel.id, +// } + +// alice = await factory.create('Identity', { id: community.id, nickname: 'alice' }) + +// john = await factory.create('Identity', { id: community.id, nickname: 'john' }) + +// message = ( +// await factory.create('Message', { +// identity: alice, +// message: generateMessageFactoryContentWithId(channel.id), +// }) +// ).message +// }) + +// beforeEach(async () => { +// jest.clearAllMocks() +// utils = await import('../common/utils') +// filePath = path.join(dirname, '/500kB-file.txt') + +// module = await Test.createTestingModule({ +// imports: [TestModule, StorageModule, IpfsModule, SocketModule, Libp2pModule, LocalDbModule], +// }).compile() + +// storageService = await module.resolve(StorageService) +// localDbService = await module.resolve(LocalDbService) +// libp2pService = await module.resolve(Libp2pService) +// ipfsService = await module.resolve(IpfsService) + +// orbitDbDir = await module.resolve(ORBIT_DB_DIR) + +// const params = await libp2pInstanceParams() +// peerId = params.peerId.peerId + +// await libp2pService.createInstance(params) +// expect(libp2pService.libp2pInstance).not.toBeNull() + +// await localDbService.open() +// expect(localDbService.getStatus()).toEqual('open') + +// await localDbService.setCommunity(community) +// await localDbService.setCurrentCommunityId(community.id) +// }) + +// afterEach(async () => { +// await libp2pService.libp2pInstance?.stop() +// await ipfsService.ipfsInstance?.stop() +// await storageService.stop() +// if (fs.existsSync(filePath)) { +// fs.rmSync(filePath) +// } +// await module.close() +// }) + +// it('should be defined', async () => { +// await storageService.init(peerId) +// }) + +// describe('Storage', () => { +// it('should not create paths if createPaths is set to false', async () => { +// const orgProcessPlatform = process.platform +// Object.defineProperty(process, 'platform', { +// value: 'android', +// }) +// expect(fs.existsSync(orbitDbDir)).toBe(false) + +// // FIXME: throws TypeError: Cannot assign to read only property 'createPaths' of object '[object Module]' and I can't be bothered to figure out how to get it to work +// // const createPathsSpy = jest.spyOn(utils, 'createPaths') + +// await storageService.init(peerId) + +// // FIXME: throws TypeError: Cannot assign to read only property 'createPaths' of object '[object Module]' and I can't be bothered to figure out how to get it to work +// // expect(createPathsSpy).not.toHaveBeenCalled() + +// Object.defineProperty(process, 'platform', { +// value: orgProcessPlatform, +// }) +// }) + +// it('db address should be the same on all platforms', () => { +// const dbAddress = StorageService.dbAddress({ root: 'zdpuABCDefgh123', path: 'channels.general_abcd' }) +// expect(dbAddress).toEqual(`/orbitdb/zdpuABCDefgh123/channels.general_abcd`) +// }) +// }) + +// describe('Channels', () => { +// it('deletes channel as owner', async () => { +// await storageService.init(peerId) +// await storageService.subscribeToChannel(channelio) + +// const result = await storageService.deleteChannel({ channelId: channelio.id, ownerPeerId: peerId.toString() }) +// expect(result).toEqual({ channelId: channelio.id }) + +// const channelFromKeyValueStore = (await storageService.getChannels()).filter(x => x.id === channelio.id) +// expect(channelFromKeyValueStore).toEqual([]) +// }) + +// it('delete channel as standard user', async () => { +// await storageService.init(peerId) +// await storageService.subscribeToChannel(channelio) + +// const result = await storageService.deleteChannel({ channelId: channelio.id, ownerPeerId: 'random peer id' }) +// expect(result).toEqual({ channelId: channelio.id }) + +// const channelFromKeyValueStore = (await storageService.getChannels()).filter(x => x.id === channelio.id) +// expect(channelFromKeyValueStore).toEqual([channelio]) +// }) +// }) + +// describe('Message access controller', () => { +// it('is saved to db if passed signature verification', async () => { +// await storageService.init(peerId) + +// await storageService.subscribeToChannel(channelio) + +// const publicChannelRepo = storageService.publicChannelsRepos.get(message.channelId) +// expect(publicChannelRepo).not.toBeUndefined() +// // @ts-expect-error +// const db = publicChannelRepo.db +// const eventSpy = jest.spyOn(db, 'add') + +// const messageCopy = { +// ...message, +// } +// delete messageCopy.media + +// await storageService.sendMessage(messageCopy) + +// // Confirm message has passed orbitdb validator (check signature verification only) +// expect(eventSpy).toHaveBeenCalled() +// // @ts-expect-error +// const savedMessages = await storageService.getAllEventLogEntries(db) +// expect(savedMessages.length).toBe(1) +// expect(savedMessages[0]).toEqual(messageCopy) +// }) + +// it('is not saved to db if did not pass signature verification', async () => { +// const aliceMessage = await factory.create['payload']>( +// 'Message', +// { +// identity: alice, +// message: generateMessageFactoryContentWithId(channel.id), +// } +// ) +// // @ts-expect-error userCertificate can be undefined +// const johnCertificate: string = john.userCertificate +// const johnPublicKey = keyFromCertificate(parseCertificate(johnCertificate)) + +// const spoofedMessage = { +// ...aliceMessage.message, +// channelId: channelio.id, +// pubKey: johnPublicKey, +// } +// delete spoofedMessage.media // Media 'undefined' is not accepted by db.add + +// await storageService.init(peerId) + +// await storageService.subscribeToChannel(channelio) + +// const publicChannelRepo = storageService.publicChannelsRepos.get(message.channelId) +// expect(publicChannelRepo).not.toBeUndefined() +// // @ts-expect-error +// const db = publicChannelRepo.db +// const eventSpy = jest.spyOn(db, 'add') + +// await storageService.sendMessage(spoofedMessage) + +// // Confirm message has passed orbitdb validator (check signature verification only) +// expect(eventSpy).toHaveBeenCalled() +// // @ts-expect-error getAllEventLogEntries is protected +// expect((await storageService.getAllEventLogEntries(db)).length).toBe(0) +// }) +// }) + +// describe('Users', () => { +// it('gets all users from db', async () => { +// const expected = [ +// { +// onionAddress: 'zghidexs7qt24ivu3jobjqdtzzwtyau4lppnunx5pkif76pkpyp7qcid.onion', +// peerId: '12D3KooWKCWstmqi5gaQvipT7xVneVGfWV7HYpCbmUu626R92hXx', +// username: 'b', +// }, +// { +// onionAddress: 'nhliujn66e346evvvxeaf4qma7pqshcgbu6t7wseoannc2fy4cnscryd.onion', +// peerId: '12D3KooWCXzUw71ovvkDky6XkV57aCWUV9JhJoKhoqXa1gdhFNoL', +// username: 'c', +// }, +// { +// onionAddress: '6vu2bxki777it3cpayv6fq6vpl4ke3kzj7gxicfygm55dhhtphyfdvyd.onion', +// peerId: '12D3KooWEHzmff5kZAvyU6Diq5uJG8QkWJxFNUcBLuWjxUGvxaqw', +// username: 'o', +// }, +// { +// onionAddress: 'y7yczmugl2tekami7sbdz5pfaemvx7bahwthrdvcbzw5vex2crsr26qd.onion', +// peerId: '12D3KooWHgLdRMqkepNiYnrur21cyASUNk1f9NZ5tuGa9He8QXNa', +// username: 'o', +// }, +// ] + +// const certs: string[] = [] +// const csrs: string[] = [] +// const rootCA = await createTestRootCA() +// for (const userData of expected) { +// const { userCsr, userCert } = await createUserCertificateTestHelper( +// { nickname: userData.username, commonName: userData.onionAddress, peerId: userData.peerId }, +// rootCA +// ) +// if (['b', 'c'].includes(userData.username)) { +// certs.push(userCert!.userCertString) +// } +// if (['c', 'o'].includes(userData.username)) { +// csrs.push(userCsr.userCsr) +// } +// } + +// // const certs = [ +// // // b +// // 'MIICITCCAcegAwIBAgIGAY8GkBEVMAoGCCqGSM49BAMCMAwxCjAIBgNVBAMTAWEwHhcNMjQwNDIyMTYwNzM1WhcNMzAwMjAxMDcwMDAwWjBJMUcwRQYDVQQDEz56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDG8SNnoS1BYoV72jcyQFVlsrwvd2Bb9/9L13Tc4SHJwitTUB3F+y/7pk0tAPrZi2qasU2PO9lTwUxXYcAfpCRSjgdcwgdQwCQYDVR0TBAIwADALBgNVHQ8EBAMCAIAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBEGCisGAQQBg4wbAgEEAxMBYjA9BgkrBgECAQ8DAQEEMBMuUW1lUGJCMjVoMWZYN1dBRk42ckZSNGFWRFdVRlFNU3RSSEdERFM0UlFaUTRZcTBJBgNVHREEQjBAgj56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiBkTZo6/D0YgNMPcDpuf7n+rDEQls6cMVxEVw/H8vxbhwIhAM+e6we9YP4JeNgOGgd0iZNEpq8N7dla4XO+YVWrh0YG', + +// // // c +// // 'MIICITCCAcegAwIBAgIGAY8Glf+pMAoGCCqGSM49BAMCMAwxCjAIBgNVBAMTAWEwHhcNMjQwNDIyMTYxNDA0WhcNMzAwMjAxMDcwMDAwWjBJMUcwRQYDVQQDEz5uaGxpdWpuNjZlMzQ2ZXZ2dnhlYWY0cW1hN3Bxc2hjZ2J1NnQ3d3Nlb2FubmMyZnk0Y25zY3J5ZC5vbmlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP1WBKQdMz5yMpv5hWj6j+auIsnfiJE8dtuxeeM4N03K1An61F0o47CWg04DydwmoPn5gwefEv8t9Cz9nv/VUGejgdcwgdQwCQYDVR0TBAIwADALBgNVHQ8EBAMCAIAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBEGCisGAQQBg4wbAgEEAxMBYzA9BgkrBgECAQ8DAQEEMBMuUW1WY1hRTXVmRWNZS0R0d3NFSlRIUGJzc3BCeU02U0hUYlJHR2VEdkVFdU1RQTBJBgNVHREEQjBAgj5uaGxpdWpuNjZlMzQ2ZXZ2dnhlYWY0cW1hN3Bxc2hjZ2J1NnQ3d3Nlb2FubmMyZnk0Y25zY3J5ZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiEAgMCBxF3oK4ituEWcAK6uawMCludZu4YujIpBIR+v2LICIBhMHXrBy1KWc70t6idB+5XkInsRZz5nw1vwgRJ4mw98', +// // ] + +// // const csrs = [ +// // // c +// // 'MIIB4TCCAYgCAQAwSTFHMEUGA1UEAxM+emdoaWRleHM3cXQyNGl2dTNqb2JqcWR0enp3dHlhdTRscHBudW54NXBraWY3NnBrcHlwN3FjaWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQxvEjZ6EtQWKFe9o3MkBVZbK8L3dgW/f/S9d03OEhycIrU1Adxfsv+6ZNLQD62YtqmrFNjzvZU8FMV2HAH6QkUoIHcMC4GCSqGSIb3DQEJDjEhMB8wHQYDVR0OBBYEFG1W6vJTK/uPuRK2LPaVZyebVVc+MA8GCSqGSIb3DQEJDDECBAAwEQYKKwYBBAGDjBsCATEDEwFiMD0GCSsGAQIBDwMBATEwEy5RbWVQYkIyNWgxZlg3V0FGTjZyRlI0YVZEV1VGUU1TdFJIR0REUzRSUVpRNFlxMEcGA1UdETFAEz56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjAKBggqhkjOPQQDAgNHADBEAiAjxneoJZtCzkd75HTT+pcj+objG3S04omjeMMw1N+B/wIgAaJRgifnWEnWFYm614UmPw9un2Uwk1gVhN2tSwJ65sM=', + +// // // o +// // 'MIIDHjCCAsMCAQAwSTFHMEUGA1UEAxM+NnZ1MmJ4a2k3NzdpdDNjcGF5djZmcTZ2cGw0a2Uza3pqN2d4aWNmeWdtNTVkaGh0cGh5ZmR2eWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATMpfp2hSfWFL26OZlZKZEWG9fyAM1ndlEzO0kLxT0pA/7/fs+a5X/s4TkzqCVVQSzhas/84q0WE99ScAcM1LQJoIICFjAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBR6VRzktP1pzZxsGUaJivNUrtgSrzCCAUcGCSqGSIb3DQEJDDGCATgEggE0KZq9s6HEViRfplVgYkulg6XV411ZRe4U1UjfXTf1pRaygfcenGbT6RRagPtZzjuq5hHdYhqDjRzZhnbn8ZASYTgBM7qcseUq5UpS1pE08DI2jePKqatp3Pzm6a/MGSziESnREx784JlKfwKMjJl33UA8lQm9nhSeAIHyBx3c4Lf8IXdW2n3rnhbVfjpBMAxwh6lt+e5agtGXy+q/xAESUeLPfUgRYWctlLgt8Op+WTpLyBkZsVFoBvJrMt2XdM0RI32YzTRr56GXFa4VyQmY5xXwlQSPgidAP7jPkVygNcoeXvAz2ZCk3IR1Cn3mX8nMko53MlDNaMYldUQA0ug28/S7BlSlaq2CDD4Ol3swTq7C4KGTxKrI36ruYUZx7NEaQDF5V7VvqPCZ0fZoTIJuSYTQ67gwEQYKKwYBBAGDjBsCATEDEwFvMD0GCSsGAQIBDwMBATEwEy5RbVhSWTRyaEF4OE11cThkTUdrcjlxa25KZEU2VUhaRGRHYURSVFFFYndGTjViMEcGA1UdETFAEz42dnUyYnhraTc3N2l0M2NwYXl2NmZxNnZwbDRrZTNremo3Z3hpY2Z5Z201NWRoaHRwaHlmZHZ5ZC5vbmlvbjAKBggqhkjOPQQDAgNJADBGAiEAt+f1u/bchg5AZHv6NTGNoXeejTRWUhX3ioGwW6TGg84CIQCHqKNzDh2JjS/hUHx5PApAmfNnQTSf19X6LnNHQweU1g==', + +// // // o +// // 'MIIDHTCCAsMCAQAwSTFHMEUGA1UEAxM+eTd5Y3ptdWdsMnRla2FtaTdzYmR6NXBmYWVtdng3YmFod3RocmR2Y2J6dzV2ZXgyY3JzcjI2cWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATMq0l4bCmjdb0grtzpwtDVLM9E1IQpL9vrB4+lD9OBZzlrx2365jV7shVu9utas8w8fxtKoBZSnT5+32ZMFTB4oIICFjAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBSoDQpTZdEvi1/Rr/muVXT1clyKRDCCAUcGCSqGSIb3DQEJDDGCATgEggE0BQvyvkiiXEf/PLKnsR1Ba9AhYsVO8o56bnftUnoVzBlRZgUzLJvOSroPk/EmbVz+okhMrcYNgCWHvxrAqHVVq0JRP6bi98BtCUotx6OPFHp5K5QCL60hod1uAnhKocyJG9tsoM9aS+krn/k+g4RCBjiPZ25cC7QG/UNr6wyIQ8elBho4MKm8iOp7EShSsZOV1f6xrnXYCC/zyUc85GEuycLzVImgAQvPATbdMzY4zSGnNLHxkvSUNxaR9LnEWf+i1jeqcOiXOvmdyU5Be3ZqhGKvvBg/5vyLQiCIfeapjZemnLqFHQBitglDm2xnKL6HzMyfZoAHPV7YcWYR4spU9Ju8Q8aqSeAryx7sx55eSR4GO5UQTo5DrQn6xtkwOZ/ytsOknFthF8jcA9uTAMDKA2TylCUwEQYKKwYBBAGDjBsCATEDEwFvMD0GCSsGAQIBDwMBATEwEy5RbVQxOFV2blVCa3NlTWMzU3FuZlB4cEh3TjhuekxySmVOU0xadGM4ckFGWGh6MEcGA1UdETFAEz55N3ljem11Z2wydGVrYW1pN3NiZHo1cGZhZW12eDdiYWh3dGhyZHZjYnp3NXZleDJjcnNyMjZxZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiEAoFrAglxmk7ciD6AHQOB1qEoLu0NARcxgwmIry8oeTHwCICyXp5NJQ9Z8vReIAQNng2H2+/XjHifZEWzhoN0VkcBx', +// // ] + +// await storageService.init(peerId) +// // @ts-ignore +// storageService.certificatesRequestsStore = { +// getEntries: jest.fn(() => { +// return csrs +// }), +// } +// // @ts-ignore +// storageService.certificatesStore = { +// getEntries: jest.fn(() => { +// return certs +// }), +// } + +// const allUsers = await storageService.getAllUsers() + +// expect(allUsers).toStrictEqual(expected) +// }) +// }) + +// describe('Files deletion', () => { +// let realFilePath: string +// let messages: { +// messages: Record +// } + +// beforeEach(async () => { +// realFilePath = path.join(dirname, '/real-file.txt') +// createArbitraryFile(realFilePath, 2147483) +// await storageService.init(peerId) + +// const metadata: FileMetadata = { +// path: realFilePath, +// name: 'test-large-file', +// ext: '.txt', +// cid: 'uploading_id', +// message: { +// id: 'id', +// channelId: channel.id, +// }, +// } + +// const aliceMessage = await factory.create['payload']>( +// 'Message', +// { +// identity: alice, +// message: generateMessageFactoryContentWithId(channel.id, MessageType.File, metadata), +// } +// ) + +// messages = { +// messages: { +// [aliceMessage.message.id]: aliceMessage.message, +// }, +// } +// }) + +// afterEach(() => { +// if (fs.existsSync(realFilePath)) { +// fs.rmSync(realFilePath) +// } +// }) + +// it('delete file correctly', async () => { +// const isFileExist = await storageService.checkIfFileExist(realFilePath) +// expect(isFileExist).toBeTruthy() + +// await expect(storageService.deleteFilesFromChannel(messages)).resolves.not.toThrowError() + +// await waitForExpect(async () => { +// expect(await storageService.checkIfFileExist(realFilePath)).toBeFalsy() +// }, 2000) +// }) + +// it('file dont exist - not throw error', async () => { +// fs.rmSync(realFilePath) + +// await waitForExpect(async () => { +// expect(await storageService.checkIfFileExist(realFilePath)).toBeFalsy() +// }, 2000) + +// await expect(storageService.deleteFilesFromChannel(messages)).resolves.not.toThrowError() +// }) +// }) +// }) diff --git a/packages/backend/src/nest/storage/storage.service.ts b/packages/backend/src/nest/storage/storage.service.ts index 36d95ff2c7..98f955319f 100644 --- a/packages/backend/src/nest/storage/storage.service.ts +++ b/packages/backend/src/nest/storage/storage.service.ts @@ -1,31 +1,19 @@ import { Inject, Injectable } from '@nestjs/common' import { CertFieldsTypes, - keyObjectFromString, - verifySignature, parseCertificate, parseCertificationRequest, getCertFieldValue, getReqFieldValue, keyFromCertificate, } from '@quiet/identity' -import { type KeyValueType, type EventsType, IPFSAccessController, type LogEntry } from '@orbitdb/core' +import { EventsType, type KeyValueType } from '@orbitdb/core' import { EventEmitter } from 'events' import { type PeerId } from '@libp2p/interface' -import { getCrypto } from 'pkijs' -import { stringToArrayBuffer } from 'pvutils' -import validate from '../validation/validators' import { - ChannelMessage, CommunityMetadata, ConnectionProcessInfo, - type CreateChannelResponse, - DeleteFilesFromChannelSocketPayload, - FileMetadata, - type MessagesLoadedPayload, - NoCryptoEngineError, PublicChannel, - PushNotificationPayload, SaveCSRPayload, SaveCertificatePayload, SocketActionTypes, @@ -36,33 +24,25 @@ import { } from '@quiet/types' import { createLibp2pAddress } from '@quiet/common' import fs from 'fs' -import { IpfsFileManagerService } from '../ipfs-file-manager/ipfs-file-manager.service' import { IPFS_REPO_PATCH, ORBIT_DB_DIR, QUIET_DIR } from '../const' -import { IpfsFilesManagerEvents } from '../ipfs-file-manager/ipfs-file-manager.types' import { LocalDbService } from '../local-db/local-db.service' import { createLogger } from '../common/logger' import { PublicChannelsRepo } from '../common/types' import { removeFiles, removeDirs, createPaths } from '../common/utils' -import { DBOptions, StorageEvents } from './storage.types' +import { StorageEvents } from './storage.types' import { CertificatesStore } from './certificates/certificates.store' import { CertificatesRequestsStore } from './certifacteRequests/certificatesRequestsStore' import { IpfsService } from '../ipfs/ipfs.service' import { OrbitDbService } from './orbitDb/orbitDb.service' import { CommunityMetadataStore } from './communityMetadata/communityMetadata.store' import { UserProfileStore } from './userProfile/userProfile.store' -import { KeyValueIndexedValidated } from './orbitDb/keyValueIndexedValidated' -import { MessagesAccessController } from './orbitDb/MessagesAccessController' -import { EventsWithStorage } from './orbitDb/eventsWithStorage' import { LocalDBKeys } from '../local-db/local-db.types' +import { ChannelsService } from './channels/channels.service' @Injectable() export class StorageService extends EventEmitter { private peerId: PeerId | null = null - public publicChannelsRepos: Map = new Map() - private publicKeysMap: Map = new Map() - private certificates: EventsType | null - private channels: KeyValueType | null private readonly logger = createLogger(StorageService.name) @@ -72,12 +52,12 @@ export class StorageService extends EventEmitter { @Inject(IPFS_REPO_PATCH) public readonly ipfsRepoPath: string, private readonly localDbService: LocalDbService, private readonly ipfsService: IpfsService, - private readonly filesManager: IpfsFileManagerService, private readonly orbitDbService: OrbitDbService, private readonly certificatesRequestsStore: CertificatesRequestsStore, private readonly certificatesStore: CertificatesStore, private readonly communityMetadataStore: CommunityMetadataStore, - private readonly userProfileStore: UserProfileStore + private readonly userProfileStore: UserProfileStore, + private readonly channelsService: ChannelsService ) { super() } @@ -105,10 +85,6 @@ export class StorageService extends EventEmitter { this.logger.info(`Creating OrbitDB service`) await this.orbitDbService.create(peerId, this.ipfsService.ipfsInstance!) - this.logger.info(`Starting file manager`) - this.attachFileManagerEvents() - await this.filesManager.init() - this.logger.info(`Initializing Databases`) await this.initDatabases() @@ -125,13 +101,10 @@ export class StorageService extends EventEmitter { } await this.communityMetadataStore.startSync() - await this.channels?.sync.start() await this.certificatesStore.startSync() await this.certificatesRequestsStore.startSync() await this.userProfileStore.startSync() - for (const channel of this.publicChannelsRepos.values()) { - await channel.db.sync.start() - } + await this.channelsService.startSync() } static dbAddress = (db: { root: string; path: string }) => { @@ -139,6 +112,10 @@ export class StorageService extends EventEmitter { return `/orbitdb/${db.root}/${db.path}` } + public get channels() { + return this.channelsService + } + public async initDatabases() { this.logger.time('Storage.initDatabases') @@ -161,8 +138,7 @@ export class StorageService extends EventEmitter { await this.userProfileStore.init() this.logger.info('3/3') - await this.createDbForChannels() - await this.initAllChannels() + await this.channelsService.init(this.peerId!) this.logger.timeEnd('Storage.initDatabases') this.logger.info('Initialized DBs') @@ -171,13 +147,7 @@ export class StorageService extends EventEmitter { } public async stop() { - try { - this.logger.info('Closing channels DB') - await this.channels?.close() - this.logger.info('Closed channels DB') - } catch (e) { - this.logger.error('Error closing channels db', e) - } + await this.channelsService.closeChannels() try { await this.certificatesStore?.close() @@ -204,13 +174,7 @@ export class StorageService extends EventEmitter { } await this.orbitDbService.stop() - - this.logger.info('Stopping IPFS files manager') - try { - await this.filesManager.stop() - } catch (e) { - this.logger.error('Error stopping IPFS files manager', e) - } + await this.channelsService.closeFileManager() try { await this.ipfsService.stop() @@ -285,337 +249,6 @@ export class StorageService extends EventEmitter { return await this.certificatesStore.getEntries() } - public async setChannel(id: string, channel: PublicChannel) { - if (!this.channels) { - throw new Error('Channels have not been initialized!') - } - await this.channels.put(id, channel) - } - - public async getChannel(id: string) { - if (!this.channels) { - throw new Error('Channels have not been initialized!') - } - return await this.channels.get(id) - } - - public async getChannels(): Promise { - if (!this.channels) { - throw new Error('Channels have not been initialized!') - } - return (await this.channels.all()).map(x => x.value) - } - - public async loadAllChannels() { - this.logger.info('Getting all channels') - this.emit(StorageEvents.CHANNELS_STORED, { - channels: await this.getChannels(), - }) - } - - private async createDbForChannels() { - this.logger.info('Creating public-channels database') - this.channels = await this.orbitDbService.orbitDb.open>('public-channels', { - sync: false, - Database: KeyValueIndexedValidated(), - AccessController: IPFSAccessController({ write: ['*'] }), - }) - - this.channels.events.on('update', async (entry: LogEntry) => { - this.logger.info('public-channels database updated') - - this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.CHANNELS_STORED) - - const channels = await this.getChannels() - - this.emit(StorageEvents.CHANNELS_STORED, { channels }) - - channels.forEach(channel => this.subscribeToChannel(channel, { replicate: true })) - }) - - const channels = await this.getChannels() - this.logger.info('Channels count:', channels.length) - this.logger.info( - 'Channels names:', - channels.map(x => x.name) - ) - channels.forEach(channel => this.subscribeToChannel(channel)) - } - - async initAllChannels() { - this.emit(StorageEvents.CHANNELS_STORED, { - channels: await this.getChannels(), - }) - } - - async verifyMessage(message: ChannelMessage): Promise { - const crypto = getCrypto() - if (!crypto) throw new NoCryptoEngineError() - - const signature = stringToArrayBuffer(message.signature) - let cryptoKey = this.publicKeysMap.get(message.pubKey) - - if (!cryptoKey) { - cryptoKey = await keyObjectFromString(message.pubKey, crypto) - this.publicKeysMap.set(message.pubKey, cryptoKey) - } - - return await verifySignature(signature, message.message, cryptoKey) - } - - protected async getAllEventLogEntries(db: EventsType): Promise { - const res: T[] = [] - - for await (const x of db.iterator()) { - res.push(x.value) - } - - return res - } - - public async subscribeToChannel( - channelData: PublicChannel, - options = { replicate: false } - ): Promise { - let db: EventsType - // @ts-ignore - if (channelData.address) { - // @ts-ignore - channelData.id = channelData.address - } - let repo = this.publicChannelsRepos.get(channelData.id) - - if (repo) { - db = repo.db - } else { - try { - db = await this.createChannel(channelData, options) - } catch (e) { - this.logger.error(`Can't subscribe to channel ${channelData.id}`, e) - return - } - if (!db) { - this.logger.error(`Can't subscribe to channel ${channelData.id}, the DB isn't initialized!`) - return - } - repo = this.publicChannelsRepos.get(channelData.id) - } - - if (repo && !repo.eventsAttached) { - this.logger.info('Subscribing to channel ', channelData.id) - - db.events.on('update', async (entry: LogEntry) => { - this.logger.info(`${channelData.id} database updated`, entry.hash, entry.payload.value?.channelId) - - const message = entry.payload.value! - const verified = await this.verifyMessage(message) - - this.emit(StorageEvents.MESSAGES_STORED, { - messages: [message], - isVerified: verified, - }) - - const ids = (await this.getAllEventLogEntries(db)).map(msg => msg.id) - const community = await this.localDbService.getCurrentCommunity() - - if (community) { - this.emit(StorageEvents.MESSAGE_IDS_STORED, { - ids, - channelId: channelData.id, - communityId: community.id, - }) - } - - // FIXME: the 'update' event runs if we replicate entries and if we add - // entries ourselves. So we may want to check if the message is written - // by us. - // - // Display push notifications on mobile - if (process.env.BACKEND === 'mobile') { - if (!verified) return - - // Do not notify about old messages - if (message.createdAt < parseInt(process.env.CONNECTION_TIME || '')) return - - const username = await this.certificatesStore.getCertificateUsername(message.pubKey) - if (!username) { - this.logger.error(`Can't send push notification, no username found for public key '${message.pubKey}'`) - return - } - - const payload: PushNotificationPayload = { - message: JSON.stringify(message), - username: username, - } - - this.emit(StorageEvents.SEND_PUSH_NOTIFICATION, payload) - } - }) - - const ids = (await this.getAllEventLogEntries(db)).map(msg => msg.id) - const community = await this.localDbService.getCurrentCommunity() - - if (community) { - this.emit(StorageEvents.MESSAGE_IDS_STORED, { - ids, - channelId: channelData.id, - communityId: community.id, - }) - } - - repo.eventsAttached = true - } - - this.logger.info(`Subscribed to channel ${channelData.id}`) - this.emit(StorageEvents.CHANNEL_SUBSCRIBED, { - channelId: channelData.id, - }) - return { channel: channelData } - } - - public async getMessages(channelId: string, ids: string[]): Promise { - const repo = this.publicChannelsRepos.get(channelId) - if (!repo) return - - const messages = await this.getAllEventLogEntries(repo.db) - const filteredMessages: ChannelMessage[] = [] - - for (const id of ids) { - filteredMessages.push(...messages.filter(i => i.id === id)) - } - - return { - messages: filteredMessages, - isVerified: true, - } - } - - private async createChannel(channelData: PublicChannel, options: DBOptions): Promise> { - if (!validate.isChannel(channelData)) { - this.logger.error('Invalid channel format') - throw new Error('Create channel validation error') - } - - this.logger.info(`Creating channel ${channelData.id}`) - - const channelId = channelData.id - const db = await this.orbitDbService.orbitDb.open>(`channels.${channelId}`, { - type: 'events', - Database: EventsWithStorage(), - AccessController: MessagesAccessController({ write: ['*'] }), - }) - const channel = await this.getChannel(channelId) - - if (channel === undefined) { - await this.setChannel(channelId, channelData) - } else { - this.logger.info(`Channel ${channelId} already exists`) - } - - this.publicChannelsRepos.set(channelId, { db, eventsAttached: false }) - this.logger.info(`Set ${channelId} to local channels`) - this.logger.info(`Created channel ${channelId}`) - - return db - } - - public async deleteChannel(payload: { channelId: string; ownerPeerId: string }) { - this.logger.info('deleting channel storage', payload) - const { channelId, ownerPeerId } = payload - const channel = await this.getChannel(channelId) - if (!this.peerId) { - this.logger.error('deleteChannel - peerId is null') - throw new Error('deleteChannel - peerId is null') - } - const isOwner = ownerPeerId === this.peerId.toString() - if (channel && isOwner) { - if (!this.channels) { - throw new Error('Channels have not been initialized!') - } - await this.channels.del(channelId) - } - let repo = this.publicChannelsRepos.get(channelId) - if (!repo) { - const db = await this.orbitDbService.orbitDb.open>(`channels.${channelId}`, { - sync: false, - type: 'events', - Database: EventsWithStorage(), - AccessController: MessagesAccessController({ write: ['*'] }), - }) - repo = { - db, - eventsAttached: false, - } - } - await repo.db.sync.stop() - await repo.db.drop() - this.publicChannelsRepos.delete(channelId) - return { channelId: payload.channelId } - } - - public async deleteChannelFiles(files: FileMetadata[]) { - for (const file of files) { - await this.deleteFile(file) - } - } - - public async deleteFile(fileMetadata: FileMetadata) { - await this.filesManager.deleteBlocks(fileMetadata) - } - - public async sendMessage(message: ChannelMessage) { - if (!validate.isMessage(message)) { - this.logger.error('STORAGE: public channel message is invalid') - return - } - const repo = this.publicChannelsRepos.get(message.channelId) - if (!repo) { - this.logger.error(`Could not send message. No '${message.channelId}' channel in saved public channels`) - return - } - try { - this.logger.info('Sending message:', message.id) - await repo.db.add(message) - } catch (e) { - this.logger.error( - `STORAGE: Could not append message (entry not allowed to write to the log). Details: ${e.message}` - ) - } - } - - private attachFileManagerEvents = () => { - this.filesManager.on(IpfsFilesManagerEvents.DOWNLOAD_PROGRESS, status => { - this.emit(StorageEvents.DOWNLOAD_PROGRESS, status) - }) - this.filesManager.on(IpfsFilesManagerEvents.MESSAGE_MEDIA_UPDATED, messageMedia => { - this.emit(StorageEvents.MESSAGE_MEDIA_UPDATED, messageMedia) - }) - this.filesManager.on(StorageEvents.REMOVE_DOWNLOAD_STATUS, payload => { - this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, payload) - }) - this.filesManager.on(StorageEvents.FILE_UPLOADED, payload => { - this.emit(StorageEvents.FILE_UPLOADED, payload) - }) - this.filesManager.on(StorageEvents.DOWNLOAD_PROGRESS, payload => { - this.emit(StorageEvents.DOWNLOAD_PROGRESS, payload) - }) - this.filesManager.on(StorageEvents.MESSAGE_MEDIA_UPDATED, payload => { - this.emit(StorageEvents.MESSAGE_MEDIA_UPDATED, payload) - }) - } - - public async uploadFile(metadata: FileMetadata) { - this.filesManager.emit(IpfsFilesManagerEvents.UPLOAD_FILE, metadata) - } - - public async downloadFile(metadata: FileMetadata) { - this.filesManager.emit(IpfsFilesManagerEvents.DOWNLOAD_FILE, metadata) - } - - public cancelDownload(mid: string) { - this.filesManager.emit(IpfsFilesManagerEvents.CANCEL_DOWNLOAD, mid) - } - public async saveCertificate(payload: SaveCertificatePayload): Promise { this.logger.info('About to save certificate...') if (!payload.certificate) { @@ -685,28 +318,6 @@ export class StorageService extends EventEmitter { return allUsers } - public async deleteFilesFromChannel(payload: DeleteFilesFromChannelSocketPayload) { - const { messages } = payload - Object.keys(messages).map(async key => { - const message = messages[key] - if (message?.media?.path) { - const mediaPath = message.media.path - this.logger.info('deleteFilesFromChannel : mediaPath', mediaPath) - const isFileExist = await this.checkIfFileExist(mediaPath) - this.logger.info(`deleteFilesFromChannel : isFileExist- ${isFileExist}`) - if (isFileExist) { - fs.unlink(mediaPath, unlinkError => { - if (unlinkError) { - this.logger.error(`deleteFilesFromChannel : unlink error`, unlinkError) - } - }) - } else { - this.logger.error(`deleteFilesFromChannel : file does not exist`, mediaPath) - } - } - }) - } - public async addUserProfile(profile: UserProfile) { await this.userProfileStore.setEntry(profile.pubKey, profile) } @@ -731,16 +342,9 @@ export class StorageService extends EventEmitter { public async clean() { this.peerId = null - // @ts-ignore - this.channels = undefined - // @ts-ignore - this.messageThreads = undefined - // @ts-ignore - this.publicChannelsRepos = new Map() - this.publicKeysMap = new Map() + await this.channelsService.clean() this.certificates = null - this.channels = null this.certificatesRequestsStore.clean() this.certificatesStore.clean() From b7d36cf16ecefe7f35b42b9bda87c72a6651027d Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 24 Jan 2025 18:04:13 -0500 Subject: [PATCH 02/10] Separate logic further into channelstore and messagesservice --- packages/backend/src/nest/common/types.ts | 6 +- .../backend/src/nest/storage/base.store.ts | 2 +- .../nest/storage/channels/channel.store.ts | 262 +++++++++ .../nest/storage/channels/channels.service.ts | 520 ++++++++++-------- .../channels/messages/messages.service.ts | 67 +++ .../src/nest/storage/storage.module.ts | 2 + .../src/nest/storage/storage.service.ts | 3 + .../backend/src/nest/storage/storage.types.ts | 2 +- 8 files changed, 630 insertions(+), 234 deletions(-) create mode 100644 packages/backend/src/nest/storage/channels/channel.store.ts create mode 100644 packages/backend/src/nest/storage/channels/messages/messages.service.ts diff --git a/packages/backend/src/nest/common/types.ts b/packages/backend/src/nest/common/types.ts index 43a3d4189e..e5a3c32856 100644 --- a/packages/backend/src/nest/common/types.ts +++ b/packages/backend/src/nest/common/types.ts @@ -1,8 +1,8 @@ -import { type EventsType } from '@orbitdb/core' -import { type ChannelMessage, type PublicChannel } from '@quiet/types' +import { type PublicChannel } from '@quiet/types' +import { ChannelStore } from '../storage/channels/channel.store' export interface PublicChannelsRepo { - db: EventsType + store: ChannelStore eventsAttached: boolean } diff --git a/packages/backend/src/nest/storage/base.store.ts b/packages/backend/src/nest/storage/base.store.ts index 290ed42df9..d4b97f926b 100644 --- a/packages/backend/src/nest/storage/base.store.ts +++ b/packages/backend/src/nest/storage/base.store.ts @@ -24,7 +24,7 @@ abstract class StoreBase | EventsType> extends E logger.info('Closed', this.getAddress()) } - abstract init(): Promise + abstract init(...args: any[]): Promise | Promise> abstract clean(): void } diff --git a/packages/backend/src/nest/storage/channels/channel.store.ts b/packages/backend/src/nest/storage/channels/channel.store.ts new file mode 100644 index 0000000000..cf3411b21a --- /dev/null +++ b/packages/backend/src/nest/storage/channels/channel.store.ts @@ -0,0 +1,262 @@ +import { Injectable } from '@nestjs/common' + +import { EventsType, LogEntry } from '@orbitdb/core' + +import { QuietLogger } from '@quiet/logger' +import { ChannelMessage, MessagesLoadedPayload, PublicChannel, PushNotificationPayload } from '@quiet/types' + +import { createLogger } from '../../common/logger' +import { EventStoreBase } from '../base.store' +import { EventsWithStorage } from '../orbitDb/eventsWithStorage' +import { MessagesAccessController } from '../orbitDb/MessagesAccessController' +import { OrbitDbService } from '../orbitDb/orbitDb.service' +import validate from '../../validation/validators' +import { MessagesService } from './messages/messages.service' +import { DBOptions, StorageEvents } from '../storage.types' +import { LocalDbService } from '../../local-db/local-db.service' +import { CertificatesStore } from '../certificates/certificates.store' + +/** + * Manages storage-level logic for a given channel in Quiet + */ +@Injectable() +export class ChannelStore extends EventStoreBase { + private channelData: PublicChannel + private initialized: boolean = false + private logger: QuietLogger + + constructor( + private readonly orbitDbService: OrbitDbService, + private readonly localDbService: LocalDbService, + private readonly messagesService: MessagesService, + private readonly certificatesStore: CertificatesStore + ) { + super() + } + + // Initialization + + /** + * Initialize this instance of ChannelStore by opening an OrbitDB database + * + * @param channelData Channel configuration metadata + * @param options Database options for OrbitDB + * @returns Initialized ChannelStore instance + */ + public async init(channelData: PublicChannel, options: DBOptions): Promise { + if (this.initialized) { + this.logger.warn(`Channel ${this.channelData.name} has already been initialized!`) + return this + } + + this.channelData = channelData + this.logger = createLogger(`storage:channels:channelStore:${this.channelData.name}`) + this.logger.info(`Initializing channel store for channel ${this.channelData.name}`) + + this.store = await this.orbitDbService.orbitDb.open>(`channels.${this.channelData.id}`, { + type: 'events', + Database: EventsWithStorage(), + AccessController: MessagesAccessController({ write: ['*'] }), + sync: options.sync, + }) + + this.logger.info('Initialized') + this.initialized = true + return this + } + + /** + * Start syncing the OrbitDB database + */ + public async startSync(): Promise { + await this.getStore().sync.start() + } + + /** + * Subscribe to new messages on this channel + * + * @emits StorageEvents.MESSAGE_IDS_STORED + * @emits StorageEvents.MESSAGES_STORED + * @emits StorageEvents.SEND_PUSH_NOTIFICATION + */ + public async subscribe(): Promise { + this.logger.info('Subscribing to channel ', this.channelData.id) + + this.getStore().events.on('update', async (entry: LogEntry) => { + this.logger.info(`${this.channelData.id} database updated`, entry.hash, entry.payload.value?.channelId) + + const message = entry.payload.value! + const verified = await this.messagesService.verifyMessage(message) + + this.emit(StorageEvents.MESSAGES_STORED, { + messages: [message], + isVerified: verified, + }) + + const ids = (await this.getEntries()).map(msg => msg.id) + const community = await this.localDbService.getCurrentCommunity() + + if (community) { + this.emit(StorageEvents.MESSAGE_IDS_STORED, { + ids, + channelId: this.channelData.id, + communityId: community.id, + }) + } + + // FIXME: the 'update' event runs if we replicate entries and if we add + // entries ourselves. So we may want to check if the message is written + // by us. + // + // Display push notifications on mobile + if (process.env.BACKEND === 'mobile') { + if (!verified) return + + // Do not notify about old messages + if (message.createdAt < parseInt(process.env.CONNECTION_TIME || '')) return + + const username = await this.certificatesStore.getCertificateUsername(message.pubKey) + if (!username) { + this.logger.error(`Can't send push notification, no username found for public key '${message.pubKey}'`) + return + } + + const payload: PushNotificationPayload = { + message: JSON.stringify(message), + username: username, + } + + this.emit(StorageEvents.SEND_PUSH_NOTIFICATION, payload) + } + }) + + await this.startSync() + + this.logger.info(`Subscribed to channel ${this.channelData.id}`) + } + + // Messages + + /** + * Validate and append a new message to this channel's OrbitDB database + * + * @param message Message to add to the OrbitDB database + */ + public async sendMessage(message: ChannelMessage): Promise { + this.logger.info(`Sending message with ID ${message.id} on channel ${this.channelData.id}`) + if (!validate.isMessage(message)) { + this.logger.error('Public channel message is invalid') + return + } + + if (message.channelId != this.channelData.id) { + this.logger.error( + `Could not send message. Message is for channel ID ${message.channelId} which does not match channel ID ${this.channelData.id}` + ) + return + } + + try { + await this.addEntry(message) + } catch (e) { + this.logger.error(`Could not append message (entry not allowed to write to the log). Details: ${e.message}`) + } + } + + /** + * Read messages from OrbitDB filtered by message ID + * + * @param ids Message IDs to read from this channel's OrbitDB database + * @returns Messages read from OrbitDB + */ + public async getMessages(ids: string[]): Promise { + const messages = await this.getEntries() + const filteredMessages: ChannelMessage[] = [] + + for (const id of ids) { + filteredMessages.push(...messages.filter(i => i.id === id)) + } + + return { + messages: filteredMessages, + isVerified: true, + } + } + + // Base Store Logic + + /** + * Add a new event to the OrbitDB event store + * + * @param message Message to add to the OrbitDB database + * @returns Hash of the new database entry + */ + public async addEntry(message: ChannelMessage): Promise { + if (!this.store) { + throw new Error('Store is not initialized') + } + + this.logger.info('Adding message to database') + return await this.store.add(message) + } + + /** + * Read all entries on the OrbitDB event store + * + * @returns All entries on the event store + */ + public async getEntries(): Promise { + this.logger.info(`Getting all messages for channel`, this.channelData.id, this.channelData.name) + const messages: ChannelMessage[] = [] + + for await (const x of this.getStore().iterator()) { + messages.push(x.value) + } + + return messages + } + + // Close Logic + + /** + * Stop syncing the OrbitDB database + */ + public async stopSync(): Promise { + await this.getStore().sync.stop() + } + + /** + * Close the OrbitDB database + */ + public async close(): Promise { + this.logger.info(`Closing channel store`) + await this.stopSync() + await this.getStore().close() + } + + /** + * Delete the channel from OrbitDB + */ + public async deleteChannel(): Promise { + this.logger.info(`Deleting channel`) + try { + await this.stopSync() + await this.getStore().drop() + } catch (e) { + // we expect an error if the database isn't synced + } + + this.clean() + } + + /** + * Clean this ChannelStore + * + * NOTE: Does NOT affect data stored in IPFS + */ + public clean(): void { + this.logger.info(`Cleaning channel store`, this.channelData.id, this.channelData.name) + this.store = undefined + this.initialized = false + } +} diff --git a/packages/backend/src/nest/storage/channels/channels.service.ts b/packages/backend/src/nest/storage/channels/channels.service.ts index 2d7acfecfe..05626ee553 100644 --- a/packages/backend/src/nest/storage/channels/channels.service.ts +++ b/packages/backend/src/nest/storage/channels/channels.service.ts @@ -1,11 +1,7 @@ import { Inject, Injectable } from '@nestjs/common' -import { keyObjectFromString, verifySignature } from '@quiet/identity' -import { type KeyValueType, type EventsType, IPFSAccessController, type LogEntry } from '@orbitdb/core' +import { type KeyValueType, IPFSAccessController, type LogEntry } from '@orbitdb/core' import { EventEmitter } from 'events' import { type PeerId } from '@libp2p/interface' -import { getCrypto } from 'pkijs' -import { stringToArrayBuffer } from 'pvutils' -import validate from '../../validation/validators' import { ChannelMessage, ConnectionProcessInfo, @@ -13,50 +9,55 @@ import { DeleteFilesFromChannelSocketPayload, FileMetadata, type MessagesLoadedPayload, - NoCryptoEngineError, PublicChannel, PushNotificationPayload, SocketActionTypes, + ChannelMessageIdsResponse, + DeleteChannelResponse, } from '@quiet/types' import fs from 'fs' import { IpfsFileManagerService } from '../../ipfs-file-manager/ipfs-file-manager.service' import { IPFS_REPO_PATCH, ORBIT_DB_DIR, QUIET_DIR } from '../../const' import { IpfsFilesManagerEvents } from '../../ipfs-file-manager/ipfs-file-manager.types' -import { LocalDbService } from '../../local-db/local-db.service' import { createLogger } from '../../common/logger' import { PublicChannelsRepo } from '../../common/types' -import { DBOptions, StorageEvents } from '../storage.types' -import { CertificatesStore } from '../certificates/certificates.store' +import { StorageEvents } from '../storage.types' import { OrbitDbService } from '../orbitDb/orbitDb.service' import { KeyValueIndexedValidated } from '../orbitDb/keyValueIndexedValidated' -import { MessagesAccessController } from '../orbitDb/MessagesAccessController' -import { EventsWithStorage } from '../orbitDb/eventsWithStorage' +import { ChannelStore } from './channel.store' +import { createContextId, ModuleRef } from '@nestjs/core' +/** + * Manages storage-level logic for all channels in Quiet + */ @Injectable() export class ChannelsService extends EventEmitter { private peerId: PeerId | null = null public publicChannelsRepos: Map = new Map() - private publicKeysMap: Map = new Map() private channels: KeyValueType | null - private readonly logger = createLogger(ChannelsService.name) + private readonly logger = createLogger(`storage:channels`) constructor( @Inject(QUIET_DIR) public readonly quietDir: string, @Inject(ORBIT_DB_DIR) public readonly orbitDbDir: string, @Inject(IPFS_REPO_PATCH) public readonly ipfsRepoPath: string, private readonly filesManager: IpfsFileManagerService, - private readonly localDbService: LocalDbService, private readonly orbitDbService: OrbitDbService, - private readonly certificatesStore: CertificatesStore + private readonly moduleRef: ModuleRef ) { super() } - // INITIALIZATION + // Initialization - public async init(peerId: PeerId) { + /** + * Initialize the ChannelsService by starting event handles, the file manager, and initializing databases in OrbitDB + * + * @param peerId Peer ID of the current user + */ + public async init(peerId: PeerId): Promise { this.logger.info(`Initializing ${ChannelsService.name}`) this.peerId = peerId @@ -70,53 +71,35 @@ export class ChannelsService extends EventEmitter { this.logger.info(`Initialized ${ChannelsService.name}`) } - public async initChannels() { + /** + * Initialize the channels management database and individual channel stores in OrbitDB + */ + public async initChannels(): Promise { this.logger.time(`Initializing channel databases`) - this.attachFileManagerEvents() - await this.createDbForChannels() - await this.initAllChannels() + await this.createChannelsDb() + await this.loadAllChannels() this.logger.timeEnd('Initializing channel databases') this.logger.info('Initialized databases') } - public async startSync() { + /** + * Start syncing the channels management database in OrbitDB + */ + public async startSync(): Promise { await this.channels?.sync.start() - for (const channel of this.publicChannelsRepos.values()) { - await channel.db.sync.start() - } - } - - public async setChannel(id: string, channel: PublicChannel) { - if (!this.channels) { - throw new Error('Channels have not been initialized!') - } - await this.channels.put(id, channel) } - public async getChannel(id: string) { - if (!this.channels) { - throw new Error('Channels have not been initialized!') - } - return await this.channels.get(id) - } + // Channels Database Management - public async getChannels(): Promise { - if (!this.channels) { - throw new Error('Channels have not been initialized!') - } - return (await this.channels.all()).map(x => x.value) - } - - public async loadAllChannels() { - this.logger.info('Getting all channels') - this.emit(StorageEvents.CHANNELS_STORED, { - channels: await this.getChannels(), - }) - } - - private async createDbForChannels() { + /** + * Create the channels management database in OrbitDB + * + * NOTE: This also subscribes to all known channel stores and handles update events on the channels management database for + * subscribing to newly created channel stores. + */ + private async createChannelsDb(): Promise { this.logger.info('Creating public-channels database') this.channels = await this.orbitDbService.orbitDb.open>('public-channels', { sync: false, @@ -125,7 +108,9 @@ export class ChannelsService extends EventEmitter { }) this.channels.events.on('update', async (entry: LogEntry) => { - this.logger.info('public-channels database updated') + const channelId = entry.payload.key + const operation = entry.payload.op + this.logger.info('public-channels database updated', channelId, operation) this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.CHANNELS_STORED) @@ -133,7 +118,10 @@ export class ChannelsService extends EventEmitter { this.emit(StorageEvents.CHANNELS_STORED, { channels }) - channels.forEach(channel => this.subscribeToChannel(channel, { replicate: true })) + if (operation === 'PUT') { + const channel = entry.payload.value as PublicChannel + await this.subscribeToChannel(channel) + } }) const channels = await this.getChannels() @@ -145,42 +133,112 @@ export class ChannelsService extends EventEmitter { channels.forEach(channel => this.subscribeToChannel(channel)) } - async initAllChannels() { + /** + * Add a channel to the channels management database + * + * @param id ID of channel to add to the channels database + * @param channel Channel configuration metadata + * @throws Error + */ + public async setChannel(id: string, channel: PublicChannel): Promise { + if (!this.channels) { + throw new Error('Channels have not been initialized!') + } + await this.channels.put(id, channel) + } + + /** + * Read channel metadata by ID from the channels management database + * + * @param id ID of channel to fetch + * @returns Channel metadata, if it exists + * @throws Error + */ + public async getChannel(id: string): Promise { + if (!this.channels) { + throw new Error('Channels have not been initialized!') + } + return await this.channels.get(id) + } + + /** + * Read entries for all keys in the channels management database + * + * @returns All channel metadata in the channels management database + * @throws Error + */ + public async getChannels(): Promise { + if (!this.channels) { + throw new Error('Channels have not been initialized!') + } + return (await this.channels.all()).map(x => x.value) + } + + /** + * Get all known channels and emit event with metadata + * + * @emits StorageEvents.CHANNELS_STORED + */ + public async loadAllChannels(): Promise { + this.logger.info('Getting all channels') this.emit(StorageEvents.CHANNELS_STORED, { channels: await this.getChannels(), }) } - async verifyMessage(message: ChannelMessage): Promise { - const crypto = getCrypto() - if (!crypto) throw new NoCryptoEngineError() + // Channel Management - const signature = stringToArrayBuffer(message.signature) - let cryptoKey = this.publicKeysMap.get(message.pubKey) + /** + * Create a new ChannelStore and, optionally, add the metadata to the channels management database + * + * @param channelData Channel metadata for new channel + * @returns Newly created ChannelStore + */ + private async createChannel(channelData: PublicChannel): Promise { + this.logger.info(`Creating channel`, channelData.id, channelData.name) - if (!cryptoKey) { - cryptoKey = await keyObjectFromString(message.pubKey, crypto) - this.publicKeysMap.set(message.pubKey, cryptoKey) - } + const channelId = channelData.id + const store = await this.createChannelStore(channelData) - return await verifySignature(signature, message.message, cryptoKey) - } + const channel = await this.getChannel(channelId) + if (channel == undefined) { + await this.setChannel(channelId, channelData) + } else { + this.logger.info(`Channel ${channelId} already exists`) + } - protected async getAllEventLogEntries(db: EventsType): Promise { - const res: T[] = [] + this.publicChannelsRepos.set(channelId, { store, eventsAttached: false }) + this.logger.info(`Set ${channelId} to local channels`) + this.logger.info(`Created channel ${channelId}`) - for await (const x of db.iterator()) { - res.push(x.value) - } + return store + } - return res + /** + * Helper method for creating and initializing ChannelStore + * + * @param channelData Channel metadata for new channel + * @returns Newly created ChannelStore + */ + private async createChannelStore(channelData: PublicChannel): Promise { + const store = await this.moduleRef.create(ChannelStore, createContextId()) + return await store.init(channelData, { sync: false }) } - public async subscribeToChannel( - channelData: PublicChannel, - options = { replicate: false } - ): Promise { - let db: EventsType + /** + * Creates a new channel store with the supplied metadata, if it doesn't exist, and subscribes + * to new events on the store, if it didn't already exist. + * + * NOTE: Storage events like MESSAGE_IDS_STORED are consumed up the chain on this service but are + * emitted on the ChannelStore instances so we consume and re-emit them on this service's event + * emitter. + * + * @param channelData Channel metadata for channel we are subscribing to + * @returns CreateChannelResponse + * @emits StorageEvents.CHANNEL_SUBSCRIBED + */ + public async subscribeToChannel(channelData: PublicChannel): Promise { + let store: ChannelStore // @ts-ignore if (channelData.address) { // @ts-ignore @@ -189,84 +247,25 @@ export class ChannelsService extends EventEmitter { let repo = this.publicChannelsRepos.get(channelData.id) if (repo) { - db = repo.db + store = repo.store } else { try { - db = await this.createChannel(channelData, options) + store = await this.createChannel(channelData) } catch (e) { this.logger.error(`Can't subscribe to channel ${channelData.id}`, e) return } - if (!db) { + if (!store) { this.logger.error(`Can't subscribe to channel ${channelData.id}, the DB isn't initialized!`) return } repo = this.publicChannelsRepos.get(channelData.id) - } - - if (repo && !repo.eventsAttached) { - this.logger.info('Subscribing to channel ', channelData.id) - db.events.on('update', async (entry: LogEntry) => { - this.logger.info(`${channelData.id} database updated`, entry.hash, entry.payload.value?.channelId) - - const message = entry.payload.value! - const verified = await this.verifyMessage(message) - - this.emit(StorageEvents.MESSAGES_STORED, { - messages: [message], - isVerified: verified, - }) - - const ids = (await this.getAllEventLogEntries(db)).map(msg => msg.id) - const community = await this.localDbService.getCurrentCommunity() - - if (community) { - this.emit(StorageEvents.MESSAGE_IDS_STORED, { - ids, - channelId: channelData.id, - communityId: community.id, - }) - } - - // FIXME: the 'update' event runs if we replicate entries and if we add - // entries ourselves. So we may want to check if the message is written - // by us. - // - // Display push notifications on mobile - if (process.env.BACKEND === 'mobile') { - if (!verified) return - - // Do not notify about old messages - if (message.createdAt < parseInt(process.env.CONNECTION_TIME || '')) return - - const username = await this.certificatesStore.getCertificateUsername(message.pubKey) - if (!username) { - this.logger.error(`Can't send push notification, no username found for public key '${message.pubKey}'`) - return - } - - const payload: PushNotificationPayload = { - message: JSON.stringify(message), - username: username, - } - - this.emit(StorageEvents.SEND_PUSH_NOTIFICATION, payload) - } - }) - - const ids = (await this.getAllEventLogEntries(db)).map(msg => msg.id) - const community = await this.localDbService.getCurrentCommunity() - - if (community) { - this.emit(StorageEvents.MESSAGE_IDS_STORED, { - ids, - channelId: channelData.id, - communityId: community.id, - }) + if (repo && !repo.eventsAttached) { + this.handleMessageEventsOnChannelStore(channelData.id, repo) + await repo.store.subscribe() + repo.eventsAttached = true } - - repo.eventsAttached = true } this.logger.info(`Subscribed to channel ${channelData.id}`) @@ -276,54 +275,43 @@ export class ChannelsService extends EventEmitter { return { channel: channelData } } - public async getMessages(channelId: string, ids: string[]): Promise { - const repo = this.publicChannelsRepos.get(channelId) - if (!repo) return - - const messages = await this.getAllEventLogEntries(repo.db) - const filteredMessages: ChannelMessage[] = [] - - for (const id of ids) { - filteredMessages.push(...messages.filter(i => i.id === id)) - } - - return { - messages: filteredMessages, - isVerified: true, - } - } - - private async createChannel(channelData: PublicChannel, options: DBOptions): Promise> { - if (!validate.isChannel(channelData)) { - this.logger.error('Invalid channel format') - throw new Error('Create channel validation error') - } - - this.logger.info(`Creating channel ${channelData.id}`) - - const channelId = channelData.id - const db = await this.orbitDbService.orbitDb.open>(`channels.${channelId}`, { - type: 'events', - Database: EventsWithStorage(), - AccessController: MessagesAccessController({ write: ['*'] }), + /** + * Capture events emitted by individual channel stores and re-emit on the channels service + * + * @param channelId ID of channel to re-emit events from + * @param repo Repo containing the store we are re-emitting events from + * @emits StorageEvents.MESSAGE_IDS_STORED + * @emits StorageEvents.MESSAGES_STORED + * @emits StorageEvents.SEND_PUSH_NOTIFICATION + */ + private handleMessageEventsOnChannelStore(channelId: string, repo: PublicChannelsRepo): void { + this.logger.info(`Subscribing to channel updates`, channelId) + repo.store.on(StorageEvents.MESSAGE_IDS_STORED, (payload: ChannelMessageIdsResponse) => { + this.logger.info(`Emitting ${StorageEvents.MESSAGE_IDS_STORED}`) + this.emit(StorageEvents.MESSAGE_IDS_STORED, payload) }) - const channel = await this.getChannel(channelId) - if (channel === undefined) { - await this.setChannel(channelId, channelData) - } else { - this.logger.info(`Channel ${channelId} already exists`) - } - - this.publicChannelsRepos.set(channelId, { db, eventsAttached: false }) - this.logger.info(`Set ${channelId} to local channels`) - this.logger.info(`Created channel ${channelId}`) + repo.store.on(StorageEvents.MESSAGES_STORED, (payload: MessagesLoadedPayload) => { + this.logger.info(`Emitting ${StorageEvents.MESSAGES_STORED}`) + this.emit(StorageEvents.MESSAGES_STORED, payload) + }) - return db + repo.store.on(StorageEvents.SEND_PUSH_NOTIFICATION, (payload: PushNotificationPayload) => { + this.logger.info(`Emitting ${StorageEvents.SEND_PUSH_NOTIFICATION}`) + this.emit(StorageEvents.SEND_PUSH_NOTIFICATION, payload) + }) } - public async deleteChannel(payload: { channelId: string; ownerPeerId: string }) { - this.logger.info('deleting channel storage', payload) + /** + * Get the store for a given channel ID and, optionally, create a temporary store if it doesn't exist then drop + * the database from OrbitDB + * + * @param payload Metadata on the channel to be deleted + * @returns Response containing metadata on the channel that was deleted + * @throws Error + */ + public async deleteChannel(payload: { channelId: string; ownerPeerId: string }): Promise { + this.logger.info('Deleting channel', payload) const { channelId, ownerPeerId } = payload const channel = await this.getChannel(channelId) if (!this.peerId) { @@ -337,56 +325,89 @@ export class ChannelsService extends EventEmitter { } await this.channels.del(channelId) } - let repo = this.publicChannelsRepos.get(channelId) - if (!repo) { - const db = await this.orbitDbService.orbitDb.open>(`channels.${channelId}`, { - sync: false, - type: 'events', - Database: EventsWithStorage(), - AccessController: MessagesAccessController({ write: ['*'] }), - }) - repo = { - db, - eventsAttached: false, + const repo = this.publicChannelsRepos.get(channelId) + let store = repo?.store + if (store == null) { + const channelData: PublicChannel = channel ?? { + id: channelId, + name: 'undefined', + owner: ownerPeerId, + description: 'undefined', + timestamp: 0, } + store = await this.createChannelStore(channelData) } - await repo.db.sync.stop() - await repo.db.drop() + await store.deleteChannel() this.publicChannelsRepos.delete(channelId) - return { channelId: payload.channelId } + return { channelId } } - public async deleteChannelFiles(files: FileMetadata[]) { - for (const file of files) { - await this.deleteFile(file) + // Messages + + /** + * Sends a message on a given channel if that channel is known + * + * @param message Message to send + */ + public async sendMessage(message: ChannelMessage): Promise { + const repo = this.publicChannelsRepos.get(message.channelId) + if (repo == null) { + this.logger.error(`Could not send message. No '${message.channelId}' channel in saved public channels`) + return } - } - public async deleteFile(fileMetadata: FileMetadata) { - await this.filesManager.deleteBlocks(fileMetadata) + await repo.store.sendMessage(message) } - public async sendMessage(message: ChannelMessage) { - if (!validate.isMessage(message)) { - this.logger.error('STORAGE: public channel message is invalid') - return - } - const repo = this.publicChannelsRepos.get(message.channelId) - if (!repo) { - this.logger.error(`Could not send message. No '${message.channelId}' channel in saved public channels`) + /** + * Read messages for a list of message IDs from a given channel if that channel is known + * + * @param channelId ID of channel to read messages from + * @param ids IDS of messages to read + * @returns Payload containing messages read + */ + public async getMessages(channelId: string, messageIds: string[]): Promise { + const repo = this.publicChannelsRepos.get(channelId) + if (repo == null) { + this.logger.error(`Could not read messages. No '${channelId}' channel in saved public channels`) return } - try { - this.logger.info('Sending message:', message.id) - await repo.db.add(message) - } catch (e) { - this.logger.error( - `STORAGE: Could not append message (entry not allowed to write to the log). Details: ${e.message}` - ) + + return await repo.store.getMessages(messageIds) + } + + // Files + + /** + * Delete multiple files from the file manager + * + * @param files List of file metadata to be deleted + */ + public async deleteChannelFiles(files: FileMetadata[]): Promise { + for (const file of files) { + await this.deleteFile(file) } } - private attachFileManagerEvents = () => { + /** + * Deleted a single file from the file manager + * + * @param fileMetadata Metadata of file to be deleted + */ + public async deleteFile(fileMetadata: FileMetadata): Promise { + await this.filesManager.deleteBlocks(fileMetadata) + } + + /** + * Consume file manager events and emit storage events on the channels service + * + * @emits StorageEvents.DOWNLOAD_PROGRESS + * @emits StorageEvents.MESSAGE_MEDIA_UPDATED + * @emits StorageEvents.REMOVE_DOWNLOAD_STATUS + * @emits StorageEvents.FILE_UPLOADED + * @emits StorageEvents.DOWNLOAD_PROGRESS + */ + private attachFileManagerEvents(): void { this.filesManager.on(IpfsFilesManagerEvents.DOWNLOAD_PROGRESS, status => { this.emit(StorageEvents.DOWNLOAD_PROGRESS, status) }) @@ -407,19 +428,42 @@ export class ChannelsService extends EventEmitter { }) } - public async uploadFile(metadata: FileMetadata) { + /** + * Emit event to trigger file upload on file manager + * + * @param metadata Metadata of file to be uploaded + * @emits IpfsFilesManagerEvents.UPLOAD_FILE + */ + public async uploadFile(metadata: FileMetadata): Promise { this.filesManager.emit(IpfsFilesManagerEvents.UPLOAD_FILE, metadata) } - public async downloadFile(metadata: FileMetadata) { + /** + * Emit event to trigger file download on file manager + * + * @param metadata Metadata of file to be downloaded + * @emits IpfsFilesManagerEvents.DOWNLOAD_FILE + */ + public async downloadFile(metadata: FileMetadata): Promise { this.filesManager.emit(IpfsFilesManagerEvents.DOWNLOAD_FILE, metadata) } - public cancelDownload(mid: string) { + /** + * Emit event to trigger file download cancellation on file manager + * + * @param metadata Metadata of file to be cancelled + * @emits IpfsFilesManagerEvents.CANCEL_DOWNLOAD + */ + public cancelDownload(mid: string): void { this.filesManager.emit(IpfsFilesManagerEvents.CANCEL_DOWNLOAD, mid) } - public async deleteFilesFromChannel(payload: DeleteFilesFromChannelSocketPayload) { + /** + * Delete files for a list of messages + * + * @param payload Payload containing file messages whose files should be deleted + */ + public async deleteFilesFromChannel(payload: DeleteFilesFromChannelSocketPayload): Promise { const { messages } = payload Object.keys(messages).map(async key => { const message = messages[key] @@ -441,6 +485,12 @@ export class ChannelsService extends EventEmitter { }) } + /** + * Check if the file with the supplied path exists on the file system + * + * @param filepath Path to file + * @returns True if file exists at the path + */ public async checkIfFileExist(filepath: string): Promise { return await new Promise(resolve => { fs.access(filepath, fs.constants.F_OK, error => { @@ -449,6 +499,11 @@ export class ChannelsService extends EventEmitter { }) } + // Close Logic + + /** + * Close the channels management database on OrbitDB + */ public async closeChannels(): Promise { try { this.logger.info('Closing channels DB') @@ -459,6 +514,9 @@ export class ChannelsService extends EventEmitter { } } + /** + * Stop the file manager + */ public async closeFileManager(): Promise { try { this.logger.info('Stopping IPFS files manager') @@ -468,7 +526,12 @@ export class ChannelsService extends EventEmitter { } } - public async clean() { + /** + * Clean the ChannelsService + * + * NOTE: Does NOT affect data stored in IPFS + */ + public async clean(): Promise { this.peerId = null // @ts-ignore @@ -477,7 +540,6 @@ export class ChannelsService extends EventEmitter { this.messageThreads = undefined // @ts-ignore this.publicChannelsRepos = new Map() - this.publicKeysMap = new Map() this.channels = null } diff --git a/packages/backend/src/nest/storage/channels/messages/messages.service.ts b/packages/backend/src/nest/storage/channels/messages/messages.service.ts new file mode 100644 index 0000000000..74e527a256 --- /dev/null +++ b/packages/backend/src/nest/storage/channels/messages/messages.service.ts @@ -0,0 +1,67 @@ +import { Injectable } from '@nestjs/common' +import { keyObjectFromString, verifySignature } from '@quiet/identity' +import { stringToArrayBuffer } from 'pvutils' +import { ChannelMessage, NoCryptoEngineError } from '@quiet/types' +import EventEmitter from 'events' +import { getCrypto, ICryptoEngine } from 'pkijs' + +import { createLogger } from '../../../common/logger' + +@Injectable() +export class MessagesService extends EventEmitter { + /** + * Map of signing keys used on messages + * + * Maps public key string -> CryptoKey + */ + private publicKeysMap: Map = new Map() + + private readonly logger = createLogger(`storage:channels:messagesService`) + + constructor() { + super() + } + + /** + * Verify signature on message + * + * @param message Message to verify + * @returns True if message is valid + */ + public async verifyMessage(message: ChannelMessage): Promise { + const crypto = this.getCrypto() + const signature = stringToArrayBuffer(message.signature) + let cryptoKey = this.publicKeysMap.get(message.pubKey) + + if (!cryptoKey) { + cryptoKey = await keyObjectFromString(message.pubKey, crypto) + this.publicKeysMap.set(message.pubKey, cryptoKey) + } + + return await verifySignature(signature, message.message, cryptoKey) + } + + /** + * Get crypto engine that was initialized previously + * + * @returns Crypto engine + * @throws NoCryptoEngineError + */ + private getCrypto(): ICryptoEngine { + const crypto = getCrypto() + if (crypto == null) { + throw new NoCryptoEngineError() + } + + return crypto + } + + /** + * Clean service + * + * NOTE: Does NOT affect data stored in IPFS + */ + public async clean(): Promise { + this.publicKeysMap = new Map() + } +} diff --git a/packages/backend/src/nest/storage/storage.module.ts b/packages/backend/src/nest/storage/storage.module.ts index ed01e66bd5..e889ed237b 100644 --- a/packages/backend/src/nest/storage/storage.module.ts +++ b/packages/backend/src/nest/storage/storage.module.ts @@ -9,6 +9,7 @@ import { CommunityMetadataStore } from './communityMetadata/communityMetadata.st import { UserProfileStore } from './userProfile/userProfile.store' import { IpfsModule } from '../ipfs/ipfs.module' import { ChannelsService } from './channels/channels.service' +import { MessagesService } from './channels/messages/messages.service' @Module({ imports: [LocalDbModule, IpfsModule, IpfsFileManagerModule], @@ -20,6 +21,7 @@ import { ChannelsService } from './channels/channels.service' CertificatesRequestsStore, UserProfileStore, ChannelsService, + MessagesService, ], exports: [StorageService], }) diff --git a/packages/backend/src/nest/storage/storage.service.ts b/packages/backend/src/nest/storage/storage.service.ts index 98f955319f..3fbb18bf89 100644 --- a/packages/backend/src/nest/storage/storage.service.ts +++ b/packages/backend/src/nest/storage/storage.service.ts @@ -112,6 +112,9 @@ export class StorageService extends EventEmitter { return `/orbitdb/${db.root}/${db.path}` } + /** + * Get the ChannelsService for managing channels and messages + */ public get channels() { return this.channelsService } diff --git a/packages/backend/src/nest/storage/storage.types.ts b/packages/backend/src/nest/storage/storage.types.ts index 20bbf4039b..6ed848fcfe 100644 --- a/packages/backend/src/nest/storage/storage.types.ts +++ b/packages/backend/src/nest/storage/storage.types.ts @@ -40,5 +40,5 @@ export interface CsrReplicatedPromiseValues { } export interface DBOptions { - replicate: boolean + sync: boolean } From 606195c5796462a9de929d03700b0e33f4d7a510 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 24 Jan 2025 18:24:17 -0500 Subject: [PATCH 03/10] Add temp placeholders for encryption/decryption --- .../channels/messages/messages.service.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/nest/storage/channels/messages/messages.service.ts b/packages/backend/src/nest/storage/channels/messages/messages.service.ts index 74e527a256..a0f6526ce3 100644 --- a/packages/backend/src/nest/storage/channels/messages/messages.service.ts +++ b/packages/backend/src/nest/storage/channels/messages/messages.service.ts @@ -1,11 +1,14 @@ import { Injectable } from '@nestjs/common' -import { keyObjectFromString, verifySignature } from '@quiet/identity' import { stringToArrayBuffer } from 'pvutils' -import { ChannelMessage, NoCryptoEngineError } from '@quiet/types' import EventEmitter from 'events' import { getCrypto, ICryptoEngine } from 'pkijs' +import { keyObjectFromString, verifySignature } from '@quiet/identity' +import { ChannelMessage, NoCryptoEngineError } from '@quiet/types' + import { createLogger } from '../../../common/logger' +import { EncryptedAndSignedPayload, EncryptedPayload } from '../../../auth/services/crypto/types' +import { SignedEnvelope } from '3rd-party/auth/packages/auth/dist' @Injectable() export class MessagesService extends EventEmitter { @@ -41,6 +44,18 @@ export class MessagesService extends EventEmitter { return await verifySignature(signature, message.message, cryptoKey) } + // TODO: https://github.com/TryQuiet/quiet/issues/2631 + // NOTE: the signature here may not be correct + public async encryptMessage(message: ChannelMessage): Promise { + throw new Error(`MessagesService.encryptMessage is not implemented!`) + } + + // TODO: https://github.com/TryQuiet/quiet/issues/2632 + // NOTE: the signature here may not be correct + public async decryptMessage(encrypted: EncryptedPayload, signature: SignedEnvelope): Promise { + throw new Error(`MessagesService.decryptMessage is not implemented!`) + } + /** * Get crypto engine that was initialized previously * From 4b0b22f9488c61fa8dc5dd73c7a6b0afba1f91f3 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 24 Jan 2025 18:27:08 -0500 Subject: [PATCH 04/10] Minor clean up --- packages/backend/src/nest/storage/storage.service.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/backend/src/nest/storage/storage.service.ts b/packages/backend/src/nest/storage/storage.service.ts index 3fbb18bf89..d5ab6a2f86 100644 --- a/packages/backend/src/nest/storage/storage.service.ts +++ b/packages/backend/src/nest/storage/storage.service.ts @@ -7,13 +7,11 @@ import { getReqFieldValue, keyFromCertificate, } from '@quiet/identity' -import { EventsType, type KeyValueType } from '@orbitdb/core' import { EventEmitter } from 'events' import { type PeerId } from '@libp2p/interface' import { CommunityMetadata, ConnectionProcessInfo, - PublicChannel, SaveCSRPayload, SaveCertificatePayload, SocketActionTypes, @@ -27,7 +25,6 @@ import fs from 'fs' import { IPFS_REPO_PATCH, ORBIT_DB_DIR, QUIET_DIR } from '../const' import { LocalDbService } from '../local-db/local-db.service' import { createLogger } from '../common/logger' -import { PublicChannelsRepo } from '../common/types' import { removeFiles, removeDirs, createPaths } from '../common/utils' import { StorageEvents } from './storage.types' import { CertificatesStore } from './certificates/certificates.store' @@ -42,7 +39,6 @@ import { ChannelsService } from './channels/channels.service' @Injectable() export class StorageService extends EventEmitter { private peerId: PeerId | null = null - private certificates: EventsType | null private readonly logger = createLogger(StorageService.name) @@ -347,8 +343,6 @@ export class StorageService extends EventEmitter { await this.channelsService.clean() - this.certificates = null - this.certificatesRequestsStore.clean() this.certificatesStore.clean() this.communityMetadataStore.clean() From 5b9ac30cd40dab2e0ec9869a1922b8398f78068f Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Mon, 27 Jan 2025 12:47:45 -0500 Subject: [PATCH 05/10] Expand messages service, tweak getting messages, fix/add tests --- packages/backend/src/nest/common/utils.ts | 8 +- .../ipfs-file-manager/big-files.long.spec.ts | 2 +- .../nest/storage/channels/channel.store.ts | 31 +- .../src/nest/storage/channels/channel.ts | 0 .../storage/channels/channels.service.spec.ts | 278 ++++++++ .../nest/storage/channels/channels.service.ts | 16 +- .../messages/messages.service.spec.ts | 110 +++ .../channels/messages/messages.service.ts | 37 +- .../src/nest/storage/storage.module.ts | 3 +- .../src/nest/storage/storage.service.spec.ts | 638 +++++++----------- .../src/nest/storage/storage.service.ts | 8 - packages/types/src/channel.ts | 4 + 12 files changed, 691 insertions(+), 444 deletions(-) delete mode 100644 packages/backend/src/nest/storage/channels/channel.ts create mode 100644 packages/backend/src/nest/storage/channels/channels.service.spec.ts create mode 100644 packages/backend/src/nest/storage/channels/messages/messages.service.spec.ts diff --git a/packages/backend/src/nest/common/utils.ts b/packages/backend/src/nest/common/utils.ts index a0fc690829..30f527e4e4 100644 --- a/packages/backend/src/nest/common/utils.ts +++ b/packages/backend/src/nest/common/utils.ts @@ -1,4 +1,5 @@ import fs from 'fs' +import fsAsync from 'fs/promises' import getPort from 'get-port' import path from 'path' import { Server } from 'socket.io' @@ -298,19 +299,16 @@ export async function createPeerId(): Promise { } } -export const createArbitraryFile = (filePath: string, sizeBytes: number) => { - const stream = fs.createWriteStream(filePath) +export const createArbitraryFile = async (filePath: string, sizeBytes: number) => { const maxChunkSize = 1048576 // 1MB let remainingSize = sizeBytes while (remainingSize > 0) { const chunkSize = Math.min(maxChunkSize, remainingSize) - stream.write(crypto.randomBytes(chunkSize)) + await fsAsync.appendFile(filePath, crypto.randomBytes(chunkSize)) remainingSize -= chunkSize } - - stream.end() } export async function* asyncGeneratorFromIterator(asyncIterator: AsyncIterable): AsyncGenerator { diff --git a/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts b/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts index 3ad536b46f..947101e956 100644 --- a/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts +++ b/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts @@ -33,7 +33,7 @@ describe('IpfsFileManagerService', () => { tmpDir = createTmpDir() filePath = new URL('./testUtils/large-file.txt', import.meta.url).pathname // Generate 2.1GB file - createArbitraryFile(filePath, BIG_FILE_SIZE) + await createArbitraryFile(filePath, BIG_FILE_SIZE) module = await Test.createTestingModule({ imports: [TestModule, IpfsFileManagerModule, IpfsModule, SocketModule, Libp2pModule], }).compile() diff --git a/packages/backend/src/nest/storage/channels/channel.store.ts b/packages/backend/src/nest/storage/channels/channel.store.ts index cf3411b21a..ec38fbca33 100644 --- a/packages/backend/src/nest/storage/channels/channel.store.ts +++ b/packages/backend/src/nest/storage/channels/channel.store.ts @@ -85,12 +85,11 @@ export class ChannelStore extends EventStoreBase { this.getStore().events.on('update', async (entry: LogEntry) => { this.logger.info(`${this.channelData.id} database updated`, entry.hash, entry.payload.value?.channelId) - const message = entry.payload.value! - const verified = await this.messagesService.verifyMessage(message) + const message = await this.messagesService.onConsume(entry.payload.value!) this.emit(StorageEvents.MESSAGES_STORED, { messages: [message], - isVerified: verified, + isVerified: message.verified, }) const ids = (await this.getEntries()).map(msg => msg.id) @@ -110,7 +109,7 @@ export class ChannelStore extends EventStoreBase { // // Display push notifications on mobile if (process.env.BACKEND === 'mobile') { - if (!verified) return + if (!message.verified) return // Do not notify about old messages if (message.createdAt < parseInt(process.env.CONNECTION_TIME || '')) return @@ -169,16 +168,10 @@ export class ChannelStore extends EventStoreBase { * @param ids Message IDs to read from this channel's OrbitDB database * @returns Messages read from OrbitDB */ - public async getMessages(ids: string[]): Promise { - const messages = await this.getEntries() - const filteredMessages: ChannelMessage[] = [] - - for (const id of ids) { - filteredMessages.push(...messages.filter(i => i.id === id)) - } - + public async getMessages(ids: string[] | undefined = undefined): Promise { + const messages = await this.getEntries(ids) return { - messages: filteredMessages, + messages, isVerified: true, } } @@ -197,7 +190,8 @@ export class ChannelStore extends EventStoreBase { } this.logger.info('Adding message to database') - return await this.store.add(message) + const processedMessage = await this.messagesService.onSend(message) + return await this.store.add(processedMessage) } /** @@ -205,12 +199,17 @@ export class ChannelStore extends EventStoreBase { * * @returns All entries on the event store */ - public async getEntries(): Promise { + public async getEntries(): Promise + public async getEntries(ids: string[] | undefined): Promise + public async getEntries(ids?: string[] | undefined): Promise { this.logger.info(`Getting all messages for channel`, this.channelData.id, this.channelData.name) const messages: ChannelMessage[] = [] for await (const x of this.getStore().iterator()) { - messages.push(x.value) + if (ids == null || ids?.includes(x.id)) { + const processedMessage = await this.messagesService.onConsume(x.value, false) + messages.push(processedMessage) + } } return messages diff --git a/packages/backend/src/nest/storage/channels/channel.ts b/packages/backend/src/nest/storage/channels/channel.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/backend/src/nest/storage/channels/channels.service.spec.ts b/packages/backend/src/nest/storage/channels/channels.service.spec.ts new file mode 100644 index 0000000000..f92b720da8 --- /dev/null +++ b/packages/backend/src/nest/storage/channels/channels.service.spec.ts @@ -0,0 +1,278 @@ +import { jest } from '@jest/globals' + +import { Test, TestingModule } from '@nestjs/testing' +import { keyFromCertificate, parseCertificate } from '@quiet/identity' +import { + prepareStore, + getFactory, + publicChannels, + generateMessageFactoryContentWithId, + Store, +} from '@quiet/state-manager' +import { + ChannelMessage, + Community, + FileMetadata, + Identity, + MessageType, + PublicChannel, + TestMessage, +} from '@quiet/types' + +import path from 'path' +import { type PeerId } from '@libp2p/interface' +import waitForExpect from 'wait-for-expect' +import { TestModule } from '../../common/test.module' +import { createArbitraryFile, libp2pInstanceParams } from '../../common/utils' +import { IpfsModule } from '../../ipfs/ipfs.module' +import { IpfsService } from '../../ipfs/ipfs.service' +import { Libp2pModule } from '../../libp2p/libp2p.module' +import { Libp2pService } from '../../libp2p/libp2p.service' +import { SocketModule } from '../../socket/socket.module' +import { StorageModule } from '../storage.module' +import { StorageService } from '../storage.service' +import fs from 'fs' +import { type FactoryGirl } from 'factory-girl' +import { fileURLToPath } from 'url' +import { LocalDbModule } from '../../local-db/local-db.module' +import { LocalDbService } from '../../local-db/local-db.service' +import { createLogger } from '../../common/logger' +import { ChannelsService } from './channels.service' + +const logger = createLogger('channelsService:test') + +const filename = fileURLToPath(import.meta.url) +const dirname = path.dirname(filename) + +describe('ChannelsService', () => { + let module: TestingModule + let storageService: StorageService + let ipfsService: IpfsService + let libp2pService: Libp2pService + let localDbService: LocalDbService + let channelsService: ChannelsService + let peerId: PeerId + + let store: Store + let factory: FactoryGirl + let community: Community + let channel: PublicChannel + let alice: Identity + let john: Identity + let message: ChannelMessage + let channelio: PublicChannel + let filePath: string + + jest.setTimeout(50000) + + beforeAll(async () => { + store = prepareStore().store + factory = await getFactory(store) + + community = await factory.create('Community') + + channel = publicChannels.selectors.publicChannels(store.getState())[0] + + channelio = { + name: channel.name, + description: channel.description, + owner: channel.owner, + timestamp: channel.timestamp, + id: channel.id, + } + + alice = await factory.create('Identity', { id: community.id, nickname: 'alice' }) + + john = await factory.create('Identity', { id: community.id, nickname: 'john' }) + + message = ( + await factory.create('Message', { + identity: alice, + message: generateMessageFactoryContentWithId(channel.id), + }) + ).message + }) + + beforeEach(async () => { + jest.clearAllMocks() + filePath = path.join(dirname, '/500kB-file.txt') + + module = await Test.createTestingModule({ + imports: [TestModule, StorageModule, IpfsModule, SocketModule, Libp2pModule, LocalDbModule], + }).compile() + + storageService = await module.resolve(StorageService) + channelsService = await module.resolve(ChannelsService) + localDbService = await module.resolve(LocalDbService) + libp2pService = await module.resolve(Libp2pService) + ipfsService = await module.resolve(IpfsService) + + const params = await libp2pInstanceParams() + peerId = params.peerId.peerId + + await libp2pService.createInstance(params) + expect(libp2pService.libp2pInstance).not.toBeNull() + + await localDbService.open() + expect(localDbService.getStatus()).toEqual('open') + + await localDbService.setCommunity(community) + await localDbService.setCurrentCommunityId(community.id) + + await storageService.init(peerId) + }) + + afterEach(async () => { + await libp2pService.libp2pInstance?.stop() + await ipfsService.ipfsInstance?.stop() + await storageService.stop() + if (fs.existsSync(filePath)) { + fs.rmSync(filePath) + } + await module.close() + }) + + describe('Channels', () => { + it('deletes channel as owner', async () => { + await channelsService.subscribeToChannel(channelio) + + const result = await channelsService.deleteChannel({ channelId: channelio.id, ownerPeerId: peerId.toString() }) + expect(result).toEqual({ channelId: channelio.id }) + + const channelFromKeyValueStore = (await channelsService.getChannels()).filter(x => x.id === channelio.id) + expect(channelFromKeyValueStore).toEqual([]) + }) + + it('delete channel as standard user', async () => { + await channelsService.subscribeToChannel(channelio) + + const result = await channelsService.deleteChannel({ channelId: channelio.id, ownerPeerId: 'random peer id' }) + expect(result).toEqual({ channelId: channelio.id }) + + const channelFromKeyValueStore = (await channelsService.getChannels()).filter(x => x.id === channelio.id) + expect(channelFromKeyValueStore).toEqual([channelio]) + }) + }) + + describe('Message access controller', () => { + it('is saved to db if passed signature verification', async () => { + await channelsService.subscribeToChannel(channelio) + + const publicChannelRepo = channelsService.publicChannelsRepos.get(message.channelId) + expect(publicChannelRepo).not.toBeUndefined() + const store = publicChannelRepo!.store + const eventSpy = jest.spyOn(store, 'addEntry') + + const messageCopy = { + ...message, + } + delete messageCopy.media + + await channelsService.sendMessage(messageCopy) + + // Confirm message has passed orbitdb validator (check signature verification only) + expect(eventSpy).toHaveBeenCalled() + const savedMessages = await channelsService.getMessages(channelio.id) + expect(savedMessages?.messages.length).toBe(1) + expect(savedMessages?.messages[0]).toEqual({ ...messageCopy, verified: true }) + }) + + it('is not saved to db if did not pass signature verification', async () => { + const aliceMessage = await factory.create['payload']>( + 'Message', + { + identity: alice, + message: generateMessageFactoryContentWithId(channel.id), + } + ) + // @ts-expect-error userCertificate can be undefined + const johnCertificate: string = john.userCertificate + const johnPublicKey = keyFromCertificate(parseCertificate(johnCertificate)) + + const spoofedMessage = { + ...aliceMessage.message, + channelId: channelio.id, + pubKey: johnPublicKey, + } + delete spoofedMessage.media // Media 'undefined' is not accepted by db.add + + await channelsService.subscribeToChannel(channelio) + + const publicChannelRepo = channelsService.publicChannelsRepos.get(message.channelId) + expect(publicChannelRepo).not.toBeUndefined() + const store = publicChannelRepo!.store + const eventSpy = jest.spyOn(store, 'addEntry') + + await channelsService.sendMessage(spoofedMessage) + + // Confirm message has passed orbitdb validator (check signature verification only) + expect(eventSpy).toHaveBeenCalled() + expect((await channelsService.getMessages(channelio.id))?.messages.length).toBe(0) + }) + }) + + describe('Files deletion', () => { + let realFilePath: string + let messages: { + messages: Record + } + + beforeEach(async () => { + realFilePath = path.join(dirname, '/real-file.txt') + await createArbitraryFile(realFilePath, 2147483) + + const metadata: FileMetadata = { + path: realFilePath, + name: 'test-large-file', + ext: '.txt', + cid: 'uploading_id', + message: { + id: 'id', + channelId: channel.id, + }, + } + + const aliceMessage = await factory.create['payload']>( + 'Message', + { + identity: alice, + message: generateMessageFactoryContentWithId(channel.id, MessageType.File, metadata), + } + ) + + messages = { + messages: { + [aliceMessage.message.id]: aliceMessage.message, + }, + } + }) + + afterEach(() => { + if (fs.existsSync(realFilePath)) { + fs.rmSync(realFilePath) + } + }) + + it('delete file correctly', async () => { + console.warn(fs.existsSync(realFilePath)) + const isFileExist = await channelsService.checkIfFileExist(realFilePath) + expect(isFileExist).toBeTruthy() + + await expect(channelsService.deleteFilesFromChannel(messages)).resolves.not.toThrowError() + + await waitForExpect(async () => { + expect(await channelsService.checkIfFileExist(realFilePath)).toBeFalsy() + }, 2000) + }) + + it('file dont exist - not throw error', async () => { + fs.rmSync(realFilePath) + + await waitForExpect(async () => { + expect(await channelsService.checkIfFileExist(realFilePath)).toBeFalsy() + }, 2000) + + await expect(channelsService.deleteFilesFromChannel(messages)).resolves.not.toThrowError() + }) + }) +}) diff --git a/packages/backend/src/nest/storage/channels/channels.service.ts b/packages/backend/src/nest/storage/channels/channels.service.ts index 05626ee553..8ec9eba158 100644 --- a/packages/backend/src/nest/storage/channels/channels.service.ts +++ b/packages/backend/src/nest/storage/channels/channels.service.ts @@ -366,7 +366,10 @@ export class ChannelsService extends EventEmitter { * @param ids IDS of messages to read * @returns Payload containing messages read */ - public async getMessages(channelId: string, messageIds: string[]): Promise { + public async getMessages( + channelId: string, + messageIds: string[] | undefined = undefined + ): Promise { const repo = this.publicChannelsRepos.get(channelId) if (repo == null) { this.logger.error(`Could not read messages. No '${channelId}' channel in saved public channels`) @@ -488,15 +491,12 @@ export class ChannelsService extends EventEmitter { /** * Check if the file with the supplied path exists on the file system * - * @param filepath Path to file + * @param filePath Path to file * @returns True if file exists at the path */ - public async checkIfFileExist(filepath: string): Promise { - return await new Promise(resolve => { - fs.access(filepath, fs.constants.F_OK, error => { - resolve(!error) - }) - }) + public async checkIfFileExist(filePath: string): Promise { + this.logger.info(`Checking if ${filePath} exists`) + return fs.existsSync(filePath) } // Close Logic diff --git a/packages/backend/src/nest/storage/channels/messages/messages.service.spec.ts b/packages/backend/src/nest/storage/channels/messages/messages.service.spec.ts new file mode 100644 index 0000000000..e6feb6d3b2 --- /dev/null +++ b/packages/backend/src/nest/storage/channels/messages/messages.service.spec.ts @@ -0,0 +1,110 @@ +import { jest } from '@jest/globals' + +import { Test, TestingModule } from '@nestjs/testing' +import { keyFromCertificate, parseCertificate } from '@quiet/identity' +import { + generateMessageFactoryContentWithId, + getFactory, + prepareStore, + publicChannels, + Store, +} from '@quiet/state-manager' +import { ChannelMessage, Community, Identity, PublicChannel, TestMessage } from '@quiet/types' +import { FactoryGirl } from 'factory-girl' +import { SigChainService } from '../../../auth/sigchain.service' +import { createLogger } from '../../../common/logger' +import { TestModule } from '../../../common/test.module' +import { StorageModule } from '../../storage.module' +import { MessagesService } from './messages.service' + +const logger = createLogger('messagesService:test') + +describe('MessagesService', () => { + let module: TestingModule + let messagesService: MessagesService + let sigChainService: SigChainService + + let store: Store + let factory: FactoryGirl + let alice: Identity + let john: Identity + let community: Community + let channel: PublicChannel + let message: ChannelMessage + + beforeAll(async () => { + store = prepareStore().store + factory = await getFactory(store) + + community = await factory.create('Community') + channel = publicChannels.selectors.publicChannels(store.getState())[0] + alice = await factory.create('Identity', { id: community.id, nickname: 'alice' }) + john = await factory.create('Identity', { id: community.id, nickname: 'john' }) + message = ( + await factory.create('Message', { + identity: alice, + message: generateMessageFactoryContentWithId(channel.id), + }) + ).message + }) + + beforeEach(async () => { + jest.clearAllMocks() + + module = await Test.createTestingModule({ + imports: [TestModule, StorageModule], + }).compile() + + sigChainService = await module.resolve(SigChainService) + messagesService = await module.resolve(MessagesService) + }) + + describe('verifyMessage', () => { + it('message with valid signature is verified', async () => { + expect(await messagesService.verifyMessage(message)).toBeTruthy() + }) + + it('message with invalid signature is not verified', async () => { + expect( + await messagesService.verifyMessage({ + ...message, + pubKey: keyFromCertificate(parseCertificate(john.userCertificate!)), + }) + ).toBeFalsy() + }) + }) + + // TODO: https://github.com/TryQuiet/quiet/issues/2631 + describe('onSend', () => { + it('does nothing but return the message as-is', async () => { + expect(await messagesService.onSend(message)).toEqual(message) + }) + }) + + // TODO: https://github.com/TryQuiet/quiet/issues/2632 + describe('onConsume', () => { + it('runs verifyMessage when verify === true', async () => { + expect(await messagesService.onConsume(message, true)).toEqual({ + ...message, + verified: true, + }) + }) + + it('skips verifyMessage when verify === false', async () => { + const fakePubKey = keyFromCertificate(parseCertificate(john.userCertificate!)) + expect( + await messagesService.onConsume( + { + ...message, + pubKey: fakePubKey, + }, + false + ) + ).toEqual({ + ...message, + pubKey: fakePubKey, + verified: true, + }) + }) + }) +}) diff --git a/packages/backend/src/nest/storage/channels/messages/messages.service.ts b/packages/backend/src/nest/storage/channels/messages/messages.service.ts index a0f6526ce3..0bb06b0958 100644 --- a/packages/backend/src/nest/storage/channels/messages/messages.service.ts +++ b/packages/backend/src/nest/storage/channels/messages/messages.service.ts @@ -4,11 +4,12 @@ import EventEmitter from 'events' import { getCrypto, ICryptoEngine } from 'pkijs' import { keyObjectFromString, verifySignature } from '@quiet/identity' -import { ChannelMessage, NoCryptoEngineError } from '@quiet/types' +import { ChannelMessage, ConsumedChannelMessage, NoCryptoEngineError } from '@quiet/types' import { createLogger } from '../../../common/logger' import { EncryptedAndSignedPayload, EncryptedPayload } from '../../../auth/services/crypto/types' import { SignedEnvelope } from '3rd-party/auth/packages/auth/dist' +import { SigChainService } from '../../../auth/sigchain.service' @Injectable() export class MessagesService extends EventEmitter { @@ -21,10 +22,38 @@ export class MessagesService extends EventEmitter { private readonly logger = createLogger(`storage:channels:messagesService`) - constructor() { + constructor(private readonly sigChainService: SigChainService) { super() } + /** + * Handle processing of message to be added to OrbitDB and sent to peers + * + * NOTE: This will call the encryption method below (https://github.com/TryQuiet/quiet/issues/2631) + * + * @param message Message to send + * @returns Processed message + */ + public async onSend(message: ChannelMessage): Promise { + return message + } + + /** + * Handle processing of message consumed from OrbitDB + * + * NOTE: This will call the decryption method below (https://github.com/TryQuiet/quiet/issues/2632) + * + * @param message Message consumed from OrbitDB + * @returns Processed message + */ + public async onConsume(message: ChannelMessage, verify: boolean = true): Promise { + const verified = verify ? await this.verifyMessage(message) : true + return { + ...message, + verified, + } + } + /** * Verify signature on message * @@ -46,13 +75,13 @@ export class MessagesService extends EventEmitter { // TODO: https://github.com/TryQuiet/quiet/issues/2631 // NOTE: the signature here may not be correct - public async encryptMessage(message: ChannelMessage): Promise { + private async encryptMessage(message: ChannelMessage): Promise { throw new Error(`MessagesService.encryptMessage is not implemented!`) } // TODO: https://github.com/TryQuiet/quiet/issues/2632 // NOTE: the signature here may not be correct - public async decryptMessage(encrypted: EncryptedPayload, signature: SignedEnvelope): Promise { + private async decryptMessage(encrypted: EncryptedPayload, signature: SignedEnvelope): Promise { throw new Error(`MessagesService.decryptMessage is not implemented!`) } diff --git a/packages/backend/src/nest/storage/storage.module.ts b/packages/backend/src/nest/storage/storage.module.ts index e889ed237b..21e1f7b37d 100644 --- a/packages/backend/src/nest/storage/storage.module.ts +++ b/packages/backend/src/nest/storage/storage.module.ts @@ -10,9 +10,10 @@ import { UserProfileStore } from './userProfile/userProfile.store' import { IpfsModule } from '../ipfs/ipfs.module' import { ChannelsService } from './channels/channels.service' import { MessagesService } from './channels/messages/messages.service' +import { SigChainModule } from '../auth/sigchain.service.module' @Module({ - imports: [LocalDbModule, IpfsModule, IpfsFileManagerModule], + imports: [LocalDbModule, IpfsModule, IpfsFileManagerModule, SigChainModule], providers: [ StorageService, OrbitDbService, diff --git a/packages/backend/src/nest/storage/storage.service.spec.ts b/packages/backend/src/nest/storage/storage.service.spec.ts index 7da77a9707..522a075c0f 100644 --- a/packages/backend/src/nest/storage/storage.service.spec.ts +++ b/packages/backend/src/nest/storage/storage.service.spec.ts @@ -1,401 +1,237 @@ -// import { jest } from '@jest/globals' - -// import { Test, TestingModule } from '@nestjs/testing' -// import { keyFromCertificate, parseCertificate } from '@quiet/identity' -// import { -// prepareStore, -// getFactory, -// publicChannels, -// generateMessageFactoryContentWithId, -// Store, -// } from '@quiet/state-manager' -// import { -// ChannelMessage, -// Community, -// FileMetadata, -// Identity, -// MessageType, -// PublicChannel, -// TestMessage, -// } from '@quiet/types' - -// import path from 'path' -// import { type PeerId } from '@libp2p/interface' -// import waitForExpect from 'wait-for-expect' -// import { TestModule } from '../common/test.module' -// import { createArbitraryFile, libp2pInstanceParams } from '../common/utils' -// import { IpfsModule } from '../ipfs/ipfs.module' -// import { IpfsService } from '../ipfs/ipfs.service' -// import { Libp2pModule } from '../libp2p/libp2p.module' -// import { Libp2pService } from '../libp2p/libp2p.service' -// import { SocketModule } from '../socket/socket.module' -// import { StorageModule } from './storage.module' -// import { StorageService } from './storage.service' -// import fs from 'fs' -// import { type FactoryGirl } from 'factory-girl' -// import { fileURLToPath } from 'url' -// import { LocalDbModule } from '../local-db/local-db.module' -// import { LocalDbService } from '../local-db/local-db.service' -// import { ORBIT_DB_DIR } from '../const' -// import { createLogger } from '../common/logger' -// import { createUserCertificateTestHelper, createTestRootCA } from '@quiet/identity' - -// const logger = createLogger('storageService:test') - -// const filename = fileURLToPath(import.meta.url) -// const dirname = path.dirname(filename) - -// describe('StorageService', () => { -// let module: TestingModule -// let storageService: StorageService -// let ipfsService: IpfsService -// let libp2pService: Libp2pService -// let localDbService: LocalDbService -// let peerId: PeerId - -// let store: Store -// let factory: FactoryGirl -// let community: Community -// let channel: PublicChannel -// let alice: Identity -// let john: Identity -// let message: ChannelMessage -// let channelio: PublicChannel -// let filePath: string -// let utils: any -// let orbitDbDir: string - -// jest.setTimeout(50000) - -// beforeAll(async () => { -// store = prepareStore().store -// factory = await getFactory(store) - -// community = await factory.create('Community') - -// channel = publicChannels.selectors.publicChannels(store.getState())[0] - -// channelio = { -// name: channel.name, -// description: channel.description, -// owner: channel.owner, -// timestamp: channel.timestamp, -// id: channel.id, -// } - -// alice = await factory.create('Identity', { id: community.id, nickname: 'alice' }) - -// john = await factory.create('Identity', { id: community.id, nickname: 'john' }) - -// message = ( -// await factory.create('Message', { -// identity: alice, -// message: generateMessageFactoryContentWithId(channel.id), -// }) -// ).message -// }) - -// beforeEach(async () => { -// jest.clearAllMocks() -// utils = await import('../common/utils') -// filePath = path.join(dirname, '/500kB-file.txt') - -// module = await Test.createTestingModule({ -// imports: [TestModule, StorageModule, IpfsModule, SocketModule, Libp2pModule, LocalDbModule], -// }).compile() - -// storageService = await module.resolve(StorageService) -// localDbService = await module.resolve(LocalDbService) -// libp2pService = await module.resolve(Libp2pService) -// ipfsService = await module.resolve(IpfsService) - -// orbitDbDir = await module.resolve(ORBIT_DB_DIR) - -// const params = await libp2pInstanceParams() -// peerId = params.peerId.peerId - -// await libp2pService.createInstance(params) -// expect(libp2pService.libp2pInstance).not.toBeNull() - -// await localDbService.open() -// expect(localDbService.getStatus()).toEqual('open') - -// await localDbService.setCommunity(community) -// await localDbService.setCurrentCommunityId(community.id) -// }) - -// afterEach(async () => { -// await libp2pService.libp2pInstance?.stop() -// await ipfsService.ipfsInstance?.stop() -// await storageService.stop() -// if (fs.existsSync(filePath)) { -// fs.rmSync(filePath) -// } -// await module.close() -// }) - -// it('should be defined', async () => { -// await storageService.init(peerId) -// }) - -// describe('Storage', () => { -// it('should not create paths if createPaths is set to false', async () => { -// const orgProcessPlatform = process.platform -// Object.defineProperty(process, 'platform', { -// value: 'android', -// }) -// expect(fs.existsSync(orbitDbDir)).toBe(false) - -// // FIXME: throws TypeError: Cannot assign to read only property 'createPaths' of object '[object Module]' and I can't be bothered to figure out how to get it to work -// // const createPathsSpy = jest.spyOn(utils, 'createPaths') - -// await storageService.init(peerId) - -// // FIXME: throws TypeError: Cannot assign to read only property 'createPaths' of object '[object Module]' and I can't be bothered to figure out how to get it to work -// // expect(createPathsSpy).not.toHaveBeenCalled() - -// Object.defineProperty(process, 'platform', { -// value: orgProcessPlatform, -// }) -// }) - -// it('db address should be the same on all platforms', () => { -// const dbAddress = StorageService.dbAddress({ root: 'zdpuABCDefgh123', path: 'channels.general_abcd' }) -// expect(dbAddress).toEqual(`/orbitdb/zdpuABCDefgh123/channels.general_abcd`) -// }) -// }) - -// describe('Channels', () => { -// it('deletes channel as owner', async () => { -// await storageService.init(peerId) -// await storageService.subscribeToChannel(channelio) - -// const result = await storageService.deleteChannel({ channelId: channelio.id, ownerPeerId: peerId.toString() }) -// expect(result).toEqual({ channelId: channelio.id }) - -// const channelFromKeyValueStore = (await storageService.getChannels()).filter(x => x.id === channelio.id) -// expect(channelFromKeyValueStore).toEqual([]) -// }) - -// it('delete channel as standard user', async () => { -// await storageService.init(peerId) -// await storageService.subscribeToChannel(channelio) - -// const result = await storageService.deleteChannel({ channelId: channelio.id, ownerPeerId: 'random peer id' }) -// expect(result).toEqual({ channelId: channelio.id }) - -// const channelFromKeyValueStore = (await storageService.getChannels()).filter(x => x.id === channelio.id) -// expect(channelFromKeyValueStore).toEqual([channelio]) -// }) -// }) - -// describe('Message access controller', () => { -// it('is saved to db if passed signature verification', async () => { -// await storageService.init(peerId) - -// await storageService.subscribeToChannel(channelio) - -// const publicChannelRepo = storageService.publicChannelsRepos.get(message.channelId) -// expect(publicChannelRepo).not.toBeUndefined() -// // @ts-expect-error -// const db = publicChannelRepo.db -// const eventSpy = jest.spyOn(db, 'add') - -// const messageCopy = { -// ...message, -// } -// delete messageCopy.media - -// await storageService.sendMessage(messageCopy) - -// // Confirm message has passed orbitdb validator (check signature verification only) -// expect(eventSpy).toHaveBeenCalled() -// // @ts-expect-error -// const savedMessages = await storageService.getAllEventLogEntries(db) -// expect(savedMessages.length).toBe(1) -// expect(savedMessages[0]).toEqual(messageCopy) -// }) - -// it('is not saved to db if did not pass signature verification', async () => { -// const aliceMessage = await factory.create['payload']>( -// 'Message', -// { -// identity: alice, -// message: generateMessageFactoryContentWithId(channel.id), -// } -// ) -// // @ts-expect-error userCertificate can be undefined -// const johnCertificate: string = john.userCertificate -// const johnPublicKey = keyFromCertificate(parseCertificate(johnCertificate)) - -// const spoofedMessage = { -// ...aliceMessage.message, -// channelId: channelio.id, -// pubKey: johnPublicKey, -// } -// delete spoofedMessage.media // Media 'undefined' is not accepted by db.add - -// await storageService.init(peerId) - -// await storageService.subscribeToChannel(channelio) - -// const publicChannelRepo = storageService.publicChannelsRepos.get(message.channelId) -// expect(publicChannelRepo).not.toBeUndefined() -// // @ts-expect-error -// const db = publicChannelRepo.db -// const eventSpy = jest.spyOn(db, 'add') - -// await storageService.sendMessage(spoofedMessage) - -// // Confirm message has passed orbitdb validator (check signature verification only) -// expect(eventSpy).toHaveBeenCalled() -// // @ts-expect-error getAllEventLogEntries is protected -// expect((await storageService.getAllEventLogEntries(db)).length).toBe(0) -// }) -// }) - -// describe('Users', () => { -// it('gets all users from db', async () => { -// const expected = [ -// { -// onionAddress: 'zghidexs7qt24ivu3jobjqdtzzwtyau4lppnunx5pkif76pkpyp7qcid.onion', -// peerId: '12D3KooWKCWstmqi5gaQvipT7xVneVGfWV7HYpCbmUu626R92hXx', -// username: 'b', -// }, -// { -// onionAddress: 'nhliujn66e346evvvxeaf4qma7pqshcgbu6t7wseoannc2fy4cnscryd.onion', -// peerId: '12D3KooWCXzUw71ovvkDky6XkV57aCWUV9JhJoKhoqXa1gdhFNoL', -// username: 'c', -// }, -// { -// onionAddress: '6vu2bxki777it3cpayv6fq6vpl4ke3kzj7gxicfygm55dhhtphyfdvyd.onion', -// peerId: '12D3KooWEHzmff5kZAvyU6Diq5uJG8QkWJxFNUcBLuWjxUGvxaqw', -// username: 'o', -// }, -// { -// onionAddress: 'y7yczmugl2tekami7sbdz5pfaemvx7bahwthrdvcbzw5vex2crsr26qd.onion', -// peerId: '12D3KooWHgLdRMqkepNiYnrur21cyASUNk1f9NZ5tuGa9He8QXNa', -// username: 'o', -// }, -// ] - -// const certs: string[] = [] -// const csrs: string[] = [] -// const rootCA = await createTestRootCA() -// for (const userData of expected) { -// const { userCsr, userCert } = await createUserCertificateTestHelper( -// { nickname: userData.username, commonName: userData.onionAddress, peerId: userData.peerId }, -// rootCA -// ) -// if (['b', 'c'].includes(userData.username)) { -// certs.push(userCert!.userCertString) -// } -// if (['c', 'o'].includes(userData.username)) { -// csrs.push(userCsr.userCsr) -// } -// } - -// // const certs = [ -// // // b -// // 'MIICITCCAcegAwIBAgIGAY8GkBEVMAoGCCqGSM49BAMCMAwxCjAIBgNVBAMTAWEwHhcNMjQwNDIyMTYwNzM1WhcNMzAwMjAxMDcwMDAwWjBJMUcwRQYDVQQDEz56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDG8SNnoS1BYoV72jcyQFVlsrwvd2Bb9/9L13Tc4SHJwitTUB3F+y/7pk0tAPrZi2qasU2PO9lTwUxXYcAfpCRSjgdcwgdQwCQYDVR0TBAIwADALBgNVHQ8EBAMCAIAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBEGCisGAQQBg4wbAgEEAxMBYjA9BgkrBgECAQ8DAQEEMBMuUW1lUGJCMjVoMWZYN1dBRk42ckZSNGFWRFdVRlFNU3RSSEdERFM0UlFaUTRZcTBJBgNVHREEQjBAgj56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiBkTZo6/D0YgNMPcDpuf7n+rDEQls6cMVxEVw/H8vxbhwIhAM+e6we9YP4JeNgOGgd0iZNEpq8N7dla4XO+YVWrh0YG', - -// // // c -// // 'MIICITCCAcegAwIBAgIGAY8Glf+pMAoGCCqGSM49BAMCMAwxCjAIBgNVBAMTAWEwHhcNMjQwNDIyMTYxNDA0WhcNMzAwMjAxMDcwMDAwWjBJMUcwRQYDVQQDEz5uaGxpdWpuNjZlMzQ2ZXZ2dnhlYWY0cW1hN3Bxc2hjZ2J1NnQ3d3Nlb2FubmMyZnk0Y25zY3J5ZC5vbmlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP1WBKQdMz5yMpv5hWj6j+auIsnfiJE8dtuxeeM4N03K1An61F0o47CWg04DydwmoPn5gwefEv8t9Cz9nv/VUGejgdcwgdQwCQYDVR0TBAIwADALBgNVHQ8EBAMCAIAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBEGCisGAQQBg4wbAgEEAxMBYzA9BgkrBgECAQ8DAQEEMBMuUW1WY1hRTXVmRWNZS0R0d3NFSlRIUGJzc3BCeU02U0hUYlJHR2VEdkVFdU1RQTBJBgNVHREEQjBAgj5uaGxpdWpuNjZlMzQ2ZXZ2dnhlYWY0cW1hN3Bxc2hjZ2J1NnQ3d3Nlb2FubmMyZnk0Y25zY3J5ZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiEAgMCBxF3oK4ituEWcAK6uawMCludZu4YujIpBIR+v2LICIBhMHXrBy1KWc70t6idB+5XkInsRZz5nw1vwgRJ4mw98', -// // ] - -// // const csrs = [ -// // // c -// // 'MIIB4TCCAYgCAQAwSTFHMEUGA1UEAxM+emdoaWRleHM3cXQyNGl2dTNqb2JqcWR0enp3dHlhdTRscHBudW54NXBraWY3NnBrcHlwN3FjaWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQxvEjZ6EtQWKFe9o3MkBVZbK8L3dgW/f/S9d03OEhycIrU1Adxfsv+6ZNLQD62YtqmrFNjzvZU8FMV2HAH6QkUoIHcMC4GCSqGSIb3DQEJDjEhMB8wHQYDVR0OBBYEFG1W6vJTK/uPuRK2LPaVZyebVVc+MA8GCSqGSIb3DQEJDDECBAAwEQYKKwYBBAGDjBsCATEDEwFiMD0GCSsGAQIBDwMBATEwEy5RbWVQYkIyNWgxZlg3V0FGTjZyRlI0YVZEV1VGUU1TdFJIR0REUzRSUVpRNFlxMEcGA1UdETFAEz56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjAKBggqhkjOPQQDAgNHADBEAiAjxneoJZtCzkd75HTT+pcj+objG3S04omjeMMw1N+B/wIgAaJRgifnWEnWFYm614UmPw9un2Uwk1gVhN2tSwJ65sM=', - -// // // o -// // 'MIIDHjCCAsMCAQAwSTFHMEUGA1UEAxM+NnZ1MmJ4a2k3NzdpdDNjcGF5djZmcTZ2cGw0a2Uza3pqN2d4aWNmeWdtNTVkaGh0cGh5ZmR2eWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATMpfp2hSfWFL26OZlZKZEWG9fyAM1ndlEzO0kLxT0pA/7/fs+a5X/s4TkzqCVVQSzhas/84q0WE99ScAcM1LQJoIICFjAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBR6VRzktP1pzZxsGUaJivNUrtgSrzCCAUcGCSqGSIb3DQEJDDGCATgEggE0KZq9s6HEViRfplVgYkulg6XV411ZRe4U1UjfXTf1pRaygfcenGbT6RRagPtZzjuq5hHdYhqDjRzZhnbn8ZASYTgBM7qcseUq5UpS1pE08DI2jePKqatp3Pzm6a/MGSziESnREx784JlKfwKMjJl33UA8lQm9nhSeAIHyBx3c4Lf8IXdW2n3rnhbVfjpBMAxwh6lt+e5agtGXy+q/xAESUeLPfUgRYWctlLgt8Op+WTpLyBkZsVFoBvJrMt2XdM0RI32YzTRr56GXFa4VyQmY5xXwlQSPgidAP7jPkVygNcoeXvAz2ZCk3IR1Cn3mX8nMko53MlDNaMYldUQA0ug28/S7BlSlaq2CDD4Ol3swTq7C4KGTxKrI36ruYUZx7NEaQDF5V7VvqPCZ0fZoTIJuSYTQ67gwEQYKKwYBBAGDjBsCATEDEwFvMD0GCSsGAQIBDwMBATEwEy5RbVhSWTRyaEF4OE11cThkTUdrcjlxa25KZEU2VUhaRGRHYURSVFFFYndGTjViMEcGA1UdETFAEz42dnUyYnhraTc3N2l0M2NwYXl2NmZxNnZwbDRrZTNremo3Z3hpY2Z5Z201NWRoaHRwaHlmZHZ5ZC5vbmlvbjAKBggqhkjOPQQDAgNJADBGAiEAt+f1u/bchg5AZHv6NTGNoXeejTRWUhX3ioGwW6TGg84CIQCHqKNzDh2JjS/hUHx5PApAmfNnQTSf19X6LnNHQweU1g==', - -// // // o -// // 'MIIDHTCCAsMCAQAwSTFHMEUGA1UEAxM+eTd5Y3ptdWdsMnRla2FtaTdzYmR6NXBmYWVtdng3YmFod3RocmR2Y2J6dzV2ZXgyY3JzcjI2cWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATMq0l4bCmjdb0grtzpwtDVLM9E1IQpL9vrB4+lD9OBZzlrx2365jV7shVu9utas8w8fxtKoBZSnT5+32ZMFTB4oIICFjAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBSoDQpTZdEvi1/Rr/muVXT1clyKRDCCAUcGCSqGSIb3DQEJDDGCATgEggE0BQvyvkiiXEf/PLKnsR1Ba9AhYsVO8o56bnftUnoVzBlRZgUzLJvOSroPk/EmbVz+okhMrcYNgCWHvxrAqHVVq0JRP6bi98BtCUotx6OPFHp5K5QCL60hod1uAnhKocyJG9tsoM9aS+krn/k+g4RCBjiPZ25cC7QG/UNr6wyIQ8elBho4MKm8iOp7EShSsZOV1f6xrnXYCC/zyUc85GEuycLzVImgAQvPATbdMzY4zSGnNLHxkvSUNxaR9LnEWf+i1jeqcOiXOvmdyU5Be3ZqhGKvvBg/5vyLQiCIfeapjZemnLqFHQBitglDm2xnKL6HzMyfZoAHPV7YcWYR4spU9Ju8Q8aqSeAryx7sx55eSR4GO5UQTo5DrQn6xtkwOZ/ytsOknFthF8jcA9uTAMDKA2TylCUwEQYKKwYBBAGDjBsCATEDEwFvMD0GCSsGAQIBDwMBATEwEy5RbVQxOFV2blVCa3NlTWMzU3FuZlB4cEh3TjhuekxySmVOU0xadGM4ckFGWGh6MEcGA1UdETFAEz55N3ljem11Z2wydGVrYW1pN3NiZHo1cGZhZW12eDdiYWh3dGhyZHZjYnp3NXZleDJjcnNyMjZxZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiEAoFrAglxmk7ciD6AHQOB1qEoLu0NARcxgwmIry8oeTHwCICyXp5NJQ9Z8vReIAQNng2H2+/XjHifZEWzhoN0VkcBx', -// // ] - -// await storageService.init(peerId) -// // @ts-ignore -// storageService.certificatesRequestsStore = { -// getEntries: jest.fn(() => { -// return csrs -// }), -// } -// // @ts-ignore -// storageService.certificatesStore = { -// getEntries: jest.fn(() => { -// return certs -// }), -// } - -// const allUsers = await storageService.getAllUsers() - -// expect(allUsers).toStrictEqual(expected) -// }) -// }) - -// describe('Files deletion', () => { -// let realFilePath: string -// let messages: { -// messages: Record -// } - -// beforeEach(async () => { -// realFilePath = path.join(dirname, '/real-file.txt') -// createArbitraryFile(realFilePath, 2147483) -// await storageService.init(peerId) - -// const metadata: FileMetadata = { -// path: realFilePath, -// name: 'test-large-file', -// ext: '.txt', -// cid: 'uploading_id', -// message: { -// id: 'id', -// channelId: channel.id, -// }, -// } - -// const aliceMessage = await factory.create['payload']>( -// 'Message', -// { -// identity: alice, -// message: generateMessageFactoryContentWithId(channel.id, MessageType.File, metadata), -// } -// ) - -// messages = { -// messages: { -// [aliceMessage.message.id]: aliceMessage.message, -// }, -// } -// }) - -// afterEach(() => { -// if (fs.existsSync(realFilePath)) { -// fs.rmSync(realFilePath) -// } -// }) - -// it('delete file correctly', async () => { -// const isFileExist = await storageService.checkIfFileExist(realFilePath) -// expect(isFileExist).toBeTruthy() - -// await expect(storageService.deleteFilesFromChannel(messages)).resolves.not.toThrowError() - -// await waitForExpect(async () => { -// expect(await storageService.checkIfFileExist(realFilePath)).toBeFalsy() -// }, 2000) -// }) - -// it('file dont exist - not throw error', async () => { -// fs.rmSync(realFilePath) - -// await waitForExpect(async () => { -// expect(await storageService.checkIfFileExist(realFilePath)).toBeFalsy() -// }, 2000) - -// await expect(storageService.deleteFilesFromChannel(messages)).resolves.not.toThrowError() -// }) -// }) -// }) +import { jest } from '@jest/globals' + +import { Test, TestingModule } from '@nestjs/testing' +import { + prepareStore, + getFactory, + publicChannels, + generateMessageFactoryContentWithId, + Store, +} from '@quiet/state-manager' +import { ChannelMessage, Community, Identity, PublicChannel, TestMessage } from '@quiet/types' + +import path from 'path' +import { type PeerId } from '@libp2p/interface' +import { TestModule } from '../common/test.module' +import { libp2pInstanceParams } from '../common/utils' +import { IpfsModule } from '../ipfs/ipfs.module' +import { IpfsService } from '../ipfs/ipfs.service' +import { Libp2pModule } from '../libp2p/libp2p.module' +import { Libp2pService } from '../libp2p/libp2p.service' +import { SocketModule } from '../socket/socket.module' +import { StorageModule } from './storage.module' +import { StorageService } from './storage.service' +import fs from 'fs' +import { type FactoryGirl } from 'factory-girl' +import { fileURLToPath } from 'url' +import { LocalDbModule } from '../local-db/local-db.module' +import { LocalDbService } from '../local-db/local-db.service' +import { ORBIT_DB_DIR } from '../const' +import { createLogger } from '../common/logger' +import { createUserCertificateTestHelper, createTestRootCA } from '@quiet/identity' + +const logger = createLogger('storageService:test') + +const filename = fileURLToPath(import.meta.url) +const dirname = path.dirname(filename) + +describe('StorageService', () => { + let module: TestingModule + let storageService: StorageService + let ipfsService: IpfsService + let libp2pService: Libp2pService + let localDbService: LocalDbService + let peerId: PeerId + + let store: Store + let factory: FactoryGirl + let community: Community + let channel: PublicChannel + let alice: Identity + let john: Identity + let message: ChannelMessage + let channelio: PublicChannel + let filePath: string + let utils: any + let orbitDbDir: string + + jest.setTimeout(50000) + + beforeAll(async () => { + store = prepareStore().store + factory = await getFactory(store) + + community = await factory.create('Community') + + channel = publicChannels.selectors.publicChannels(store.getState())[0] + + channelio = { + name: channel.name, + description: channel.description, + owner: channel.owner, + timestamp: channel.timestamp, + id: channel.id, + } + + alice = await factory.create('Identity', { id: community.id, nickname: 'alice' }) + + john = await factory.create('Identity', { id: community.id, nickname: 'john' }) + + message = ( + await factory.create('Message', { + identity: alice, + message: generateMessageFactoryContentWithId(channel.id), + }) + ).message + }) + + beforeEach(async () => { + jest.clearAllMocks() + utils = await import('../common/utils') + filePath = path.join(dirname, '/500kB-file.txt') + + module = await Test.createTestingModule({ + imports: [TestModule, StorageModule, IpfsModule, SocketModule, Libp2pModule, LocalDbModule], + }).compile() + + storageService = await module.resolve(StorageService) + localDbService = await module.resolve(LocalDbService) + libp2pService = await module.resolve(Libp2pService) + ipfsService = await module.resolve(IpfsService) + + orbitDbDir = await module.resolve(ORBIT_DB_DIR) + + const params = await libp2pInstanceParams() + peerId = params.peerId.peerId + + await libp2pService.createInstance(params) + expect(libp2pService.libp2pInstance).not.toBeNull() + + await localDbService.open() + expect(localDbService.getStatus()).toEqual('open') + + await localDbService.setCommunity(community) + await localDbService.setCurrentCommunityId(community.id) + }) + + afterEach(async () => { + await libp2pService.libp2pInstance?.stop() + await ipfsService.ipfsInstance?.stop() + await storageService.stop() + if (fs.existsSync(filePath)) { + fs.rmSync(filePath) + } + await module.close() + }) + + it('should be defined', async () => { + await storageService.init(peerId) + }) + + describe('Storage', () => { + it('should not create paths if createPaths is set to false', async () => { + const orgProcessPlatform = process.platform + Object.defineProperty(process, 'platform', { + value: 'android', + }) + expect(fs.existsSync(orbitDbDir)).toBe(false) + + // FIXME: throws TypeError: Cannot assign to read only property 'createPaths' of object '[object Module]' and I can't be bothered to figure out how to get it to work + // const createPathsSpy = jest.spyOn(utils, 'createPaths') + + await storageService.init(peerId) + + // FIXME: throws TypeError: Cannot assign to read only property 'createPaths' of object '[object Module]' and I can't be bothered to figure out how to get it to work + // expect(createPathsSpy).not.toHaveBeenCalled() + + Object.defineProperty(process, 'platform', { + value: orgProcessPlatform, + }) + }) + + it('db address should be the same on all platforms', () => { + const dbAddress = StorageService.dbAddress({ root: 'zdpuABCDefgh123', path: 'channels.general_abcd' }) + expect(dbAddress).toEqual(`/orbitdb/zdpuABCDefgh123/channels.general_abcd`) + }) + }) + + describe('Users', () => { + it('gets all users from db', async () => { + const expected = [ + { + onionAddress: 'zghidexs7qt24ivu3jobjqdtzzwtyau4lppnunx5pkif76pkpyp7qcid.onion', + peerId: '12D3KooWKCWstmqi5gaQvipT7xVneVGfWV7HYpCbmUu626R92hXx', + username: 'b', + }, + { + onionAddress: 'nhliujn66e346evvvxeaf4qma7pqshcgbu6t7wseoannc2fy4cnscryd.onion', + peerId: '12D3KooWCXzUw71ovvkDky6XkV57aCWUV9JhJoKhoqXa1gdhFNoL', + username: 'c', + }, + { + onionAddress: '6vu2bxki777it3cpayv6fq6vpl4ke3kzj7gxicfygm55dhhtphyfdvyd.onion', + peerId: '12D3KooWEHzmff5kZAvyU6Diq5uJG8QkWJxFNUcBLuWjxUGvxaqw', + username: 'o', + }, + { + onionAddress: 'y7yczmugl2tekami7sbdz5pfaemvx7bahwthrdvcbzw5vex2crsr26qd.onion', + peerId: '12D3KooWHgLdRMqkepNiYnrur21cyASUNk1f9NZ5tuGa9He8QXNa', + username: 'o', + }, + ] + + const certs: string[] = [] + const csrs: string[] = [] + const rootCA = await createTestRootCA() + for (const userData of expected) { + const { userCsr, userCert } = await createUserCertificateTestHelper( + { nickname: userData.username, commonName: userData.onionAddress, peerId: userData.peerId }, + rootCA + ) + if (['b', 'c'].includes(userData.username)) { + certs.push(userCert!.userCertString) + } + if (['c', 'o'].includes(userData.username)) { + csrs.push(userCsr.userCsr) + } + } + + // const certs = [ + // // b + // 'MIICITCCAcegAwIBAgIGAY8GkBEVMAoGCCqGSM49BAMCMAwxCjAIBgNVBAMTAWEwHhcNMjQwNDIyMTYwNzM1WhcNMzAwMjAxMDcwMDAwWjBJMUcwRQYDVQQDEz56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDG8SNnoS1BYoV72jcyQFVlsrwvd2Bb9/9L13Tc4SHJwitTUB3F+y/7pk0tAPrZi2qasU2PO9lTwUxXYcAfpCRSjgdcwgdQwCQYDVR0TBAIwADALBgNVHQ8EBAMCAIAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBEGCisGAQQBg4wbAgEEAxMBYjA9BgkrBgECAQ8DAQEEMBMuUW1lUGJCMjVoMWZYN1dBRk42ckZSNGFWRFdVRlFNU3RSSEdERFM0UlFaUTRZcTBJBgNVHREEQjBAgj56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiBkTZo6/D0YgNMPcDpuf7n+rDEQls6cMVxEVw/H8vxbhwIhAM+e6we9YP4JeNgOGgd0iZNEpq8N7dla4XO+YVWrh0YG', + + // // c + // 'MIICITCCAcegAwIBAgIGAY8Glf+pMAoGCCqGSM49BAMCMAwxCjAIBgNVBAMTAWEwHhcNMjQwNDIyMTYxNDA0WhcNMzAwMjAxMDcwMDAwWjBJMUcwRQYDVQQDEz5uaGxpdWpuNjZlMzQ2ZXZ2dnhlYWY0cW1hN3Bxc2hjZ2J1NnQ3d3Nlb2FubmMyZnk0Y25zY3J5ZC5vbmlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP1WBKQdMz5yMpv5hWj6j+auIsnfiJE8dtuxeeM4N03K1An61F0o47CWg04DydwmoPn5gwefEv8t9Cz9nv/VUGejgdcwgdQwCQYDVR0TBAIwADALBgNVHQ8EBAMCAIAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBEGCisGAQQBg4wbAgEEAxMBYzA9BgkrBgECAQ8DAQEEMBMuUW1WY1hRTXVmRWNZS0R0d3NFSlRIUGJzc3BCeU02U0hUYlJHR2VEdkVFdU1RQTBJBgNVHREEQjBAgj5uaGxpdWpuNjZlMzQ2ZXZ2dnhlYWY0cW1hN3Bxc2hjZ2J1NnQ3d3Nlb2FubmMyZnk0Y25zY3J5ZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiEAgMCBxF3oK4ituEWcAK6uawMCludZu4YujIpBIR+v2LICIBhMHXrBy1KWc70t6idB+5XkInsRZz5nw1vwgRJ4mw98', + // ] + + // const csrs = [ + // // c + // 'MIIB4TCCAYgCAQAwSTFHMEUGA1UEAxM+emdoaWRleHM3cXQyNGl2dTNqb2JqcWR0enp3dHlhdTRscHBudW54NXBraWY3NnBrcHlwN3FjaWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQxvEjZ6EtQWKFe9o3MkBVZbK8L3dgW/f/S9d03OEhycIrU1Adxfsv+6ZNLQD62YtqmrFNjzvZU8FMV2HAH6QkUoIHcMC4GCSqGSIb3DQEJDjEhMB8wHQYDVR0OBBYEFG1W6vJTK/uPuRK2LPaVZyebVVc+MA8GCSqGSIb3DQEJDDECBAAwEQYKKwYBBAGDjBsCATEDEwFiMD0GCSsGAQIBDwMBATEwEy5RbWVQYkIyNWgxZlg3V0FGTjZyRlI0YVZEV1VGUU1TdFJIR0REUzRSUVpRNFlxMEcGA1UdETFAEz56Z2hpZGV4czdxdDI0aXZ1M2pvYmpxZHR6end0eWF1NGxwcG51bng1cGtpZjc2cGtweXA3cWNpZC5vbmlvbjAKBggqhkjOPQQDAgNHADBEAiAjxneoJZtCzkd75HTT+pcj+objG3S04omjeMMw1N+B/wIgAaJRgifnWEnWFYm614UmPw9un2Uwk1gVhN2tSwJ65sM=', + + // // o + // 'MIIDHjCCAsMCAQAwSTFHMEUGA1UEAxM+NnZ1MmJ4a2k3NzdpdDNjcGF5djZmcTZ2cGw0a2Uza3pqN2d4aWNmeWdtNTVkaGh0cGh5ZmR2eWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATMpfp2hSfWFL26OZlZKZEWG9fyAM1ndlEzO0kLxT0pA/7/fs+a5X/s4TkzqCVVQSzhas/84q0WE99ScAcM1LQJoIICFjAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBR6VRzktP1pzZxsGUaJivNUrtgSrzCCAUcGCSqGSIb3DQEJDDGCATgEggE0KZq9s6HEViRfplVgYkulg6XV411ZRe4U1UjfXTf1pRaygfcenGbT6RRagPtZzjuq5hHdYhqDjRzZhnbn8ZASYTgBM7qcseUq5UpS1pE08DI2jePKqatp3Pzm6a/MGSziESnREx784JlKfwKMjJl33UA8lQm9nhSeAIHyBx3c4Lf8IXdW2n3rnhbVfjpBMAxwh6lt+e5agtGXy+q/xAESUeLPfUgRYWctlLgt8Op+WTpLyBkZsVFoBvJrMt2XdM0RI32YzTRr56GXFa4VyQmY5xXwlQSPgidAP7jPkVygNcoeXvAz2ZCk3IR1Cn3mX8nMko53MlDNaMYldUQA0ug28/S7BlSlaq2CDD4Ol3swTq7C4KGTxKrI36ruYUZx7NEaQDF5V7VvqPCZ0fZoTIJuSYTQ67gwEQYKKwYBBAGDjBsCATEDEwFvMD0GCSsGAQIBDwMBATEwEy5RbVhSWTRyaEF4OE11cThkTUdrcjlxa25KZEU2VUhaRGRHYURSVFFFYndGTjViMEcGA1UdETFAEz42dnUyYnhraTc3N2l0M2NwYXl2NmZxNnZwbDRrZTNremo3Z3hpY2Z5Z201NWRoaHRwaHlmZHZ5ZC5vbmlvbjAKBggqhkjOPQQDAgNJADBGAiEAt+f1u/bchg5AZHv6NTGNoXeejTRWUhX3ioGwW6TGg84CIQCHqKNzDh2JjS/hUHx5PApAmfNnQTSf19X6LnNHQweU1g==', + + // // o + // 'MIIDHTCCAsMCAQAwSTFHMEUGA1UEAxM+eTd5Y3ptdWdsMnRla2FtaTdzYmR6NXBmYWVtdng3YmFod3RocmR2Y2J6dzV2ZXgyY3JzcjI2cWQub25pb24wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATMq0l4bCmjdb0grtzpwtDVLM9E1IQpL9vrB4+lD9OBZzlrx2365jV7shVu9utas8w8fxtKoBZSnT5+32ZMFTB4oIICFjAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBSoDQpTZdEvi1/Rr/muVXT1clyKRDCCAUcGCSqGSIb3DQEJDDGCATgEggE0BQvyvkiiXEf/PLKnsR1Ba9AhYsVO8o56bnftUnoVzBlRZgUzLJvOSroPk/EmbVz+okhMrcYNgCWHvxrAqHVVq0JRP6bi98BtCUotx6OPFHp5K5QCL60hod1uAnhKocyJG9tsoM9aS+krn/k+g4RCBjiPZ25cC7QG/UNr6wyIQ8elBho4MKm8iOp7EShSsZOV1f6xrnXYCC/zyUc85GEuycLzVImgAQvPATbdMzY4zSGnNLHxkvSUNxaR9LnEWf+i1jeqcOiXOvmdyU5Be3ZqhGKvvBg/5vyLQiCIfeapjZemnLqFHQBitglDm2xnKL6HzMyfZoAHPV7YcWYR4spU9Ju8Q8aqSeAryx7sx55eSR4GO5UQTo5DrQn6xtkwOZ/ytsOknFthF8jcA9uTAMDKA2TylCUwEQYKKwYBBAGDjBsCATEDEwFvMD0GCSsGAQIBDwMBATEwEy5RbVQxOFV2blVCa3NlTWMzU3FuZlB4cEh3TjhuekxySmVOU0xadGM4ckFGWGh6MEcGA1UdETFAEz55N3ljem11Z2wydGVrYW1pN3NiZHo1cGZhZW12eDdiYWh3dGhyZHZjYnp3NXZleDJjcnNyMjZxZC5vbmlvbjAKBggqhkjOPQQDAgNIADBFAiEAoFrAglxmk7ciD6AHQOB1qEoLu0NARcxgwmIry8oeTHwCICyXp5NJQ9Z8vReIAQNng2H2+/XjHifZEWzhoN0VkcBx', + // ] + + await storageService.init(peerId) + // @ts-ignore + storageService.certificatesRequestsStore = { + getEntries: jest.fn(() => { + return csrs + }), + } + // @ts-ignore + storageService.certificatesStore = { + getEntries: jest.fn(() => { + return certs + }), + } + + const allUsers = await storageService.getAllUsers() + + expect(allUsers).toStrictEqual(expected) + }) + }) +}) diff --git a/packages/backend/src/nest/storage/storage.service.ts b/packages/backend/src/nest/storage/storage.service.ts index d5ab6a2f86..a4ac532a61 100644 --- a/packages/backend/src/nest/storage/storage.service.ts +++ b/packages/backend/src/nest/storage/storage.service.ts @@ -321,14 +321,6 @@ export class StorageService extends EventEmitter { await this.userProfileStore.setEntry(profile.pubKey, profile) } - public async checkIfFileExist(filepath: string): Promise { - return await new Promise(resolve => { - fs.access(filepath, fs.constants.F_OK, error => { - resolve(!error) - }) - }) - } - public async setIdentity(identity: Identity) { await this.localDbService.setIdentity(identity) this.emit(SocketActionTypes.IDENTITY_STORED, identity) diff --git a/packages/types/src/channel.ts b/packages/types/src/channel.ts index e4c0eb782f..05c12c41ea 100644 --- a/packages/types/src/channel.ts +++ b/packages/types/src/channel.ts @@ -42,6 +42,10 @@ export interface ChannelMessage { media?: FileMetadata } +export interface ConsumedChannelMessage extends ChannelMessage { + verified?: boolean +} + export interface DisplayableMessage { id: string type: number From d494fe2c24ee083834baeb6b5b4f70c28342ac99 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Tue, 28 Jan 2025 10:24:41 -0500 Subject: [PATCH 06/10] Fix subscribing not getting old messages on join --- .../nest/storage/channels/channel.store.ts | 57 +++++++++++++------ .../nest/storage/channels/channels.service.ts | 17 +++--- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/nest/storage/channels/channel.store.ts b/packages/backend/src/nest/storage/channels/channel.store.ts index ec38fbca33..842e49bb48 100644 --- a/packages/backend/src/nest/storage/channels/channel.store.ts +++ b/packages/backend/src/nest/storage/channels/channel.store.ts @@ -22,7 +22,8 @@ import { CertificatesStore } from '../certificates/certificates.store' @Injectable() export class ChannelStore extends EventStoreBase { private channelData: PublicChannel - private initialized: boolean = false + private _subscribing: boolean = false + private logger: QuietLogger constructor( @@ -44,7 +45,7 @@ export class ChannelStore extends EventStoreBase { * @returns Initialized ChannelStore instance */ public async init(channelData: PublicChannel, options: DBOptions): Promise { - if (this.initialized) { + if (this.store != null) { this.logger.warn(`Channel ${this.channelData.name} has already been initialized!`) return this } @@ -61,7 +62,6 @@ export class ChannelStore extends EventStoreBase { }) this.logger.info('Initialized') - this.initialized = true return this } @@ -72,6 +72,12 @@ export class ChannelStore extends EventStoreBase { await this.getStore().sync.start() } + // Accessors + + public get isSubscribing(): boolean { + return this._subscribing + } + /** * Subscribe to new messages on this channel * @@ -81,6 +87,7 @@ export class ChannelStore extends EventStoreBase { */ public async subscribe(): Promise { this.logger.info('Subscribing to channel ', this.channelData.id) + this._subscribing = true this.getStore().events.on('update', async (entry: LogEntry) => { this.logger.info(`${this.channelData.id} database updated`, entry.hash, entry.payload.value?.channelId) @@ -92,16 +99,7 @@ export class ChannelStore extends EventStoreBase { isVerified: message.verified, }) - const ids = (await this.getEntries()).map(msg => msg.id) - const community = await this.localDbService.getCurrentCommunity() - - if (community) { - this.emit(StorageEvents.MESSAGE_IDS_STORED, { - ids, - channelId: this.channelData.id, - communityId: community.id, - }) - } + await this.refreshMessageIds() // FIXME: the 'update' event runs if we replicate entries and if we add // entries ourselves. So we may want to check if the message is written @@ -130,6 +128,8 @@ export class ChannelStore extends EventStoreBase { }) await this.startSync() + await this.refreshMessageIds() + this._subscribing = false this.logger.info(`Subscribed to channel ${this.channelData.id}`) } @@ -163,7 +163,7 @@ export class ChannelStore extends EventStoreBase { } /** - * Read messages from OrbitDB filtered by message ID + * Read messages from OrbitDB, optionally filtered by message ID * * @param ids Message IDs to read from this channel's OrbitDB database * @returns Messages read from OrbitDB @@ -176,6 +176,24 @@ export class ChannelStore extends EventStoreBase { } } + /** + * Get the latest state of messages in OrbitDB and emit an event to trigger redux updates + * + * @emits StorageEvents.MESSAGE_IDS_STORED + */ + private async refreshMessageIds(): Promise { + const ids = (await this.getEntries()).map(msg => msg.id) + const community = await this.localDbService.getCurrentCommunity() + + if (community) { + this.emit(StorageEvents.MESSAGE_IDS_STORED, { + ids, + channelId: this.channelData.id, + communityId: community.id, + }) + } + } + // Base Store Logic /** @@ -195,9 +213,10 @@ export class ChannelStore extends EventStoreBase { } /** - * Read all entries on the OrbitDB event store + * Read a list of entries on the OrbitDB event store * - * @returns All entries on the event store + * @param ids Optional list of message IDs to filter by + * @returns All matching entries on the event store */ public async getEntries(): Promise public async getEntries(ids: string[] | undefined): Promise @@ -206,7 +225,9 @@ export class ChannelStore extends EventStoreBase { const messages: ChannelMessage[] = [] for await (const x of this.getStore().iterator()) { - if (ids == null || ids?.includes(x.id)) { + if (ids == null || ids?.includes(x.value.id)) { + // NOTE: we skipped the verification process when reading many messages in the previous version + // so I'm skipping it here - is that really the correct behavior? const processedMessage = await this.messagesService.onConsume(x.value, false) messages.push(processedMessage) } @@ -256,6 +277,6 @@ export class ChannelStore extends EventStoreBase { public clean(): void { this.logger.info(`Cleaning channel store`, this.channelData.id, this.channelData.name) this.store = undefined - this.initialized = false + this._subscribing = false } } diff --git a/packages/backend/src/nest/storage/channels/channels.service.ts b/packages/backend/src/nest/storage/channels/channels.service.ts index 8ec9eba158..824742f6fd 100644 --- a/packages/backend/src/nest/storage/channels/channels.service.ts +++ b/packages/backend/src/nest/storage/channels/channels.service.ts @@ -130,7 +130,9 @@ export class ChannelsService extends EventEmitter { 'Channels names:', channels.map(x => x.name) ) - channels.forEach(channel => this.subscribeToChannel(channel)) + for (const channel of channels.values()) { + await this.subscribeToChannel(channel) + } } /** @@ -260,12 +262,12 @@ export class ChannelsService extends EventEmitter { return } repo = this.publicChannelsRepos.get(channelData.id) + } - if (repo && !repo.eventsAttached) { - this.handleMessageEventsOnChannelStore(channelData.id, repo) - await repo.store.subscribe() - repo.eventsAttached = true - } + if (repo && !repo.eventsAttached && !repo.store.isSubscribing) { + this.handleMessageEventsOnChannelStore(channelData.id, repo) + await repo.store.subscribe() + repo.eventsAttached = true } this.logger.info(`Subscribed to channel ${channelData.id}`) @@ -287,17 +289,14 @@ export class ChannelsService extends EventEmitter { private handleMessageEventsOnChannelStore(channelId: string, repo: PublicChannelsRepo): void { this.logger.info(`Subscribing to channel updates`, channelId) repo.store.on(StorageEvents.MESSAGE_IDS_STORED, (payload: ChannelMessageIdsResponse) => { - this.logger.info(`Emitting ${StorageEvents.MESSAGE_IDS_STORED}`) this.emit(StorageEvents.MESSAGE_IDS_STORED, payload) }) repo.store.on(StorageEvents.MESSAGES_STORED, (payload: MessagesLoadedPayload) => { - this.logger.info(`Emitting ${StorageEvents.MESSAGES_STORED}`) this.emit(StorageEvents.MESSAGES_STORED, payload) }) repo.store.on(StorageEvents.SEND_PUSH_NOTIFICATION, (payload: PushNotificationPayload) => { - this.logger.info(`Emitting ${StorageEvents.SEND_PUSH_NOTIFICATION}`) this.emit(StorageEvents.SEND_PUSH_NOTIFICATION, payload) }) } From b9851eeeb45bc60468cff486d58c4e7960514f96 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 31 Jan 2025 11:24:56 -0500 Subject: [PATCH 07/10] Fix peer list not updating --- .../connections-manager.service.ts | 22 ++++++++++--------- .../nest/storage/channels/channel.store.ts | 12 +++++----- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/packages/backend/src/nest/connections-manager/connections-manager.service.ts b/packages/backend/src/nest/connections-manager/connections-manager.service.ts index b0bdfcc75e..6f8b1d8d21 100644 --- a/packages/backend/src/nest/connections-manager/connections-manager.service.ts +++ b/packages/backend/src/nest/connections-manager/connections-manager.service.ts @@ -1051,13 +1051,7 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI private attachStorageListeners() { if (!this.storageService) return - this.storageService.on(SocketActionTypes.CONNECTION_PROCESS_INFO, data => { - this.serverIoProvider.io.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, data) - }) - this.storageService.on(StorageEvents.CERTIFICATES_STORED, (payload: SendCertificatesResponse) => { - this.logger.info(`Storage - ${StorageEvents.CERTIFICATES_STORED}`) - this.serverIoProvider.io.emit(SocketActionTypes.CERTIFICATES_STORED, payload) - }) + // Channel and Message Events this.storageService.channels.on(StorageEvents.CHANNELS_STORED, (payload: ChannelsReplicatedPayload) => { this.serverIoProvider.io.emit(SocketActionTypes.CHANNELS_STORED, payload) }) @@ -1085,12 +1079,20 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI this.storageService.channels.on(StorageEvents.MESSAGE_MEDIA_UPDATED, (payload: FileMetadata) => { this.serverIoProvider.io.emit(SocketActionTypes.MESSAGE_MEDIA_UPDATED, payload) }) - this.storageService.channels.on(StorageEvents.COMMUNITY_UPDATED, (payload: Community) => { - this.serverIoProvider.io.emit(SocketActionTypes.COMMUNITY_UPDATED, payload) - }) this.storageService.channels.on(StorageEvents.SEND_PUSH_NOTIFICATION, (payload: PushNotificationPayload) => { this.serverIoProvider.io.emit(SocketActionTypes.PUSH_NOTIFICATION, payload) }) + // Other Events + this.storageService.on(SocketActionTypes.CONNECTION_PROCESS_INFO, data => { + this.serverIoProvider.io.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, data) + }) + this.storageService.on(StorageEvents.CERTIFICATES_STORED, (payload: SendCertificatesResponse) => { + this.logger.info(`Storage - ${StorageEvents.CERTIFICATES_STORED}`) + this.serverIoProvider.io.emit(SocketActionTypes.CERTIFICATES_STORED, payload) + }) + this.storageService.on(StorageEvents.COMMUNITY_UPDATED, (payload: Community) => { + this.serverIoProvider.io.emit(SocketActionTypes.COMMUNITY_UPDATED, payload) + }) this.storageService.on(StorageEvents.CSRS_STORED, async (payload: { csrs: string[] }) => { this.logger.info(`Storage - ${StorageEvents.CSRS_STORED}`) const users = await getUsersFromCsrs(payload.csrs) diff --git a/packages/backend/src/nest/storage/channels/channel.store.ts b/packages/backend/src/nest/storage/channels/channel.store.ts index 842e49bb48..cd25b379af 100644 --- a/packages/backend/src/nest/storage/channels/channel.store.ts +++ b/packages/backend/src/nest/storage/channels/channel.store.ts @@ -203,13 +203,13 @@ export class ChannelStore extends EventStoreBase { * @returns Hash of the new database entry */ public async addEntry(message: ChannelMessage): Promise { - if (!this.store) { - throw new Error('Store is not initialized') - } - this.logger.info('Adding message to database') - const processedMessage = await this.messagesService.onSend(message) - return await this.store.add(processedMessage) + const encryptedMessage = await this.messagesService.onSend(message) + try { + return await this.getStore().add(encryptedMessage) + } catch (e) { + throw new CompoundError(`Could not append message (entry not allowed to write to the log)`, e) + } } /** From 03cf584370968e75f7389c10dca766373ed81c0d Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 31 Jan 2025 15:01:46 -0500 Subject: [PATCH 08/10] Add compounderror --- .../backend/src/nest/storage/channels/channel.store.ts | 8 +++++++- packages/types/src/errors.ts | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/nest/storage/channels/channel.store.ts b/packages/backend/src/nest/storage/channels/channel.store.ts index cd25b379af..0ab02260cb 100644 --- a/packages/backend/src/nest/storage/channels/channel.store.ts +++ b/packages/backend/src/nest/storage/channels/channel.store.ts @@ -3,7 +3,13 @@ import { Injectable } from '@nestjs/common' import { EventsType, LogEntry } from '@orbitdb/core' import { QuietLogger } from '@quiet/logger' -import { ChannelMessage, MessagesLoadedPayload, PublicChannel, PushNotificationPayload } from '@quiet/types' +import { + ChannelMessage, + CompoundError, + MessagesLoadedPayload, + PublicChannel, + PushNotificationPayload, +} from '@quiet/types' import { createLogger } from '../../common/logger' import { EventStoreBase } from '../base.store' diff --git a/packages/types/src/errors.ts b/packages/types/src/errors.ts index 4f70fcec8b..90cda5ee87 100644 --- a/packages/types/src/errors.ts +++ b/packages/types/src/errors.ts @@ -53,3 +53,12 @@ export enum ErrorMessages { // Storage Server STORAGE_SERVER_CONNECTION_FAILED = 'Connecting to storage server failed', } + +export class CompoundError extends Error { + constructor( + message: string, + public readonly originalError: T + ) { + super(message) + } +} From da141d1a3b3aa57b9e00fb9931c3080d07a1ecdc Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Tue, 4 Feb 2025 15:07:22 -0500 Subject: [PATCH 09/10] Update auth --- 3rd-party/auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party/auth b/3rd-party/auth index fd7101145f..56cf854389 160000 --- a/3rd-party/auth +++ b/3rd-party/auth @@ -1 +1 @@ -Subproject commit fd7101145fc15aeb14bda46578b7a4d6d84e4e5b +Subproject commit 56cf85438990004975f34bf121bdef22d4b81068 From 1b2cc2b140d9cb441454b00195608d553600dcce Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Tue, 4 Feb 2025 15:23:42 -0500 Subject: [PATCH 10/10] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6888216b3..636ebdcaa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Chores * Add `trace` level logs to `@quiet/logger` ([#2716](https://github.com/TryQuiet/quiet/issues/2716)) +* Refactor the `StorageService` and create `ChannelService`, `MessageService` and `ChannelStore` for handling channel-related persistence ([#2631](https://github.com/TryQuiet/quiet/issues/2631)) ## [3.0.0]