From ca47aaa9ab66ad0f2161df81067a0b25ab4f7bd9 Mon Sep 17 00:00:00 2001 From: Igor Shadurin Date: Wed, 6 Dec 2023 16:08:33 +0300 Subject: [PATCH] feat: ability to upload any amount of pods --- package-lock.json | 26 ++- package.json | 4 +- src/account/index.ts | 1 - src/content-items/utils.ts | 40 ++-- src/feed/api.ts | 18 +- src/file/file.ts | 2 +- src/file/handler.ts | 128 +++++++---- src/file/types.ts | 14 ++ src/file/utils.ts | 19 +- src/pod/api.ts | 41 ++-- src/pod/cache/api.ts | 7 +- src/pod/personal-storage.ts | 44 ++-- src/pod/types.ts | 15 ++ src/pod/utils.ts | 200 ++++++++++++++++-- src/utils/compression.ts | 17 ++ src/utils/hex.ts | 1 - test/integration/node/fdp-class.spec.ts | 31 +-- .../node/pod/pod-limit-error-message.spec.ts | 18 -- .../node/pod/pods-limitation-check.spec.ts | 15 ++ test/unit/feed/api.spec.ts | 24 +++ test/unit/feed/epoch.spec.ts | 19 +- 21 files changed, 522 insertions(+), 162 deletions(-) create mode 100644 src/utils/compression.ts delete mode 100644 test/integration/node/pod/pod-limit-error-message.spec.ts create mode 100644 test/integration/node/pod/pods-limitation-check.spec.ts create mode 100644 test/unit/feed/api.spec.ts diff --git a/package-lock.json b/package-lock.json index 98df323c..c0523b5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "@fairdatasociety/fdp-contracts-js": "^3.8.0", "crypto-js": "^4.2.0", "ethers": "^5.5.2", - "js-sha3": "^0.9.2" + "js-sha3": "^0.9.2", + "pako": "^2.1.0" }, "devDependencies": { "@babel/core": "^7.23.2", @@ -35,6 +36,7 @@ "@types/jest": "^29.5.6", "@types/jest-environment-puppeteer": "^5.0.5", "@types/node": "^20.8.9", + "@types/pako": "^2.0.3", "@types/webpack-bundle-analyzer": "^4.6.2", "@types/ws": "^8.5.8", "@typescript-eslint/eslint-plugin": "^6.9.0", @@ -4849,6 +4851,12 @@ "integrity": "sha512-ehPtgRgaULsFG8x0NeYJvmyH1hmlfsNLujHe9dQEia/7MAJYdzMSi19JtchUHjmBA6XC/75dK55mzZH+RyieSg==", "dev": true }, + "node_modules/@types/pako": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.3.tgz", + "integrity": "sha512-bq0hMV9opAcrmE0Byyo0fY3Ew4tgOevJmQ9grUhpXQhYfyLJ1Kqg3P33JT5fdbT2AjeAjR51zqqVjAL/HMkx7Q==", + "dev": true + }, "node_modules/@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -12590,6 +12598,11 @@ "node": ">= 14" } }, + "node_modules/pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -18796,6 +18809,12 @@ "integrity": "sha512-ehPtgRgaULsFG8x0NeYJvmyH1hmlfsNLujHe9dQEia/7MAJYdzMSi19JtchUHjmBA6XC/75dK55mzZH+RyieSg==", "dev": true }, + "@types/pako": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.3.tgz", + "integrity": "sha512-bq0hMV9opAcrmE0Byyo0fY3Ew4tgOevJmQ9grUhpXQhYfyLJ1Kqg3P33JT5fdbT2AjeAjR51zqqVjAL/HMkx7Q==", + "dev": true + }, "@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -24417,6 +24436,11 @@ "netmask": "^2.0.2" } }, + "pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==" + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/package.json b/package.json index 2bd0fcc4..a96b51d0 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,8 @@ "@fairdatasociety/fdp-contracts-js": "^3.8.0", "crypto-js": "^4.2.0", "ethers": "^5.5.2", - "js-sha3": "^0.9.2" + "js-sha3": "^0.9.2", + "pako": "^2.1.0" }, "devDependencies": { "@babel/core": "^7.23.2", @@ -79,6 +80,7 @@ "@types/jest": "^29.5.6", "@types/jest-environment-puppeteer": "^5.0.5", "@types/node": "^20.8.9", + "@types/pako": "^2.0.3", "@types/webpack-bundle-analyzer": "^4.6.2", "@types/ws": "^8.5.8", "@typescript-eslint/eslint-plugin": "^6.9.0", diff --git a/src/account/index.ts b/src/account/index.ts index e62083bb..69065539 100644 --- a/src/account/index.ts +++ b/src/account/index.ts @@ -1,4 +1,3 @@ export * as Account from './account' export * as Encryption from '../utils/encryption' export * as Utils from './utils' -//TODO export every exportable diff --git a/src/content-items/utils.ts b/src/content-items/utils.ts index 1a4a7ccc..3c193f5e 100644 --- a/src/content-items/utils.ts +++ b/src/content-items/utils.ts @@ -1,4 +1,4 @@ -import { Bee, Reference, BeeRequestOptions } from '@ethersphere/bee-js' +import { Bee, Reference, BeeRequestOptions, Data } from '@ethersphere/bee-js' import { EthAddress } from '@ethersphere/bee-js/dist/types/utils/eth' import { RawDirectoryMetadata, RawFileMetadata } from '../pod/types' import { DELETE_FEED_MAGIC_WORD, getFeedData, writeFeedData } from '../feed/api' @@ -12,6 +12,32 @@ import { utils, Wallet } from 'ethers' import { Epoch } from '../feed/lookup/epoch' import { stringToBytes } from '../utils/bytes' +/** + * Extracts metadata from encrypted source data using a pod password + * + * @param {Data} sourceData - The encrypted source data from which to extract metadata. + * @param {PodPasswordBytes} podPassword - The pod password used to decrypt the source data. + * @throws {Error} If the metadata is invalid. + * @returns {RawDirectoryMetadata | RawFileMetadata} The extracted metadata. + */ +export function extractMetadata( + sourceData: Data, + podPassword: PodPasswordBytes, +): RawDirectoryMetadata | RawFileMetadata { + const data = decryptJson(podPassword, sourceData) + let metadata + + if (isRawDirectoryMetadata(data)) { + metadata = data as RawDirectoryMetadata + } else if (isRawFileMetadata(data)) { + metadata = data as RawFileMetadata + } else { + throw new Error('Invalid metadata') + } + + return metadata +} + /** * Get raw metadata by path * @@ -29,20 +55,10 @@ export async function getRawMetadata( requestOptions?: BeeRequestOptions, ): Promise { const feedData = await getFeedData(bee, path, address, requestOptions) - const data = decryptJson(podPassword, feedData.data.chunkContent()) - let metadata - - if (isRawDirectoryMetadata(data)) { - metadata = data as RawDirectoryMetadata - } else if (isRawFileMetadata(data)) { - metadata = data as RawFileMetadata - } else { - throw new Error('Invalid metadata') - } return { epoch: feedData.epoch, - metadata, + metadata: extractMetadata(feedData.data.chunkContent(), podPassword), } } diff --git a/src/feed/api.ts b/src/feed/api.ts index 743faaa7..da1898c0 100644 --- a/src/feed/api.ts +++ b/src/feed/api.ts @@ -2,7 +2,7 @@ import { Bee, Data, Reference, BeeRequestOptions, Utils } from '@ethersphere/bee import { bmtHashString } from '../account/utils' import { getId } from './handler' import { lookup } from './lookup/linear' -import { Epoch, HIGHEST_LEVEL } from './lookup/epoch' +import { Epoch, getFirstEpoch } from './lookup/epoch' import { bytesToHex } from '../utils/hex' import { getUnixTimestamp } from '../utils/time' import { LookupAnswer } from './types' @@ -57,14 +57,22 @@ export async function writeFeedData( podPassword: PodPasswordBytes, epoch?: Epoch, ): Promise { - if (!epoch) { - epoch = new Epoch(HIGHEST_LEVEL, getUnixTimestamp()) - } + epoch = prepareEpoch(epoch) data = encryptBytes(podPassword, data) - const topicHash = bmtHashString(topic) const id = getId(topicHash, epoch.time, epoch.level) const socWriter = connection.bee.makeSOCWriter(wallet.privateKey) return socWriter.upload(connection.postageBatchId, id, data) } + +/** + * Prepares an epoch for further processing. + * + * @param {Epoch} [epoch] - The epoch to prepare. If not provided, a new epoch will be created. + * + * @return {Epoch} The prepared epoch. + */ +export function prepareEpoch(epoch?: Epoch): Epoch { + return epoch ?? getFirstEpoch(getUnixTimestamp()) +} diff --git a/src/file/file.ts b/src/file/file.ts index 0b72cb81..77e5d774 100644 --- a/src/file/file.ts +++ b/src/file/file.ts @@ -78,7 +78,7 @@ export class File { assertAccount(this.accountData, { writeRequired: true }) assertPodName(podName) - return uploadData(podName, fullPath, data, this.accountData, options) + return (await uploadData(podName, fullPath, data, this.accountData, options)).meta } /** diff --git a/src/file/handler.ts b/src/file/handler.ts index eb55d6d3..1c3bfc83 100644 --- a/src/file/handler.ts +++ b/src/file/handler.ts @@ -1,5 +1,5 @@ import { assertMinLength, stringToBytes, wrapBytesWithHelpers } from '../utils/bytes' -import { Bee, Data, BeeRequestOptions } from '@ethersphere/bee-js' +import { Bee, BeeRequestOptions, Data } from '@ethersphere/bee-js' import { EthAddress } from '@ethersphere/bee-js/dist/types/utils/eth' import { assertFullPathWithName, @@ -16,7 +16,7 @@ import { updateUploadProgress, uploadBytes, } from './utils' -import { FileMetadata } from '../pod/types' +import { FileMetadata, FileMetadataWithLookupAnswer } from '../pod/types' import { blocksToManifest, getFileMetadataRawBytes, rawFileMetadataToFileMetadata } from './adapter' import { assertRawFileMetadata } from '../directory/utils' import { getCreationPathInfo, getRawMetadata } from '../content-items/utils' @@ -24,6 +24,7 @@ import { PodPasswordBytes } from '../utils/encryption' import { Block, Blocks, + Compression, DataDownloadOptions, DataUploadOptions, DownloadProgressType, @@ -34,12 +35,14 @@ import { import { assertPodName, getExtendedPodsListByAccountData, META_VERSION } from '../pod/utils' import { getUnixTimestamp } from '../utils/time' import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS, MINIMUM_BLOCK_SIZE } from '../content-items/handler' -import { writeFeedData } from '../feed/api' +import { prepareEpoch, writeFeedData } from '../feed/api' import { AccountData } from '../account/account-data' -import { prepareEthAddress } from '../utils/wallet' +import { prepareEthAddress, preparePrivateKey } from '../utils/wallet' import { assertWallet } from '../utils/type' import { getNextEpoch } from '../feed/lookup/utils' import { Connection } from '../connection/connection' +import { compress } from '../utils/compression' +import { wrapChunkHelper } from '../feed/utils' /** * File prefix @@ -111,32 +114,21 @@ export async function getFileMetadataWithBlocks( } /** - * Downloads file parts and compile them into Data + * Downloads entire data using metadata. * - * @param accountData account data - * @param podName pod name - * @param fullPath full path to the file - * @param downloadOptions download options - * @param dataDownloadOptions data download options + * @param {Block[]} blocks - The array of blocks to download. + * @param {Bee} bee - The Bee object to use for downloading. + * @param {BeeRequestOptions} [downloadOptions] - The options to be passed to the bee.downloadData() method. + * @param {DataDownloadOptions} [dataDownloadOptions] - The options for tracking progress during downloading. + * @returns {Promise} - A promise that resolves with the downloaded data. */ -export async function downloadData( - accountData: AccountData, - podName: string, - fullPath: string, +export async function prepareDataByMeta( + blocks: Block[], + bee: Bee, downloadOptions?: BeeRequestOptions, dataDownloadOptions?: DataDownloadOptions, ): Promise { dataDownloadOptions = dataDownloadOptions ?? {} - const bee = accountData.connection.bee - const { blocks } = await getFileMetadataWithBlocks( - bee, - accountData, - podName, - fullPath, - downloadOptions, - dataDownloadOptions, - ) - let totalLength = 0 for (const block of blocks) { totalLength += block.size @@ -163,6 +155,36 @@ export async function downloadData( return wrapBytesWithHelpers(result) } +/** + * Downloads file parts and compile them into Data + * + * @param accountData account data + * @param podName pod name + * @param fullPath full path to the file + * @param downloadOptions download options + * @param dataDownloadOptions data download options + */ +export async function downloadData( + accountData: AccountData, + podName: string, + fullPath: string, + downloadOptions?: BeeRequestOptions, + dataDownloadOptions?: DataDownloadOptions, +): Promise { + dataDownloadOptions = dataDownloadOptions ?? {} + const bee = accountData.connection.bee + const { blocks } = await getFileMetadataWithBlocks( + bee, + accountData, + podName, + fullPath, + downloadOptions, + dataDownloadOptions, + ) + + return prepareDataByMeta(blocks, bee, downloadOptions, dataDownloadOptions) +} + /** * Generate block name by block number */ @@ -185,9 +207,15 @@ export async function uploadData( data: Uint8Array | string | ExternalDataBlock[], accountData: AccountData, options: DataUploadOptions, -): Promise { - assertPodName(podName) - assertFullPathWithName(fullPath) +): Promise { + // empty pod name is acceptable in case of uploading the list of pods + const isPodUploading = podName === '' + + if (podName) { + assertPodName(podName) + assertFullPathWithName(fullPath) + } + assertWallet(accountData.wallet) const blockSize = options.blockSize ?? Number(DEFAULT_UPLOAD_OPTIONS!.blockSize) @@ -195,7 +223,18 @@ export async function uploadData( const contentType = options.contentType ?? String(DEFAULT_UPLOAD_OPTIONS!.contentType) const connection = accountData.connection updateUploadProgress(options, UploadProgressType.GetPodInfo) - const { podWallet, pod } = await getExtendedPodsListByAccountData(accountData, podName) + + let podWallet + let pod + + // if pod name is empty, we use root wallet which is for pods management + if (podName) { + ;({ podWallet, pod } = await getExtendedPodsListByAccountData(accountData, podName)) + } else if (!podName && accountData.wallet) { + podWallet = accountData.wallet + } else { + throw new Error('Pod name or root wallet is required') + } updateUploadProgress(options, UploadProgressType.GetPathInfo) const fullPathInfo = await getCreationPathInfo( @@ -204,7 +243,7 @@ export async function uploadData( prepareEthAddress(podWallet.address), connection.options?.requestOptions, ) - const pathInfo = extractPathInfo(fullPath) + const pathInfo = extractPathInfo(fullPath, isPodUploading) const now = getUnixTimestamp() const blocks: Blocks = { blocks: [] } @@ -224,7 +263,12 @@ export async function uploadData( percentage: calcUploadBlockPercentage(i, totalBlocks), } updateUploadProgress(options, UploadProgressType.UploadBlockStart, blockData) - const currentBlock = getDataBlock(data, blockSize, i) + let currentBlock = getDataBlock(data, blockSize, i) + + if (options.compression === Compression.GZIP) { + currentBlock = compress(currentBlock) + } + blocks.blocks.push(await uploadDataBlock(connection, currentBlock)) updateUploadProgress(options, UploadProgressType.UploadBlockEnd, blockData) } @@ -235,12 +279,12 @@ export async function uploadData( const blocksReference = (await uploadBytes(connection, manifestBytes)).reference const meta: FileMetadata = { version: META_VERSION, - filePath: pathInfo.path, + filePath: isPodUploading ? '' : pathInfo.path, fileName: pathInfo.filename, fileSize, blockSize, contentType, - compression: '', + compression: options.compression ?? '', creationTime: now, accessTime: now, modificationTime: now, @@ -249,20 +293,30 @@ export async function uploadData( } updateUploadProgress(options, UploadProgressType.WriteDirectoryInfo) - await addEntryToDirectory(connection, podWallet, pod.password, pathInfo.path, pathInfo.filename, true) + + // add entry to the directory only if pod provided, if not pod provided it means we are uploading pods list + if (pod) { + await addEntryToDirectory(connection, podWallet, pod.password, pathInfo.path, pathInfo.filename, true) + } + updateUploadProgress(options, UploadProgressType.WriteFileInfo) + const nextEpoch = prepareEpoch(getNextEpoch(fullPathInfo?.lookupAnswer.epoch)) + const fileMetadataRawBytes = getFileMetadataRawBytes(meta) await writeFeedData( connection, fullPath, - getFileMetadataRawBytes(meta), + fileMetadataRawBytes, podWallet, - pod.password, - getNextEpoch(fullPathInfo?.lookupAnswer.epoch), + pod ? pod.password : preparePrivateKey(podWallet.privateKey), + nextEpoch, ) updateUploadProgress(options, UploadProgressType.Done) - return meta + return { + lookupAnswer: { data: wrapChunkHelper(wrapBytesWithHelpers(fileMetadataRawBytes)), epoch: nextEpoch }, + meta, + } } /** diff --git a/src/file/types.ts b/src/file/types.ts index ade834fd..31baf2c0 100644 --- a/src/file/types.ts +++ b/src/file/types.ts @@ -127,6 +127,16 @@ export interface ProgressCallback { progressCallback?: (info: T) => void } +/** + * Enumeration representing compression algorithms + */ +export enum Compression { + /** + * GZIP compression + */ + GZIP = 'gzip', +} + /** * File upload options */ @@ -139,6 +149,10 @@ export interface DataUploadOptions extends ProgressCallback * Content type of the file */ contentType?: string + /** + * Compression algorithm + */ + compression?: Compression } /** diff --git a/src/file/utils.ts b/src/file/utils.ts index 84ca9c87..c4ee3fd4 100644 --- a/src/file/utils.ts +++ b/src/file/utils.ts @@ -86,9 +86,13 @@ export async function uploadBytes(connection: Connection, data: Uint8Array): Pro * Extracts filename and path from full path * * @param fullPath full absolute path with filename + * @param isPod is pod path */ -export function extractPathInfo(fullPath: string): PathInfo { - assertFullPathWithName(fullPath) +export function extractPathInfo(fullPath: string, isPod = false): PathInfo { + if (!isPod) { + assertFullPathWithName(fullPath) + } + const exploded = splitPath(fullPath) const filename = exploded.pop() @@ -125,7 +129,16 @@ export async function downloadBlocksManifest( reference: Reference, downloadOptions?: BeeRequestOptions, ): Promise { - const data = (await bee.downloadData(reference, downloadOptions)).text() + return extractBlocksManifest((await bee.downloadData(reference, downloadOptions)).text()) +} + +/** + * Extracts blocks manifest from the given data. + * + * @param {string} data - The data from which to extract the blocks manifest. + * @return {Blocks} The extracted blocks manifest. + */ +export function extractBlocksManifest(data: string): Blocks { const rawBlocks = jsonParse(data, 'blocks manifest') assertRawBlocks(rawBlocks) diff --git a/src/pod/api.ts b/src/pod/api.ts index b9e380ae..2259776b 100644 --- a/src/pod/api.ts +++ b/src/pod/api.ts @@ -1,65 +1,64 @@ -import { Bee } from '@ethersphere/bee-js' -import { getFeedData } from '../feed/api' -import { POD_TOPIC } from './personal-storage' -import { ExtendedPodInfo, extractPods, PodsInfo } from './utils' +import { ExtendedPodInfo, getPodsData, migratePodV1ToV2, PodsVersion, PodsInfo, extractPodsV2 } from './utils' import { prepareEthAddress, privateKeyToBytes } from '../utils/wallet' import { utils } from 'ethers' import { DownloadOptions } from '../content-items/types' import { getWalletByIndex } from '../utils/cache/wallet' import { getPodsList as getPodsListCached } from './cache/api' +import { AccountData } from '../account/account-data' +import { Bee } from '@ethersphere/bee-js' /** * Gets pods list with lookup answer * - * @param bee Bee instance + * @param accountData account data + * @param bee Bee client * @param userWallet root wallet for downloading and decrypting data * @param downloadOptions request download */ export async function getPodsList( + accountData: AccountData, bee: Bee, userWallet: utils.HDNode, downloadOptions?: DownloadOptions, ): Promise { - let lookupAnswer - try { - lookupAnswer = await getFeedData( - bee, - POD_TOPIC, - prepareEthAddress(userWallet.address), - downloadOptions?.requestOptions, - ) - // eslint-disable-next-line no-empty - } catch (e) {} + let podsData = await getPodsData(bee, prepareEthAddress(userWallet.address), downloadOptions?.requestOptions) - if (!lookupAnswer) { - throw new Error('Pods data can not be found') + if (podsData.podsVersion === PodsVersion.V1) { + podsData = await migratePodV1ToV2(accountData, podsData, privateKeyToBytes(userWallet.privateKey)) } - const podsList = extractPods(lookupAnswer.data.chunkContent(), privateKeyToBytes(userWallet.privateKey)) + const podsList = await extractPodsV2( + accountData, + podsData.lookupAnswer.data.chunkContent(), + privateKeyToBytes(userWallet.privateKey), + downloadOptions?.requestOptions, + ) return { podsList, - epoch: lookupAnswer.epoch, + epoch: podsData.lookupAnswer.epoch, } } /** * Gets pods list with lookup answer and extended info about pod * - * @param bee Bee instance + * @param accountData account data + * @param bee Bee client * @param podName pod to find * @param userWallet root wallet for downloading and decrypting data * @param seed seed of wallet owns the FDP account * @param downloadOptions request options */ export async function getExtendedPodsList( + accountData: AccountData, bee: Bee, podName: string, userWallet: utils.HDNode, seed: Uint8Array, downloadOptions?: DownloadOptions, ): Promise { - const { podsList, epoch } = await getPodsListCached(bee, userWallet, downloadOptions) + const { podsList, epoch } = await getPodsListCached(accountData, bee, userWallet, downloadOptions) const pod = podsList.pods.find(item => item.name === podName) if (!pod) { diff --git a/src/pod/cache/api.ts b/src/pod/cache/api.ts index a136d1c9..cdd5e22a 100644 --- a/src/pod/cache/api.ts +++ b/src/pod/cache/api.ts @@ -5,15 +5,18 @@ import { jsonToPodsList, podListToJSON, PodsInfo } from '../utils' import { CacheEpochData } from '../../cache/types' import { getCacheKey, processCacheData } from '../../cache/utils' import { getPodsList as getPodsNoCache } from '../api' +import { AccountData } from '../../account/account-data' /** * Gets pods list with lookup answer * - * @param bee Bee instance + * @param accountData account data + * @param bee Bee client * @param userWallet root wallet for downloading and decrypting data * @param downloadOptions request download */ export async function getPodsList( + accountData: AccountData, bee: Bee, userWallet: utils.HDNode, downloadOptions?: DownloadOptions, @@ -21,7 +24,7 @@ export async function getPodsList( return processCacheData({ key: getCacheKey(userWallet.address), onGetData: async (): Promise => { - const data = await getPodsNoCache(bee, userWallet, downloadOptions) + const data = await getPodsNoCache(accountData, bee, userWallet, downloadOptions) return { epoch: data.epoch, diff --git a/src/pod/personal-storage.ts b/src/pod/personal-storage.ts index 95655690..a8ec7aa9 100644 --- a/src/pod/personal-storage.ts +++ b/src/pod/personal-storage.ts @@ -1,6 +1,5 @@ -import { SharedPod, PodReceiveOptions, PodShareInfo, PodsList, Pod, PodsListPrepared } from './types' +import { Pod, PodReceiveOptions, PodShareInfo, PodsList, PodsListPrepared, SharedPod } from './types' import { assertAccount } from '../account/utils' -import { writeFeedData } from '../feed/api' import { AccountData } from '../account/account-data' import { assertPod, @@ -11,23 +10,22 @@ import { createPodShareInfo, getSharedPodInfo, podListToBytes, - podsListPreparedToPodsList, + podListToJSON, podPreparedToPod, + podsListPreparedToPodsList, sharedPodPreparedToSharedPod, - podListToJSON, + uploadPodDataV2, } from './utils' import { getExtendedPodsList } from './api' import { uploadBytes } from '../file/utils' import { stringToBytes } from '../utils/bytes' import { Reference, Utils } from '@ethersphere/bee-js' import { assertEncryptedReference, EncryptedReference } from '../utils/hex' -import { prepareEthAddress, preparePrivateKey } from '../utils/wallet' +import { prepareEthAddress } from '../utils/wallet' import { getCacheKey, setEpochCache } from '../cache/utils' import { getPodsList } from './cache/api' import { getNextEpoch } from '../feed/lookup/utils' -export const POD_TOPIC = 'Pods' - export class PersonalStorage { constructor(private accountData: AccountData) {} @@ -48,7 +46,7 @@ export class PersonalStorage { try { podsList = ( - await getPodsList(this.accountData.connection.bee, this.accountData.wallet!, { + await getPodsList(this.accountData, this.accountData.connection.bee, this.accountData.wallet!, { requestOptions: this.accountData.connection.options?.requestOptions, cacheInfo: this.accountData.connection.cacheInfo, }) @@ -70,7 +68,7 @@ export class PersonalStorage { assertAccount(this.accountData, { writeRequired: true }) const pod = await createPod( - this.accountData.connection.bee, + this.accountData, this.accountData.connection, this.accountData.wallet!, this.accountData.seed!, @@ -94,7 +92,7 @@ export class PersonalStorage { async delete(name: string): Promise { assertAccount(this.accountData, { writeRequired: true }) name = name.trim() - const podsInfo = await getPodsList(this.accountData.connection.bee, this.accountData.wallet!, { + const podsInfo = await getPodsList(this.accountData, this.accountData.connection.bee, this.accountData.wallet!, { requestOptions: this.accountData.connection.options?.requestOptions, cacheInfo: this.accountData.connection.cacheInfo, }) @@ -111,14 +109,7 @@ export class PersonalStorage { const allPodsData = podListToBytes(podsFiltered, podsSharedFiltered) const wallet = this.accountData.wallet! const epoch = getNextEpoch(podsInfo.epoch) - await writeFeedData( - this.accountData.connection, - POD_TOPIC, - allPodsData, - wallet, - preparePrivateKey(wallet.privateKey), - epoch, - ) + await uploadPodDataV2(this.accountData, allPodsData) await setEpochCache(this.accountData.connection.cacheInfo, getCacheKey(wallet.address), { epoch, data: podListToJSON(podsFiltered, podsSharedFiltered), @@ -139,10 +130,17 @@ export class PersonalStorage { assertPodName(name) const wallet = this.accountData.wallet! const address = prepareEthAddress(wallet.address) - const podInfo = await getExtendedPodsList(this.accountData.connection.bee, name, wallet, this.accountData.seed!, { - requestOptions: this.accountData.connection.options?.requestOptions, - cacheInfo: this.accountData.connection.cacheInfo, - }) + const podInfo = await getExtendedPodsList( + this.accountData, + this.accountData.connection.bee, + name, + wallet, + this.accountData.seed!, + { + requestOptions: this.accountData.connection.options?.requestOptions, + cacheInfo: this.accountData.connection.cacheInfo, + }, + ) const data = stringToBytes( JSON.stringify(createPodShareInfo(name, podInfo.podAddress, address, podInfo.pod.password)), @@ -183,7 +181,7 @@ export class PersonalStorage { const data = await this.getSharedInfo(reference) const pod = await createPod( - this.accountData.connection.bee, + this.accountData, this.accountData.connection, this.accountData.wallet!, this.accountData.seed!, diff --git a/src/pod/types.ts b/src/pod/types.ts index 4fe74f6c..78c29862 100644 --- a/src/pod/types.ts +++ b/src/pod/types.ts @@ -1,6 +1,7 @@ import { Utils, Reference } from '@ethersphere/bee-js' import { PodPasswordBytes } from '../utils/encryption' import { HexString } from '../utils/hex' +import { LookupAnswer } from '../feed/types' /** * Pods information prepared for internal usage @@ -93,6 +94,20 @@ export interface FileMetadata { mode: number } +/** + * Information about a file with lookup answer + */ +export interface FileMetadataWithLookupAnswer { + /** + * File metadata + */ + meta: FileMetadata + /** + * Lookup answer + */ + lookupAnswer: LookupAnswer +} + /** * Information about a directory */ diff --git a/src/pod/utils.ts b/src/pod/utils.ts index 252fc947..08437733 100644 --- a/src/pod/utils.ts +++ b/src/pod/utils.ts @@ -1,18 +1,18 @@ import { + FileMetadataWithLookupAnswer, Pod, - PodPrepared, PodName, + PodPrepared, PodShareInfo, + PodsList, + PodsListPrepared, RawDirectoryMetadata, SharedPod, SharedPodPrepared, - PodsListPrepared, - PodsList, } from './types' -import { Bee, Data, Utils } from '@ethersphere/bee-js' +import { Bee, BeeRequestOptions, Data, Utils } from '@ethersphere/bee-js' import { assertAllowedZeroBytes, - assertMaxLength, bytesToString, MAX_ZEROS_PERCENTAGE_ALLOWED, stringToBytes, @@ -36,25 +36,68 @@ import { bytesToHex, EncryptedReference, isHexEthAddress } from '../utils/hex' import { getExtendedPodsList } from './api' import { Epoch, getFirstEpoch } from '../feed/lookup/epoch' import { getUnixTimestamp } from '../utils/time' -import { writeFeedData } from '../feed/api' -import { preparePrivateKey } from '../utils/wallet' +import { getFeedData } from '../feed/api' import { createRootDirectory } from '../directory/handler' -import { POD_TOPIC } from './personal-storage' import { Connection } from '../connection/connection' import { AccountData } from '../account/account-data' import { decryptBytes, POD_PASSWORD_LENGTH, PodPasswordBytes } from '../utils/encryption' import CryptoJS from 'crypto-js' import { jsonParse } from '../utils/json' -import { DEFAULT_DIRECTORY_PERMISSIONS, getDirectoryMode } from '../directory/utils' +import { assertRawFileMetadata, DEFAULT_DIRECTORY_PERMISSIONS, getDirectoryMode } from '../directory/utils' import { getCacheKey, setEpochCache } from '../cache/utils' import { getWalletByIndex } from '../utils/cache/wallet' import { getPodsList } from './cache/api' -import { CHUNK_SIZE } from '../account/utils' +import { LookupAnswer } from '../feed/types' +import { prepareDataByMeta, uploadData } from '../file/handler' +import { Compression } from '../file/types' +import { MINIMUM_BLOCK_SIZE } from '../content-items/handler' +import { downloadBlocksManifest } from '../file/utils' +import { extractMetadata } from '../content-items/utils' +import { rawFileMetadataToFileMetadata } from '../file/adapter' +import { decompress } from '../utils/compression' export const META_VERSION = 2 export const MAX_PODS_COUNT = 65536 export const MAX_POD_NAME_LENGTH = 64 +/** + * Represents the topic for Pods version 1 + */ +export const POD_TOPIC = 'Pods' + +/** + * Represents the topic for Pods version 2 + */ +export const POD_TOPIC_V2 = 'PodsV2' + +/** + * Enum representing the version of PODs + */ +export enum PodsVersion { + /** + * Version 1 + */ + V1 = 'v1', + /** + * Version 2 + */ + V2 = 'v2', +} + +/** + * Answer with LookupAnswer and pods version + */ +export interface PodsLookupAnswer { + /** + * Pods version + */ + podsVersion: PodsVersion + /** + * Lookup answer + */ + lookupAnswer: LookupAnswer +} + /** * Information about pods list */ @@ -83,12 +126,45 @@ export interface PathInfo { /** * Extracts pod information from raw data - * * @param data raw data with pod information * @param podPassword bytes of pod password */ -export function extractPods(data: Data, podPassword: PodPasswordBytes): PodsListPrepared { - return jsonToPodsList(bytesToString(decryptBytes(bytesToHex(podPassword), data))) +export function extractPodsV1(data: Data, podPassword: PodPasswordBytes): PodsListPrepared { + return jsonToPodsList(bytesToString(extractPodsBytes(data, podPassword))) +} + +/** + * Extracts and prepares a list of pods from the given V2 data using the provided pod password + * @param accountData + * @param {Data} data - The data from which to extract the pods. + * @param {PodPasswordBytes} podPassword - The password to decrypt the pods. + * @param downloadOptions + * @return {PodsListPrepared} - The list of pods that have been extracted and prepared. + */ +export async function extractPodsV2( + accountData: AccountData, + data: Data, + podPassword: PodPasswordBytes, + downloadOptions?: BeeRequestOptions, +): Promise { + const meta = extractMetadata(data, podPassword) + assertRawFileMetadata(meta) + const fileMeta = rawFileMetadataToFileMetadata(meta) + const blocksData = await downloadBlocksManifest(accountData.connection.bee, fileMeta.blocksReference, downloadOptions) + const preparedData = bytesToString( + decompress(await prepareDataByMeta(blocksData.blocks, accountData.connection.bee, downloadOptions)), + ) + + return jsonToPodsList(preparedData) +} + +/** + * Decrypts pod information from raw data + * @param data raw data with pod information + * @param podPassword bytes of pod password + */ +export function extractPodsBytes(data: Data, podPassword: PodPasswordBytes): Uint8Array { + return decryptBytes(bytesToHex(podPassword), data) } /** @@ -380,14 +456,14 @@ export function getRandomPodPassword(): PodPasswordBytes { /** * Creates user's pod or add a shared pod to an account * - * @param bee Bee instance + * @param accountData AccountData instance * @param connection Connection instance * @param userWallet FDP account wallet * @param seed FDP account seed * @param pod pod information to create */ export async function createPod( - bee: Bee, + accountData: AccountData, connection: Connection, userWallet: utils.HDNode, seed: Uint8Array, @@ -401,7 +477,7 @@ export async function createPod( let podsList: PodsListPrepared = { pods: [], sharedPods: [] } let podsInfo try { - podsInfo = await getPodsList(bee, userWallet, { + podsInfo = await getPodsList(accountData, connection.bee, userWallet, { requestOptions: connection.options?.requestOptions, cacheInfo, }) @@ -432,8 +508,7 @@ export async function createPod( } const allPodsData = podListToBytes(pods, sharedPods) - assertMaxLength(allPodsData.length, CHUNK_SIZE, `Exceeded pod list size by ${allPodsData.length - CHUNK_SIZE} bytes`) - await writeFeedData(connection, POD_TOPIC, allPodsData, userWallet, preparePrivateKey(userWallet.privateKey), epoch) + await uploadPodDataV2(accountData, allPodsData) if (isPod(realPod)) { const podWallet = await getWalletByIndex(seed, nextIndex, cacheInfo) @@ -448,6 +523,21 @@ export async function createPod( return realPod } +/** + * Uploads pods data using version 2 method + * @param accountData AccountData instance + * @param allPodsData pods data + */ +export async function uploadPodDataV2( + accountData: AccountData, + allPodsData: Uint8Array, +): Promise { + return uploadData('', POD_TOPIC_V2, allPodsData, accountData, { + blockSize: MINIMUM_BLOCK_SIZE, + compression: Compression.GZIP, + }) +} + /** * Gets extended information about pods using AccountData instance and pod name * @@ -458,7 +548,7 @@ export async function getExtendedPodsListByAccountData( accountData: AccountData, podName: string, ): Promise { - return getExtendedPodsList(accountData.connection.bee, podName, accountData.wallet!, accountData.seed!, { + return getExtendedPodsList(accountData, accountData.connection.bee, podName, accountData.wallet!, accountData.seed!, { requestOptions: accountData.connection.options?.requestOptions, cacheInfo: accountData.connection.cacheInfo, }) @@ -551,7 +641,11 @@ export function assertJsonPod(value: unknown): asserts value is Pod { } /** - * Asserts that json shared pod is correct + * Asserts that a value is a valid SharedPod. + * + * @param {unknown} value - The value to assert. + * + * @throws {Error} Invalid json shared pod if the value is not a valid SharedPod. */ export function assertJsonSharedPod(value: unknown): asserts value is SharedPod { if (!isJsonSharedPod(value)) { @@ -560,7 +654,10 @@ export function assertJsonSharedPod(value: unknown): asserts value is SharedPod } /** - * Converts JsonSharedPod to SharedPod + * Converts a SharedPod object from JSON format to a preprocessed SharedPod object. + * + * @param {SharedPod} pod - The SharedPod object in JSON format. + * @return {SharedPodPrepared} - The preprocessed SharedPod object. */ export function jsonSharedPodToSharedPod(pod: SharedPod): SharedPodPrepared { const password = Utils.hexToBytes(pod.password) as PodPasswordBytes @@ -569,3 +666,64 @@ export function jsonSharedPodToSharedPod(pod: SharedPod): SharedPodPrepared { return { ...pod, password, address } } + +/** + * Retrieves pods data for a given address. + * + * @param {Bee} bee - The bee client object. + * @param {Utils.EthAddress} address - The address to look up pods data for. + * @param {BeeRequestOptions} [requestOptions] - The request options. + * @returns {Promise} The pods data. + * @throws {Error} If the pods data cannot be found. + */ +export async function getPodsData( + bee: Bee, + address: Utils.EthAddress, + requestOptions?: BeeRequestOptions, +): Promise { + let podsVersion = PodsVersion.V2 + let lookupAnswer + try { + lookupAnswer = await getFeedData(bee, POD_TOPIC_V2, address, requestOptions) + // eslint-disable-next-line no-empty + } catch (e) {} + + // if V2 does not exist, try V1 + if (!lookupAnswer) { + try { + lookupAnswer = await getFeedData(bee, POD_TOPIC, address, requestOptions) + podsVersion = PodsVersion.V1 + // eslint-disable-next-line no-empty + } catch (e) {} + } + + if (!lookupAnswer) { + throw new Error('Pods data can not be found') + } + + return { + podsVersion, + lookupAnswer, + } +} + +/** + * Migrates a pod from version 1 to version 2. + * + * @param {AccountData} accountData - The account data. + * @param {PodsLookupAnswer} podsData - The pods data. + * @param {PodPasswordBytes} privateKey - The private key used for migration. + * @return {Promise} - The pods data with the migrated version. + */ +export async function migratePodV1ToV2( + accountData: AccountData, + podsData: PodsLookupAnswer, + privateKey: PodPasswordBytes, +): Promise { + const podsBytes = extractPodsBytes(podsData.lookupAnswer.data, privateKey) + + return { + podsVersion: PodsVersion.V2, + lookupAnswer: (await uploadPodDataV2(accountData, podsBytes)).lookupAnswer, + } +} diff --git a/src/utils/compression.ts b/src/utils/compression.ts new file mode 100644 index 00000000..895447c9 --- /dev/null +++ b/src/utils/compression.ts @@ -0,0 +1,17 @@ +import pako from 'pako' + +/** + * Compresses the given data using gzip + * @param data input data + */ +export function compress(data: Uint8Array): Uint8Array { + return pako.deflate(data) +} + +/** + * Decompresses the given data using gzip + * @param data input data + */ +export function decompress(data: Uint8Array): Uint8Array { + return pako.inflate(data) +} diff --git a/src/utils/hex.ts b/src/utils/hex.ts index 85dfa387..da89cac0 100644 --- a/src/utils/hex.ts +++ b/src/utils/hex.ts @@ -6,7 +6,6 @@ export type HexEthAddress = HexString<40> /** * Nominal type to represent hex strings WITHOUT '0x' prefix. * For example for 32 bytes hex representation you have to use 64 length. - * TODO: Make Length mandatory: https://github.com/ethersphere/bee-js/issues/208 */ export type HexString = FlavoredType< string & { diff --git a/test/integration/node/fdp-class.spec.ts b/test/integration/node/fdp-class.spec.ts index 31568c7b..9274772d 100644 --- a/test/integration/node/fdp-class.spec.ts +++ b/test/integration/node/fdp-class.spec.ts @@ -12,7 +12,6 @@ import { PodShareInfo, RawFileMetadata } from '../../../src/pod/types' import { FileShareInfo } from '../../../src/file/types' import { getFeedData } from '../../../src/feed/api' import * as feedApi from '../../../src/feed/api' -import { POD_TOPIC } from '../../../src/pod/personal-storage' import { decryptBytes } from '../../../src/utils/encryption' import { Wallet } from 'ethers' import { removeZeroFromHex } from '../../../src/account/utils' @@ -26,6 +25,7 @@ import { ETH_ADDR_HEX_LENGTH } from '../../../src/utils/type' import * as walletApi from '../../../src/utils/wallet' import { HIGHEST_LEVEL } from '../../../src/feed/lookup/epoch' import { getWalletByIndex } from '../../../src/utils/cache/wallet' +import { POD_TOPIC_V2 } from '../../../src/pod/utils' jest.setTimeout(400000) describe('Fair Data Protocol class', () => { @@ -725,13 +725,15 @@ describe('Fair Data Protocol class', () => { // check pod metadata const pod1 = await fdp.personalStorage.create(pod) - const podData = await getFeedData(bee, POD_TOPIC, prepareEthAddress(user.address)) + const podData = await getFeedData(bee, POD_TOPIC_V2, prepareEthAddress(user.address)) const encryptedText1 = podData.data.chunkContent().text() const encryptedBytes1 = podData.data.chunkContent() // data decrypts with wallet for the pod. Data inside the pod will be encrypted with a password stored in the pod const decryptedText1 = bytesToString(decryptBytes(privateKey, encryptedBytes1)) expect(encryptedText1).not.toContain(pod) - expect(decryptedText1).toContain(pod) + expect(decryptedText1).toContain('version') + expect(decryptedText1).toContain('filePath') + expect(decryptedText1).toContain(POD_TOPIC_V2) // HDNode with index 1 is for first pod const node1 = await getWalletByIndex(seed, 1) const rootDirectoryData = await getFeedData(bee, '/', prepareEthAddress(node1.address)) @@ -811,15 +813,16 @@ describe('Fair Data Protocol class', () => { await fdpNoCache.personalStorage.create(pod1) // for the first feed write it should be the highest level expect(writeFeedDataSpy.mock.calls[0][5]?.level).toEqual(HIGHEST_LEVEL) - // getting info about pods from the network - expect(getFeedDataSpy).toBeCalledTimes(1) + // getting old V1 pods info + getting V2 pods info + getting V2 pods info for data uploading (todo: can be reduced) + expect(getFeedDataSpy).toBeCalledTimes(3) // calculating wallet by index for the pod expect(getWalletByIndexSpy).toBeCalledTimes(1) jest.clearAllMocks() await fdpNoCache.personalStorage.create(pod2) expect(writeFeedDataSpy.mock.calls[0][5]?.level).toEqual(HIGHEST_LEVEL - 1) - expect(getFeedDataSpy).toBeCalledTimes(1) + // get V1 pods info + V2 pods info + expect(getFeedDataSpy).toBeCalledTimes(2) expect(getWalletByIndexSpy).toBeCalledTimes(1) jest.clearAllMocks() @@ -858,8 +861,8 @@ describe('Fair Data Protocol class', () => { await fdpNoCache.personalStorage.delete(pod1) // writing info about pods to the network expect(writeFeedDataSpy).toBeCalledTimes(1) - // get pods info - expect(getFeedDataSpy).toBeCalledTimes(1) + // get V1 pods info + V2 pods info + expect(getFeedDataSpy).toBeCalledTimes(2) expect(getWalletByIndexSpy).toBeCalledTimes(0) jest.restoreAllMocks() @@ -889,15 +892,15 @@ describe('Fair Data Protocol class', () => { await fdpWithCache.personalStorage.create(pod1) // for the first feed write it should be the highest level expect(writeFeedDataSpy.mock.calls[0][5]?.level).toEqual(HIGHEST_LEVEL) - // getting info about pods from the network - expect(getFeedDataSpy).toBeCalledTimes(1) + // getting V1 pods info + V2 pods info + V2 pods info for data uploading (todo: can be reduced) + expect(getFeedDataSpy).toBeCalledTimes(3) // calculating wallet by index for the pod expect(getWalletByIndexSpy).toBeCalledTimes(1) jest.clearAllMocks() await fdpWithCache.personalStorage.create(pod2) - // data about pods shouldn't be retrieved - expect(getFeedDataSpy).toBeCalledTimes(0) + // V2 pods info for data uploading (todo: can be reduced) + expect(getFeedDataSpy).toBeCalledTimes(1) // should be calculated correct level without getting previous one from the network expect(writeFeedDataSpy.mock.calls[0][5]?.level).toEqual(HIGHEST_LEVEL - 1) // calc a wallet for the new pod @@ -939,8 +942,8 @@ describe('Fair Data Protocol class', () => { await fdpWithCache.personalStorage.delete(pod1) // writing info about pods to the network expect(writeFeedDataSpy).toBeCalledTimes(1) - // should not get pods info - expect(getFeedDataSpy).toBeCalledTimes(0) + // V2 pods info for data uploading (todo: can be reduced) + expect(getFeedDataSpy).toBeCalledTimes(1) expect(getWalletByIndexSpy).toBeCalledTimes(0) // recovering cache data diff --git a/test/integration/node/pod/pod-limit-error-message.spec.ts b/test/integration/node/pod/pod-limit-error-message.spec.ts deleted file mode 100644 index 430920c0..00000000 --- a/test/integration/node/pod/pod-limit-error-message.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { createFdp, generateRandomHexString, generateUser } from '../../../utils' -import { MAX_POD_NAME_LENGTH } from '../../../../src/pod/utils' - -jest.setTimeout(400000) -it('Pod list size error message', async () => { - const fdp = createFdp() - generateUser(fdp) - - for (let i = 0; i < 25; i++) { - const longPodName = generateRandomHexString(MAX_POD_NAME_LENGTH) - - if (i === 24) { - await expect(fdp.personalStorage.create(longPodName)).rejects.toThrow('Exceeded pod list size by 46 bytes') - } else { - await fdp.personalStorage.create(longPodName) - } - } -}) diff --git a/test/integration/node/pod/pods-limitation-check.spec.ts b/test/integration/node/pod/pods-limitation-check.spec.ts new file mode 100644 index 00000000..62d6d584 --- /dev/null +++ b/test/integration/node/pod/pods-limitation-check.spec.ts @@ -0,0 +1,15 @@ +import { createFdp, generateRandomHexString, generateUser } from '../../../utils' +import { MAX_POD_NAME_LENGTH } from '../../../../src' +import { HIGHEST_LEVEL } from '../../../../src/feed/lookup/epoch' + +jest.setTimeout(400000) +it('Pods limitation check', async () => { + const fdp = createFdp() + generateUser(fdp) + + // todo should work with more updates than HIGHEST_LEVEL + for (let i = 0; i < HIGHEST_LEVEL; i++) { + const longPodName = generateRandomHexString(MAX_POD_NAME_LENGTH) + await fdp.personalStorage.create(longPodName) + } +}) diff --git a/test/unit/feed/api.spec.ts b/test/unit/feed/api.spec.ts new file mode 100644 index 00000000..34eca8fb --- /dev/null +++ b/test/unit/feed/api.spec.ts @@ -0,0 +1,24 @@ +import { getFeedData, writeFeedData } from '../../../src/feed/api' +import { createFdp, generateUser } from '../../utils' +import { stringToBytes } from '../../../src/utils/bytes' +import { prepareEthAddress, preparePrivateKey } from '../../../src/utils/wallet' +import { decryptBytes } from '../../../src/utils/encryption' +import { bytesToHex } from '../../../src/utils/hex' + +jest.setTimeout(400000) +describe('feed/api', () => { + it('write-read check', async () => { + const fdp = createFdp() + generateUser(fdp) + + const wallet = fdp.account.wallet! + const podPassword = preparePrivateKey(wallet.privateKey) + const topic = '/' + const data = stringToBytes(JSON.stringify({ hello: 'world of bees' })) + await writeFeedData(fdp.connection, topic, data, wallet, podPassword) + const feedData = await getFeedData(fdp.connection.bee, topic, prepareEthAddress(wallet.address)) + const result = decryptBytes(bytesToHex(podPassword), feedData.data.chunkContent()) + + expect(result).toEqual(data) + }) +}) diff --git a/test/unit/feed/epoch.spec.ts b/test/unit/feed/epoch.spec.ts index d1a40abf..31207901 100644 --- a/test/unit/feed/epoch.spec.ts +++ b/test/unit/feed/epoch.spec.ts @@ -1,4 +1,5 @@ -import { Epoch, getBaseTime } from '../../../src/feed/lookup/epoch' +import { Epoch, getBaseTime, HIGHEST_LEVEL } from '../../../src/feed/lookup/epoch' +import { bytesToHex } from '../../../src/utils/hex' describe('feed/epoch', () => { it('getBaseTime', () => { @@ -101,4 +102,20 @@ describe('feed/epoch', () => { expect(result.time).toEqual(example.result.time) } }) + + it('getNextEpoch limits', () => { + const ids: { [key: string]: boolean } = {} + let epoch = new Epoch(HIGHEST_LEVEL, 1648635721) + // todo should work with more updates than HIGHEST_LEVEL + for (let i = 0; i <= HIGHEST_LEVEL; i++) { + epoch = epoch.getNextEpoch(epoch.time) + const id = bytesToHex(epoch.id()) as string + + if (ids[id]) { + throw new Error('epoch id already exists') + } else { + ids[id] = true + } + } + }) })