Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: stop writing to blocks v1 table #72

Merged
merged 1 commit into from
Feb 13, 2023
Merged

fix: stop writing to blocks v1 table #72

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ _Variables in bold are required._
| --------------------------- | ------------------ | ------------------------------------------------------------------------------ |
| CONCURRENCY | `32` | Concurrent batch inserts of blocks. |
| BLOCKS_BATCH_SIZE | `10` | Batch size for blocks ops (insert, publish). 10 is max for SQS, 25 is max for Dynamo |
| DYNAMO_BLOCKS_TABLE | `v1-blocks` | The DynamoDB table where store CIDs informations to. |
| DYNAMO_CARS_TABLE | `v1-cars` | The DynamoDB table where store CAR files informations to. |
| DYNAMO_LINK_TABLE | `v1-blocks-cars-position` | The DynamoDB table with CARs-blocks links. |
| DYNAMO_MAX_RETRIES | `3` | DynamoDB max attempts in case of query failure. |
Expand Down
3 changes: 0 additions & 3 deletions src/config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ require('dotenv').config({ path: process.env.ENV_FILE_PATH || resolve(process.cw
const {
CONCURRENCY: rawConcurrency,

DYNAMO_BLOCKS_TABLE: blocksTable,
DYNAMO_CARS_TABLE: carsTable,
DYNAMO_LINK_TABLE: linkTable,

Expand Down Expand Up @@ -49,11 +48,9 @@ const concurrency = parseInt(rawConcurrency)

module.exports = {
RAW_BLOCK_CODEC,
blocksTable: blocksTable ?? 'v1-blocks',
carsTable: carsTable ?? 'v1-cars',
linkTable: linkTable ?? 'v1-blocks-cars-position',

blocksTablePrimaryKey: 'multihash',
carsTablePrimaryKey: 'path',
linkTableBlockKey: 'blockmultihash',
linkTableCarKey: 'carpath',
Expand Down
51 changes: 1 addition & 50 deletions src/lib/block.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,51 +1,11 @@
'use strict'

const {
UnixFS: { unmarshal: decodeUnixFs }
} = require('ipfs-unixfs')
const config = require('../config')
const { now, queuedTasks } = require('./util')
const { queuedTasks } = require('./util')
const { serializeError } = require('./logging')
const { cidToKey, batchWriteDynamoItems } = require('./storage')
const { publishBatch } = require('./publish')

function blockType(blockId, codecs = config.codecs) {
return codecs[blockId.code]?.label ?? 'unsupported'
}

function blockData(block) {
if (!block.data) {
return
}

const data = decodeBlock(block)
if (data.Links) {
for (const link of data.Links) {
link.Hash = link.Hash?.toString()
}
}

return data
}

function decodeBlock(block, codecs = config.codecs) {
const codec = codecs[block.cid.code]

if (!codec) {
throw new Error(`Unsupported codec ${block.cid.code} in the block at offset ${block.blockOffset}`)
}

const data = codec.decode(block.data)

if (codec.label.startsWith('dag')) {
const { type, blocks } = decodeUnixFs(data.Data)

data.Data = { type, blocks }
}

return data
}

async function publishBlocks({ blocks, logger, queue = config.publishingQueue }) {
await publishBatch({ queue, messages: blocks.map(b => b.key), logger })
}
Expand Down Expand Up @@ -98,7 +58,6 @@ async function storeBlocks({ car, source, logger, batchSize = config.blocksBatch
* @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html#Programming.Errors.BatchOperations
*/
async function writeBlocksBatch({ blocks, car, logger }) {
const blockItems = []
const linkItems = []
const keys = []
for (let i = 0; i < blocks.length; i++) {
Expand All @@ -112,13 +71,6 @@ async function writeBlocksBatch({ blocks, car, logger }) {
continue
}

blockItems.push({
[config.blocksTablePrimaryKey]: block.key,
type: blockType(block.cid),
createdAt: now(),
data: blockData(block)
})

linkItems.push({
[config.linkTableBlockKey]: block.key,
[config.linkTableCarKey]: car.id,
Expand All @@ -131,7 +83,6 @@ async function writeBlocksBatch({ blocks, car, logger }) {
}

const batch = [
{ table: config.blocksTable, items: blockItems },
{ table: config.linkTable, items: linkItems }
]

Expand Down
96 changes: 0 additions & 96 deletions tap-snapshots/test/index.test.js.test.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,68 +11,6 @@ Object {
"batchCreates": Array [
Object {
"RequestItems": Object {
"v1-blocks": Array [
Object {
"PutRequest": Object {
"Item": Object {
"createdAt": Object {
"S": "2022-06-24T15:15:17.401Z",
},
"multihash": Object {
"S": "zQmY13QWtykrcwmQmLVdxAQnJsRq7xBs5FAqH5zpG9ZvJpC",
},
"type": Object {
"S": "raw",
},
},
},
},
Object {
"PutRequest": Object {
"Item": Object {
"createdAt": Object {
"S": "2022-06-24T15:15:17.401Z",
},
"multihash": Object {
"S": "zQmSGtsqx7aYH8gP21AgidxXuX5vsseFJgHKa75kg8HepXL",
},
"type": Object {
"S": "dag-pb",
},
},
},
},
Object {
"PutRequest": Object {
"Item": Object {
"createdAt": Object {
"S": "2022-06-24T15:15:17.401Z",
},
"multihash": Object {
"S": "zQmSHc8o3PxQgMccYgGtuStaNQKXTBX1rTHN5W9cUCwrcHX",
},
"type": Object {
"S": "dag-pb",
},
},
},
},
Object {
"PutRequest": Object {
"Item": Object {
"createdAt": Object {
"S": "2022-06-24T15:15:17.401Z",
},
"multihash": Object {
"S": "zQmTgGQZ3ZcbcHxZiFNHs76Y7Ca8DfFGjdsxXDVnr41h339",
},
"type": Object {
"S": "dag-pb",
},
},
},
},
],
"v1-blocks-cars-position": Array [
Object {
"PutRequest": Object {
Expand Down Expand Up @@ -151,23 +89,6 @@ Object {
},
Object {
"RequestItems": Object {
"v1-blocks": Array [
Object {
"PutRequest": Object {
"Item": Object {
"createdAt": Object {
"S": "2022-06-24T15:15:17.401Z",
},
"multihash": Object {
"S": "zQmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn",
},
"type": Object {
"S": "dag-pb",
},
},
},
},
],
"v1-blocks-cars-position": Array [
Object {
"PutRequest": Object {
Expand Down Expand Up @@ -271,23 +192,6 @@ Object {
"batchCreates": Array [
Object {
"RequestItems": Object {
"v1-blocks": Array [
Object {
"PutRequest": Object {
"Item": Object {
"createdAt": Object {
"S": "2022-06-24T15:15:17.401Z",
},
"multihash": Object {
"S": "zQmPH3Su9xAqw4WRbXT6DvwNpmaXYvTKKAY2hBKJsC7j2b4",
},
"type": Object {
"S": "unsupported",
},
},
},
},
],
"v1-blocks-cars-position": Array [
Object {
"PutRequest": Object {
Expand Down