Skip to content

Commit

Permalink
feat: remove .jsonlines hack from simple swing store
Browse files Browse the repository at this point in the history
  • Loading branch information
@FUDCo
FUDCo committed May 30, 2021
1 parent 94c1688 commit ef87997
Show file tree
Hide file tree
Showing 8 changed files with 61 additions and 225 deletions.
4 changes: 2 additions & 2 deletions packages/agoric-cli/lib/open.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import crypto from 'crypto';
import path from 'path';
import os from 'os';

import { openSwingStore } from '@agoric/swing-store-simple';
import { openSwingStore } from '@agoric/swing-store-lmdb';

import { assert, details as X } from '@agoric/assert';

Expand Down Expand Up @@ -43,7 +43,7 @@ export async function getAccessToken(port, powers = {}) {
await fs.mkdir(sharedStateDir, { mode: 0o700, recursive: true });

// Ensure an access token exists.
const { kvStore, commit, close } = openSwingStore(sharedStateDir, 'state');
const { kvStore, commit, close } = openSwingStore(sharedStateDir);
const accessTokenKey = `accessToken/${port}`;
if (!kvStore.has(accessTokenKey)) {
kvStore.set(accessTokenKey, await generateAccessToken());
Expand Down
2 changes: 1 addition & 1 deletion packages/agoric-cli/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"@agoric/install-ses": "^0.5.13",
"@agoric/nat": "^4.0.0",
"@agoric/promise-kit": "^0.2.13",
"@agoric/swing-store-simple": "^0.3.9",
"@agoric/swing-store-lmdb": "^0.4.11",
"@iarna/toml": "^2.2.3",
"anylogger": "^0.21.0",
"chalk": "^2.4.2",
Expand Down
1 change: 0 additions & 1 deletion packages/solo/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
"@agoric/spawner": "^0.4.14",
"@agoric/store": "^0.4.14",
"@agoric/swing-store-lmdb": "^0.4.12",
"@agoric/swing-store-simple": "^0.3.9",
"@agoric/swingset-vat": "^0.17.2",
"@agoric/vats": "^0.2.2",
"anylogger": "^0.21.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/solo/src/access-token.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import crypto from 'crypto';
import os from 'os';
import path from 'path';

import { openSwingStore } from '@agoric/swing-store-simple';
import { openSwingStore } from '@agoric/swing-store-lmdb';

// Adapted from https://stackoverflow.com/a/43866992/14073862
export function generateAccessToken({
Expand Down Expand Up @@ -32,7 +32,7 @@ export async function getAccessToken(port) {
await fs.promises.mkdir(sharedStateDir, { mode: 0o700, recursive: true });

// Ensure an access token exists.
const { kvStore, commit, close } = openSwingStore(sharedStateDir, 'state');
const { kvStore, commit, close } = openSwingStore(sharedStateDir);
const accessTokenKey = `accessToken/${port}`;
if (!kvStore.has(accessTokenKey)) {
kvStore.set(accessTokenKey, await generateAccessToken());
Expand Down
12 changes: 0 additions & 12 deletions packages/swing-store-lmdb/test/test-state.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import 'node-lmdb';
import '@agoric/install-ses';

import fs from 'fs';
import path from 'path';

import test from 'ava';
import { getAllState } from '@agoric/swing-store-simple';
Expand Down Expand Up @@ -145,14 +144,3 @@ test('streamStore mode interlock', t => {
commit();
close();
});

test('rejectSimple under SES', t => {
const simpleDir = 'testdb-simple';
t.teardown(() => fs.rmdirSync(simpleDir, { recursive: true }));
fs.mkdirSync(simpleDir, { recursive: true });
fs.writeFileSync(
path.resolve(simpleDir, 'swingset-kernel-state.jsonlines'),
'some data\n',
);
t.is(isSwingStore(simpleDir), false);
});
187 changes: 48 additions & 139 deletions packages/swing-store-simple/src/simpleSwingStore.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
// @ts-check
import fs from 'fs';
import path from 'path';
import Readlines from 'n-readlines';

import { assert, details as X, q } from '@agoric/assert';

/**
Expand Down Expand Up @@ -35,29 +31,14 @@ import { assert, details as X, q } from '@agoric/assert';
* }} SwingStore
*/

function safeUnlink(filePath) {
try {
fs.unlinkSync(filePath);
} catch (e) {
if (e.code !== 'ENOENT') {
throw e;
}
}
}
const streamPeek = new WeakMap(); // for tests to get raw access to the streams

/**
* Create a new instance of a RAM-based implementation of the Storage API.
*
* The "Storage API" is a set of functions { has, getKeys, get, set, delete }
* that work on string keys and accept string values. A lot of kernel-side
* code expects to get an object that implements the Storage API.
* Do the work of `initSwingStore` and `openSwingStore`.
*
* @returns {{
* kvStore: KVStore, // the storage API object itself
* state: any, // the underlying map that holds the state in memory
* }}
* @returns {SwingStore}
*/
function makeStorageInMemory() {
function makeSwingStore() {
const state = new Map();

/**
Expand Down Expand Up @@ -153,78 +134,6 @@ function makeStorageInMemory() {
delete: del,
};

return harden({ kvStore, state });
}

const streamPeek = new WeakMap(); // for tests to get raw access to the streams

/**
* Do the work of `initSwingStore` and `openSwingStore`.
*
* @param {string} [dirPath] Path to a directory in which database files may be kept, or
* null.
* @param {boolean} [forceReset] If true, initialize the database to an empty state
*
* @returns {SwingStore}
*/
function makeSwingStore(dirPath, forceReset = false) {
const { kvStore, state } = makeStorageInMemory();

let storeFile;
if (dirPath) {
fs.mkdirSync(dirPath, { recursive: true });
storeFile = path.resolve(dirPath, 'swingset-kernel-state.jsonlines');
if (forceReset) {
safeUnlink(storeFile);
} else {
let lines;
try {
lines = new Readlines(storeFile);
} catch (e) {
// storeFile will be missing the first time we try to use it. That's OK;
// commit will create it.
if (e.code !== 'ENOENT') {
throw e;
}
}
if (lines) {
let line = lines.next();
while (line) {
// @ts-ignore JSON.parse can take a Buffer
const [key, value] = JSON.parse(line);
kvStore.set(key, value);
line = lines.next();
}
}
}
}

/**
* Commit unsaved changes.
*/
function commit() {
if (dirPath) {
const tempFile = `${storeFile}.tmp`;
const fd = fs.openSync(tempFile, 'w');

for (const [key, value] of state.entries()) {
const line = JSON.stringify([key, value]);
fs.writeSync(fd, line);
fs.writeSync(fd, '\n');
}
fs.closeSync(fd);
fs.renameSync(tempFile, storeFile);
}
}

/**
* Close the "database", abandoning any changes made since the last commit
* (if you want to save them, call commit() first).
*/
function close() {
// Nothing to do here.
}

/** @type {Map<string, Array<string>>} */
const streams = new Map();
/** @type {Map<string, string>} */
Expand Down Expand Up @@ -336,53 +245,75 @@ function makeSwingStore(dirPath, forceReset = false) {
return harden({ itemCount: position.itemCount + 1 });
}

const streamStore = harden({
const streamStore = {
readStream,
writeStreamItem,
closeStream,
STREAM_START,
});
};

/**
* Commit unsaved changes.
*/
function commit() {
// Nothing to do here.
}

/**
* Close the "database", abandoning any changes made since the last commit
* (if you want to save them, call commit() first).
*/
function close() {
// Nothing to do here.
}

streamPeek.set(streamStore, streams);

return harden({ kvStore, streamStore, commit, close });
}

/**
* Create a swingset store that is an in-memory map, normally backed by JSON
* serialized to a text file. If there is an existing store at the given
* `dirPath`, it will be reinitialized to an empty state.
* Create a swingset store that is an in-memory map.
*
* @param {string=} dirPath Path to a directory in which database files may be kept.
* This directory need not actually exist yet (if it doesn't it will be
* created) but it is reserved (by the caller) for the exclusive use of this
* swing store instance. If this is nullish, the swing store created will
* have no backing store and thus be non-persistent.
* @param {string=} dirPath Optional path to a directory in which database files
* might be kept, if this were a persistent form of swingset store. If a path
* is provided, a warning will be output to the console. This parameter is
* provided so that the in-memory store may be substituted for a persistent
* store for testing or debugging purposes without needing to change the
* client.
*
* @returns {SwingStore}
*/
export function initSwingStore(dirPath) {
if (dirPath !== null && dirPath !== undefined && `${dirPath}` !== dirPath) {
throw new Error('dirPath must be a string or nullish');
if (dirPath) {
console.log(
`Warning: initSwingStore ignoring dirPath, simpleStore is memory only`,
);
}
return makeSwingStore(dirPath, true);
return makeSwingStore();
}

/**
* Open a swingset store that is an in-memory map, backed by JSON serialized to
* a text file. If there is no existing store at the given `dirPath`, a new,
* empty store will be created.
* Open a swingset store that is an in-memory map. Note that "open" is a
* misnomer here, because you will always get a fresh, empty store. This entry
* point is provided for testing purposes only.
*
* @param {string} dirPath Path to a directory in which database files may be kept.
* This directory need not actually exist yet (if it doesn't it will be
* created) but it is reserved (by the caller) for the exclusive use of this
* swing store instance.
* @param {string=} dirPath Optional path to a directory in which database files
* might be kept, if this were a persistent form of swingset store. If a path
* is provided, a warning will be output to the console. This parameter is
* provided so that the in-memory store may be substituted for a persistent
* store for testing or debugging purposes without needing to change the
* client.
*
* @returns {SwingStore}
*/
export function openSwingStore(dirPath) {
assert.typeof(dirPath, 'string');
return makeSwingStore(dirPath, false);
if (dirPath) {
console.log(
`Warning: openSwingStore ignoring dirPath, simpleStore is memory only`,
);
}
return makeSwingStore();
}

/**
Expand Down Expand Up @@ -445,25 +376,3 @@ export function setAllState(swingStore, stuff) {
streamStore.closeStream(streamName);
}
}

/**
* Is this directory a compatible swing store?
*
* @param {string} dirPath Path to a directory in which database files might be present.
* This directory need not actually exist
*
* @returns {boolean}
* If the directory is present and contains the files created by initSwingStore
* or openSwingStore, returns true. Else returns false.
*
*/
export function isSwingStore(dirPath) {
assert.typeof(dirPath, 'string');
if (fs.existsSync(dirPath)) {
const storeFile = path.resolve(dirPath, 'swingset-kernel-state.jsonlines');
if (fs.existsSync(storeFile)) {
return true;
}
}
return false;
}
48 changes: 5 additions & 43 deletions packages/swing-store-simple/test/test-state.js
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,12 @@
import '@agoric/install-ses';

import fs from 'fs';
import path from 'path';

import test from 'ava';
import {
initSwingStore,
openSwingStore,
getAllState,
isSwingStore,
} from '../src/simpleSwingStore';

function testKVStore(t, store) {
import { initSwingStore, getAllState } from '../src/simpleSwingStore';

test('kvStore read/write', t => {
const store = initSwingStore();
const kvStore = store.kvStore;

t.falsy(kvStore.has('missing'));
t.is(kvStore.get('missing'), undefined);

Expand Down Expand Up @@ -44,38 +38,6 @@ function testKVStore(t, store) {
streamStuff: new Map(),
};
t.deepEqual(getAllState(store), reference, 'check state after changes');
}

test('storageInMemory', t => {
const store = initSwingStore();
testKVStore(t, store);
});

test('storageInFile', t => {
const dbDir = 'testdb';
t.teardown(() => fs.rmdirSync(dbDir, { recursive: true }));
fs.rmdirSync(dbDir, { recursive: true });
t.is(isSwingStore(dbDir), false);
const store = initSwingStore(dbDir);
const { commit, close } = store;
testKVStore(t, store);
commit();
const before = getAllState(store);
close();
t.is(isSwingStore(dbDir), true);

const afterStore = openSwingStore(dbDir);
t.deepEqual(getAllState(afterStore), before, 'check state after reread');
t.is(isSwingStore(dbDir), true);
});

test('rejectLMDB', t => {
const notSimpleDir = 'testdb-lmdb';
t.teardown(() => fs.rmdirSync(notSimpleDir, { recursive: true }));
fs.mkdirSync(notSimpleDir, { recursive: true });
fs.writeFileSync(path.resolve(notSimpleDir, 'data.mdb'), 'some data\n');
fs.writeFileSync(path.resolve(notSimpleDir, 'lock.mdb'), 'lock stuff\n');
t.is(isSwingStore(notSimpleDir), false);
});

test('streamStore read/write', t => {
Expand Down
Loading

0 comments on commit ef87997

Please sign in to comment.