Skip to content

Commit

Permalink
HCK-8736: hckFetch (#14)
Browse files Browse the repository at this point in the history
* feat: replaced aws-sdk with a dedicated `@aws-sdk/client-neptune` and our `hckFetch` request handler

* chore: prettier ignore package-lock.json

* chore: removed unused method
  • Loading branch information
chulanovskyi-bs authored Nov 14, 2024
1 parent 176fd12 commit 13613da
Show file tree
Hide file tree
Showing 13 changed files with 1,650 additions and 462 deletions.
1 change: 1 addition & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@
release
node_modules
package.json
package-lock.json
1 change: 1 addition & 0 deletions esbuild.package.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,6 @@ esbuild
}),
addReleaseFlag(path.resolve(RELEASE_FOLDER_PATH, 'package.json')),
],
external: ['electron'],
})
.catch(() => process.exit(1));
28 changes: 14 additions & 14 deletions forward_engineering/api.js
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
const gremlinHelper = require('./gremlinHelper');
const _ = require('lodash');
const { generateVertices, generateEdges } = require('./gremlinHelper');
const reverseEngineeringApi = require('../reverse_engineering/api');
const applyToInstanceHelper = require('./applyToInstanceHelper');

module.exports = {
generateContainerScript(data, logger, cb, app) {
let { collections, relationships, jsonData, containerData } = data;
generateContainerScript(data, logger, cb) {
let { collections, relationships, jsonData } = data;
logger.clear();
try {
const _ = app.require('lodash');
const helper = gremlinHelper(_);
let resultScript = '';

collections = collections.map(JSON.parse);
relationships = relationships.map(JSON.parse);

const verticesScript = helper.generateVertices(collections, jsonData);
const edgesScript = helper.generateEdges(collections, relationships, jsonData);
const verticesScript = generateVertices(collections, jsonData);
const edgesScript = generateEdges(collections, relationships, jsonData);

if (verticesScript) {
resultScript += verticesScript;
Expand All @@ -35,10 +34,11 @@ module.exports = {

async applyToInstance(data, logger, cb, app) {
try {
const _ = app.require('lodash');
const aws = app.require('aws-sdk');
const sshService = app.require('@hackolade/ssh-service');
const helper = applyToInstanceHelper(_, aws, sshService);
const { parseScriptStatements, getGremlinClient, runGremlinQueries } = applyToInstanceHelper({
sshService,
});

logger.clear();
logger.log('info', data, data.hiddenKeys);

Expand All @@ -49,16 +49,16 @@ module.exports = {

progress('Applying Gremlin script ...');

const { labels, edges } = helper.parseScriptStatements(data.script);
const gremlinClient = await helper.getGremlinClient(data);
const { labels, edges } = parseScriptStatements(data.script);
const gremlinClient = await getGremlinClient(data);

progress('Uploading vertices ...');

await helper.runGremlinQueries(gremlinClient, labels);
await runGremlinQueries(gremlinClient, labels);

progress('Uploading edges ...');

await helper.runGremlinQueries(gremlinClient, edges);
await runGremlinQueries(gremlinClient, edges);

cb();
} catch (err) {
Expand Down
12 changes: 6 additions & 6 deletions forward_engineering/applyToInstanceHelper.js
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
const { partition } = require('lodash');
const connectionHelper = require('../reverse_engineering/connectionHelper');
const neptuneHelper = require('../reverse_engineering/neptuneHelper');
const neptuneHelper = require('../shared/awsNeptuneClient');

const applyToInstanceHelper = (_, aws, sshService) => ({
const applyToInstanceHelper = ({ sshService }) => ({
async getGremlinClient(connectionInfo) {
const neptuneInstance = await neptuneHelper.connect(aws, connectionInfo);
const neptuneInstance = await neptuneHelper.connect({ connectionInfo });
const clusterInfo = await neptuneInstance.getBucketInfo();
const info = {
...connectionInfo,
host: clusterInfo.ReaderEndpoint,
port: clusterInfo.Port,
};
const connection = await connectionHelper.connect(info, sshService);
return connection;
return await connectionHelper.connect(info, sshService);
},

runGremlinQueries(gremlinClient, queries) {
Expand All @@ -24,7 +24,7 @@ const applyToInstanceHelper = (_, aws, sshService) => ({

parseScriptStatements(script) {
const scriptStatements = script.split('\n\n').map(item => item.replace(/\.\s+/g, '.'));
const [labels, edges] = _.partition(scriptStatements, statement => statement.startsWith('g.addV'));
const [labels, edges] = partition(scriptStatements, statement => statement.startsWith('g.addV'));

return { labels, edges };
},
Expand Down
239 changes: 119 additions & 120 deletions forward_engineering/gremlinHelper.js
Original file line number Diff line number Diff line change
@@ -1,149 +1,148 @@
const { isEmpty, isString, get } = require('lodash');

const DEFAULT_INDENT = ' ';
let graphName = 'g';
const graphName = 'g';

module.exports = _ => {
const generateVertex = (collection, vertexData) => {
const vertexName = transformToValidGremlinName(collection.collectionName);
const propertiesScript = addPropertiesScript(collection, vertexData);
const transformToValidGremlinName = name => {
const DEFAULT_NAME = 'New_vertex';
const DEFAULT_PREFIX = 'v_';

return `${graphName}.addV(${JSON.stringify(vertexName)})${propertiesScript}`;
};
if (!name || !isString(name)) {
return DEFAULT_NAME;
}

const generateVertices = (collections, jsonData) => {
const vertices = collections.map(collection => {
const vertexData = JSON.parse(jsonData[collection.GUID]);
const nameWithoutSpecialCharacters = name.replace(/[\s`~!@#%^&*()_|+\-=?;:'",.<>{}[\]\\/]/gi, '_');
const startsFromDigit = nameWithoutSpecialCharacters.match(/^\d.*$/);

return generateVertex(collection, vertexData);
});
if (startsFromDigit) {
return `${DEFAULT_PREFIX}_${nameWithoutSpecialCharacters}`;
}

const script = vertices.join(';\n\n');
if (!script) {
return '';
}
return nameWithoutSpecialCharacters;
};

return script + ';';
};
const generateVertex = (collection, vertexData) => {
const vertexName = transformToValidGremlinName(collection.collectionName);
const propertiesScript = addPropertiesScript(collection, vertexData);

const generateEdge = (from, to, relationship, edgeData) => {
const edgeName = transformToValidGremlinName(relationship.name);
const propertiesScript = addPropertiesScript(relationship, edgeData, { cardinality: false });
return `${graphName}.addV(${JSON.stringify(vertexName)})${propertiesScript}`;
};

const generateVertices = (collections, jsonData) => {
const vertices = collections.map(collection => {
const vertexData = JSON.parse(jsonData[collection.GUID]);

return `${graphName}.addE(${JSON.stringify(edgeName)}).\n${DEFAULT_INDENT}from(${from}).\n${DEFAULT_INDENT}to(${to})${propertiesScript}`;
};
return generateVertex(collection, vertexData);
});

const getVertexVariableScript = vertexName => `${graphName}.V().hasLabel(${JSON.stringify(vertexName)})`;
const script = vertices.join(';\n\n');
if (!script) {
return '';
}

const generateEdges = (collections, relationships, jsonData) => {
const edges = relationships.reduce((edges, relationship) => {
const parentCollection = collections.find(collection => collection.GUID === relationship.parentCollection);
const childCollection = collections.find(collection => collection.GUID === relationship.childCollection);
if (!parentCollection || !childCollection) {
return edges;
}
const from = transformToValidGremlinName(parentCollection.collectionName);
const to = transformToValidGremlinName(childCollection.collectionName);
const edgeData = JSON.parse(jsonData[relationship.GUID]);
return script + ';';
};

return edges.concat(
generateEdge(getVertexVariableScript(from), getVertexVariableScript(to), relationship, edgeData),
);
}, []);
const generateEdge = (from, to, relationship, edgeData) => {
const edgeName = transformToValidGremlinName(relationship.name);
const propertiesScript = addPropertiesScript(relationship, edgeData, { cardinality: false });

if (_.isEmpty(edges)) {
return '';
return `${graphName}.addE(${JSON.stringify(edgeName)}).\n${DEFAULT_INDENT}from(${from}).\n${DEFAULT_INDENT}to(${to})${propertiesScript}`;
};

const getVertexVariableScript = vertexName => `${graphName}.V().hasLabel(${JSON.stringify(vertexName)})`;

const generateEdges = (collections, relationships, jsonData) => {
const edges = relationships.reduce((edges, relationship) => {
const parentCollection = collections.find(collection => collection.GUID === relationship.parentCollection);
const childCollection = collections.find(collection => collection.GUID === relationship.childCollection);
if (!parentCollection || !childCollection) {
return edges;
}
const from = transformToValidGremlinName(parentCollection.collectionName);
const to = transformToValidGremlinName(childCollection.collectionName);
const edgeData = JSON.parse(jsonData[relationship.GUID]);

return edges.join(';\n\n') + ';';
};
return edges.concat(
generateEdge(getVertexVariableScript(from), getVertexVariableScript(to), relationship, edgeData),
);
}, []);

const addPropertiesScript = (collection, vertexData, features = { cardinality: true }) => {
const properties = _.get(collection, 'properties', {});
if (isEmpty(edges)) {
return '';
}

return Object.keys(properties).reduce((script, name) => {
const property = properties[name];
return edges.join(';\n\n') + ';';
};

return script + createProperty(property, name, vertexData[name], features);
}, '');
};
const addPropertiesScript = (collection, vertexData, features = { cardinality: true }) => {
const properties = get(collection, 'properties', {});

const createProperty = (property, name, vertexData, features) => {
const type = property.type;
return Object.keys(properties).reduce((script, name) => {
const property = properties[name];

if (type === 'multi-property') {
return convertSet(property, name, vertexData, features);
}
return script + createProperty(property, name, vertexData[name], features);
}, '');
};

const valueScript = convertPropertyValue(property, 2, type, vertexData);
const cardinality = features.cardinality ? `${property.propCardinality || 'single'}, ` : '';

return `.\n${DEFAULT_INDENT}property(${cardinality}${JSON.stringify(name)}, ${valueScript})`;
};

const convertSet = (property, name, vertexData, features) => {
const items = Array.isArray(property.items) ? property.items : [property.items];

return items
.map((item, i) =>
createProperty(
{ ...item, propCardinality: 'set' },
name,
Array.isArray(vertexData) ? vertexData[i] || '' : '',
features,
),
)
.join('');
};

const convertDate = value => `datetime("${value}")`;

const convertNumber = (property, value) => {
const mode = property.mode;
const numberValue = JSON.stringify(value);

switch (mode) {
case 'double':
return `${numberValue}d`;
case 'float':
return `${numberValue}f`;
case 'long':
return `${numberValue}l`;
default:
return numberValue;
}
};

const convertPropertyValue = (property, level, type, value) => {
switch (type) {
case 'date':
return convertDate(value);
case 'number':
return convertNumber(property, value);
}
const createProperty = (property, name, vertexData, features) => {
const type = property.type;

return `${JSON.stringify(value === undefined ? '' : value)}`;
};
if (type === 'multi-property') {
return convertSet(property, name, vertexData, features);
}

const transformToValidGremlinName = name => {
const DEFAULT_NAME = 'New_vertex';
const DEFAULT_PREFIX = 'v_';
const valueScript = convertPropertyValue(property, 2, type, vertexData);
const cardinality = features.cardinality ? `${property.propCardinality || 'single'}, ` : '';

if (!name || !_.isString(name)) {
return DEFAULT_NAME;
}
return `.\n${DEFAULT_INDENT}property(${cardinality}${JSON.stringify(name)}, ${valueScript})`;
};

const nameWithoutSpecialCharacters = name.replace(/[\s`~!@#%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/gi, '_');
const startsFromDigit = nameWithoutSpecialCharacters.match(/^[0-9].*$/);
const convertSet = (property, name, vertexData, features) => {
const items = Array.isArray(property.items) ? property.items : [property.items];

return items
.map((item, i) =>
createProperty(
{ ...item, propCardinality: 'set' },
name,
Array.isArray(vertexData) ? vertexData[i] || '' : '',
features,
),
)
.join('');
};

if (startsFromDigit) {
return `${DEFAULT_PREFIX}_${nameWithoutSpecialCharacters}`;
}
const convertDate = value => `datetime("${value}")`;

const convertNumber = (property, value) => {
const mode = property.mode;
const numberValue = JSON.stringify(value);

switch (mode) {
case 'double':
return `${numberValue}d`;
case 'float':
return `${numberValue}f`;
case 'long':
return `${numberValue}l`;
default:
return numberValue;
}
};

return nameWithoutSpecialCharacters;
};
const convertPropertyValue = (property, level, type, value) => {
switch (type) {
case 'date':
return convertDate(value);
case 'number':
return convertNumber(property, value);
}

return `${JSON.stringify(value === undefined ? '' : value)}`;
};

return {
transformToValidGremlinName,
generateVertices,
generateEdges,
};
module.exports = {
generateVertices,
generateEdges,
};
Loading

0 comments on commit 13613da

Please sign in to comment.