assets on file but not in index will now save to index on submission r…

…esolves #586 (#588)
terascope · Oct 30, 2017 · bdbe05a · bdbe05a
1 parent 73feeb2
commit bdbe05a
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 4 deletions.
diff --git a/lib/cluster/storage/assets.js b/lib/cluster/storage/assets.js
@@ -6,6 +6,7 @@ var crypto = require('crypto');
 var _ = require('lodash');
 var parseError = require('../../utils/error_utils').parseError;
 var saveAsset = require('../../utils/file_utils').saveAsset;
+var normalizeZipFile = require('../../utils/file_utils').normalizeZipFile;
 var fse = require('fs-extra');
 
 // Module to manager job states in Elasticsearch.
@@ -24,10 +25,31 @@ module.exports = function module(context) {
         var esData = data.toString('base64');
         var id = crypto.createHash('sha1').update(esData).digest("hex");
         return fs.openAsync(`${assets_path}/${id}`, 'r')
-            .then(function(alreadyExists) {
-                return id;
+            .then(function() {
+                // directory exists, need to check if the index has it, if not then save it
+                return backend.get(id, null, ['name'])
+                // it exists, just return the id
+                    .then(() => id)
+                    .catch((err) => {
+                        if (err === 'Not Found') {
+                            logger.info(`asset: ${id} exists on disk but not in index, saving asset to index`);
+                            return normalizeZipFile(id, `${assets_path}/${id}`, logger)
+                                .then((_metaData) => {
+                                    metaData = _metaData;
+                                    const fileData = { blob: esData, _created: new Date() };
+                                    const file = _.assign(fileData, _metaData);
+                                    return backend.indexWithId(id, file);
+                                })
+                                .then(() => {
+                                    logger.info(`assets: ${metaData.name}, id: ${id} has been saved to elasticsearch`);
+                                    return id;
+                                });
+                        }
+                        return Promise.reject(`error checking asset index, could not get asset with id: ${id}`);
+                    });
             })
             .catch(function(err) {
+                // directory does not exist, need to save it
                 if (err.code === 'ENOENT') {
                     return Promise.resolve(saveAsset(logger, assets_path, id, data, _metaIsUnqiue))
                         .then(function(_metaData) {

diff --git a/lib/cluster/storage/backends/elasticsearch_store.js b/lib/cluster/storage/backends/elasticsearch_store.js
@@ -19,14 +19,18 @@ module.exports = function module(context, indexName, recordType, idField, _bulkS
     let bulkSize = 500;
     if (_bulkSize) bulkSize = _bulkSize;
 
-    function getRecord(recordId, indexArg) {
+    function getRecord(recordId, indexArg, fields) {
         logger.trace(`getting record id: ${recordId}`);
         const query = {
             index: indexArg || indexName,
             type: recordType,
             id: recordId
         };
 
+        if (fields) {
+            query._source = fields;
+        }
+
         return elasticsearch.get(query);
     }
 

diff --git a/lib/utils/file_utils.js b/lib/utils/file_utils.js
@@ -128,5 +128,6 @@ function saveAsset(logger, assets_path, id, binaryData, metaCheck) {
 module.exports = {
     existsSync: existsSync,
     saveAsset: saveAsset,
-    deleteDir: deleteDir
+    deleteDir: deleteDir,
+    normalizeZipFile: normalizeZipFile
 };