Persistent storage with google cloud

[TomokoOG]

Eh, I think it was obvious I couldn't deal with that myself either, sowwy.

After dropbox I tried giving this https://github.com/VeliovGroup/Meteor-Files/wiki/Google-Cloud-Storage-Integration a try. How I went:

• Installed all the packages mentioned in the guide
• Did all the google cloud configurations (downloaded json file etc.)
• then took the code from the guide and inserted my stuff:

import { Meteor } from 'meteor/meteor';
import { Random } from 'meteor/random'
import { FilesCollection } from 'meteor/ostrio:files';

let gcloud, gcs, bucket, bucketMetadata, Request, bound, Collections = {};

if (Meteor.isServer) {
  // use require() as "'import' and 'export' may only appear at the top level"
  const Random = require('meteor/random');
  const Storage = require('@google-cloud/storage');
  gcs = new Storage('google-cloud')({
    projectId: 'my-project-123', // <-- Replace this with your project ID
    keyFilename: 'imports\api\bucketkey.json'  // <-- Replace this with the path to your key.json
  });
  bucket = gcs.bucket('my_bucket'); // <-- Replace this with your bucket name
  bucket.getMetadata(function(error, metadata, apiResponse) {
    if (error) {
      console.error(error);
    }
  });
  Request = Npm.require('request');
  bound = Meteor.bindEnvironment(function(callback) {
    return callback();
  });
}

Collections.files = new FilesCollection({
  debug: false, // Set to true to enable debugging messages
  storagePath: 'assets/app/uploads/uploadedFiles',
  collectionName: 'Images',
  allowClientCode: false,
  onAfterUpload(fileRef) {
    // In the onAfterUpload callback, we will move the file to Google Cloud Storage
    _.each(fileRef.versions, (vRef, version) => {
      // We use Random.id() instead of real file's _id
      // to secure files from reverse engineering
      // As after viewing this code it will be easy
      // to get access to unlisted and protected files
      const filePath = 'files/' + (Random.id()) + '-' + version + '.' + fileRef.extension;
      // Here we set the neccesary options to upload the file, for more options, see
      // https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.36.0/storage/bucket?method=upload
      const options = {
        destination: filePath,
        resumable: true
      };

      bucket.upload(fileRef.path, options, (error, file) => {
        bound(() => {
          let upd;
          if (error) {
            console.error(error);
          } else {
            upd = {
              $set: {}
            };
            upd['$set'][`versions.${version}.meta.pipePath`] = filePath;
            this.collection.update({
              _id: fileRef._id
            }, upd, (updError) => {
              if (updError) {
                console.error(updError);
              } else {
                // Unlink original files from FS
                // after successful upload to Google Cloud Storage
                this.unlink(this.collection.findOne(fileRef._id), version);
              }
            });
          }
        });
      });
    });
  },
  interceptDownload(http, fileRef, version) {
    let ref, ref1, ref2;
    const path = (ref= fileRef.versions) != null ? (ref1 = ref[version]) != null ? (ref2 = ref1.meta) != null ? ref2.pipePath : void 0 : void 0 : void 0;
    const vRef = ref1;
    if (path) {
      // If file is moved to Google Cloud Storage
      // We will pipe request to Google Cloud Storage
      // So, original link will stay always secure
      const remoteReadStream = getReadableStream(http, path, vRef);
      this.serve(http, fileRef, vRef, version, remoteReadStream);
      return true;
    }
    // While the file has not been uploaded to Google Cloud Storage, we will serve it from the filesystem
    return false;
  }
});

if (Meteor.isServer) {
  // Intercept file's collection remove method to remove file from Google Cloud Storage
  const _origRemove = Collections.files.remove;

  Collections.files.remove = function(search) {
    const cursor = this.collection.find(search);
    cursor.forEach((fileRef) => {
      _.each(fileRef.versions, (vRef) => {
        let ref;
        if (vRef != null ? (ref = vRef.meta) != null ? ref.pipePath : void 0 : void 0) {
          bucket.file(vRef.meta.pipePath).delete((error) => {
            bound(() => {
              if (error) {
                console.error(error);
              }
            });
          });
        }
      });
    });
    // Call the original removal method
    _origRemove.call(this, search);
  };
}

function getReadableStream(http, path, vRef){
  let array, end, partial, remoteReadStream, reqRange, responseType, start, take;

  if (http.request.headers.range) {
    partial = true;
    array = http.request.headers.range.split(/bytes=([0-9]*)-([0-9]*)/);
    start = parseInt(array[1]);
    end = parseInt(array[2]);
    if (isNaN(end)) {
      end = vRef.size - 1;
    }
    take = end - start;
  } else {
    start = 0;
    end = vRef.size - 1;
    take = vRef.size;
  }

  if (partial || (http.params.query.play && http.params.query.play === 'true')) {
    reqRange = {
      start: start,
      end: end
    };
    if (isNaN(start) && !isNaN(end)) {
      reqRange.start = end - take;
      reqRange.end = end;
    }
    if (!isNaN(start) && isNaN(end)) {
      reqRange.start = start;
      reqRange.end = start + take;
    }
    if ((start + take) >= vRef.size) {
      reqRange.end = vRef.size - 1;
    }
    if ((reqRange.start >= (vRef.size - 1) || reqRange.end > (vRef.size - 1))) {
      responseType = '416';
    } else {
      responseType = '206';
    }
  } else {
    responseType = '200';
  }

  if (responseType === '206') {
    remoteReadStream = bucket.file(path).createReadStream({
      start: reqRange.start,
      end: reqRange.end
    });
  } else if (responseType === '200') {
    remoteReadStream = bucket.file(path).createReadStream();
  }

  return remoteReadStream;
}

This error occurs the moment I save the images.js file with that code:

W20190907-01:17:32.025(2)? (STDERR) C:\Users\NERV\AppData\Local\.meteor\packages\meteor-tool\1.8.1\mt-os.windows.x86_64\dev_bundle\server-lib\node_modules\fibers\future.js:280
W20190907-01:17:32.027(2)? (STDERR)                                             throw(ex);
W20190907-01:17:32.030(2)? (STDERR)                                             ^
W20190907-01:17:32.030(2)? (STDERR)
W20190907-01:17:32.031(2)? (STDERR) TypeError: Storage is not a constructor
W20190907-01:17:32.032(2)? (STDERR)     at images.js (imports/api/images/images.js:62:9)
W20190907-01:17:32.032(2)? (STDERR)     at fileEvaluate (packages\modules-runtime.js:336:7)
W20190907-01:17:32.033(2)? (STDERR)     at Module.require (packages\modules-runtime.js:238:14)
W20190907-01:17:32.034(2)? (STDERR)     at Module.moduleLink [as link] (C:\Users\NERV\AppData\Local\.meteor\packages\modules\0.13.0\npm\node_modules\reify\lib\runtime\index.js:38:38)
W20190907-01:17:32.035(2)? (STDERR)     at methods.js (imports/api/characters/methods.js:1:463)
W20190907-01:17:32.037(2)? (STDERR)     at fileEvaluate (packages\modules-runtime.js:336:7)
W20190907-01:17:32.042(2)? (STDERR)     at Module.require (packages\modules-runtime.js:238:14)
W20190907-01:17:32.043(2)? (STDERR)     at Module.moduleLink [as link] (C:\Users\NERV\AppData\Local\.meteor\packages\modules\0.13.0\npm\node_modules\reify\lib\runtime\index.js:38:38)
W20190907-01:17:32.044(2)? (STDERR)     at main.js (server/main.js:1:146)
W20190907-01:17:32.045(2)? (STDERR)     at fileEvaluate (packages\modules-runtime.js:336:7)
W20190907-01:17:32.045(2)? (STDERR)     at Module.require (packages\modules-runtime.js:238:14)
W20190907-01:17:32.048(2)? (STDERR)     at require (packages\modules-runtime.js:258:21)
W20190907-01:17:32.049(2)? (STDERR)     at C:\Users\NERV\Desktop\Projekte\DDCFull\.meteor\local\build\programs\server\app\app.js:4682:1
W20190907-01:17:32.049(2)? (STDERR)     at C:\Users\NERV\Desktop\Projekte\DDCFull\.meteor\local\build\programs\server\boot.js:419:36
W20190907-01:17:32.052(2)? (STDERR)     at Array.forEach (<anonymous>)
W20190907-01:17:32.052(2)? (STDERR)     at C:\Users\NERV\Desktop\Projekte\DDCFull\.meteor\local\build\programs\server\boot.js:228:19

Unable to resolve some modules:

  "worker_threads" in /C/Users/NERV/Desktop/Projekte/DDCFull/node_modules/write-file-atomic/index.js (web.browser.legacy)

If you notice problems related to these missing modules, consider running:

  meteor npm install --save worker_threads

=> Exited with code: 1
=> Your application is crashing. Waiting for file change.

I guess it's about npm install @google-cloud/storage" ? I installed it as it is. I installed it while the cmd was inside my project directory, without writing meteor. Regarding the unresolved moduls "worker_threads" I tried doing what the log told me by doing meteor npm install --save worker_threads but it put out this

C:\Users\NERV\Desktop\Projekte\DDCFull>meteor npm install --save worker_threads
npm ERR! code E404
npm ERR! 404 Not Found - GET https://registry.npmjs.org/worker_threads - Not found
npm ERR! 404
npm ERR! 404  'worker_threads@latest' is not in the npm registry.
npm ERR! 404 You should bug the author to publish it (or use the name yourself!)
npm ERR! 404
npm ERR! 404 Note that you can also install from a
npm ERR! 404 tarball, folder, http url, or git url.

npm ERR! A complete log of this run can be found in:
npm ERR!     C:\Users\NERV\AppData\Roaming\npm-cache\_logs\2019-09-06T23_26_45_570Z-debug.log

Which makes me think this is not what I should do. Thank you for any help.

[dr-dimitru]

@TomokoOG

1. Try to use older versions of @google-cloud/storage library, previous releases can be found here. To install use: meteor npm install @google-storage/cloud@version --save
2. Make sure this library supports (your platform) Windows
3. Maybe author of this tutorial @salmanhasni can give you more info

[TomokoOG]

If I want to install an older version I just have to put the number version in where version is?
I tried installing 2.1.0 like so meteor npm install @google-storage/cloud@2.1.0 --save But this error occurs.

C:\Users\NERV\Desktop\Projekte\DDCFull>meteor npm install @google-storage/cloud@2.1.0 --save
npm ERR! code E404
npm ERR! 404 Not Found - GET https://registry.npmjs.org/@google-storage%2fcloud - Not found
npm ERR! 404
npm ERR! 404  '@google-storage/cloud@2.1.0' is not in the npm registry.
npm ERR! 404 You should bug the author to publish it (or use the name yourself!)
npm ERR! 404
npm ERR! 404 Note that you can also install from a
npm ERR! 404 tarball, folder, http url, or git url.

npm ERR! A complete log of this run can be found in:
npm ERR!     C:\Users\NERV\AppData\Roaming\npm-cache\_logs\2019-09-07T02_00_59_714Z-debug.log

[dr-dimitru]

@TomokoOG same for me here, can not install @google-storage/cloud with or without version constraint it always returns 404 error, you should file a bug on their repository

[TomokoOG]

I found this GoogleCloudPlatform/nodejs-docs-samples#731 where it's mentioned that instead of
const storage = new Storage.Storage(); you got to use
const {Storage} = require('@google-cloud/storage');
This doesn't solve my problem. However, the error changes:

Storage seems to be found this way, I guess

W20190907-14:12:37.825(2)? (STDERR) C:\Users\NERV\AppData\Local\.meteor\packages\meteor-tool\1.8.1\mt-os.windows.x86_64\dev_bundle\server-lib\node_modules\fibers\future.js:280
W20190907-14:12:37.827(2)? (STDERR)                                             throw(ex);
W20190907-14:12:37.828(2)? (STDERR)                                             ^
W20190907-14:12:37.829(2)? (STDERR)
W20190907-14:12:37.829(2)? (STDERR) TypeError: Cannot create property 'apiEndpoint' on string 'google-cloud'
W20190907-14:12:37.830(2)? (STDERR)     at new Storage (C:\Users\NERV\Desktop\Projekte\DDCFull\node_modules\@google-cloud\storage\build\src\storage.js:92:29)
W20190907-14:12:37.831(2)? (STDERR)     at images.js (imports/api/images/images.js:63:9)
W20190907-14:12:37.831(2)? (STDERR)     at fileEvaluate (packages\modules-runtime.js:336:7)
W20190907-14:12:37.832(2)? (STDERR)     at Module.require (packages\modules-runtime.js:238:14)
W20190907-14:12:37.833(2)? (STDERR)     at Module.moduleLink [as link] (C:\Users\NERV\AppData\Local\.meteor\packages\modules\0.13.0\npm\node_modules\reify\lib\runtime\index.js:38:38)
W20190907-14:12:37.833(2)? (STDERR)     at methods.js (imports/api/characters/methods.js:1:463)
W20190907-14:12:37.834(2)? (STDERR)     at fileEvaluate (packages\modules-runtime.js:336:7)
W20190907-14:12:37.835(2)? (STDERR)     at Module.require (packages\modules-runtime.js:238:14)
W20190907-14:12:37.838(2)? (STDERR)     at Module.moduleLink [as link] (C:\Users\NERV\AppData\Local\.meteor\packages\modules\0.13.0\npm\node_modules\reify\lib\runtime\index.js:38:38)
W20190907-14:12:37.842(2)? (STDERR)     at main.js (server/main.js:1:146)
W20190907-14:12:37.843(2)? (STDERR)     at fileEvaluate (packages\modules-runtime.js:336:7)
W20190907-14:12:37.844(2)? (STDERR)     at Module.require (packages\modules-runtime.js:238:14)
W20190907-14:12:37.845(2)? (STDERR)     at require (packages\modules-runtime.js:258:21)
W20190907-14:12:37.845(2)? (STDERR)     at C:\Users\NERV\Desktop\Projekte\DDCFull\.meteor\local\build\programs\server\app\app.js:4688:1
W20190907-14:12:37.848(2)? (STDERR)     at C:\Users\NERV\Desktop\Projekte\DDCFull\.meteor\local\build\programs\server\boot.js:419:36
W20190907-14:12:37.849(2)? (STDERR)     at Array.forEach (<anonymous>)

Unable to resolve some modules:

  "worker_threads" in /C/Users/NERV/Desktop/Projekte/DDCFull/node_modules/write-file-atomic/index.js (web.browser.legacy)

If you notice problems related to these missing modules, consider running:

  meteor npm install --save worker_threads

=> Exited with code: 1
=> Your application is crashing. Waiting for file change.

Following the tutorial that was mentioned in the link above I also saw irregularities concerning
const storage = new Storage({projectId, keyFilename}) in the tutorial.
https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/eefa4303ec2a9e5b828b2169645d2ae59a9cf949/auth/auth.js#L59
On the guide

 gcs = new Storage('google-cloud')({
    projectId: 'my-project-123', // <-- Replace this with your project ID
    keyFilename: 'imports\api\bucketkey.json'  // <-- Replace this with the path to your key.json
  });

So I changed it to this

    projectId: 'my-project-****', // <-- Replace this with your project ID
    keyFilename: './bucketkey.json'  // <-- Replace this with the path to your key.json
  });

which got rid of the W20190907-14:12:37.829(2)? (STDERR) TypeError: Cannot create property 'apiEndpoint' on string 'google-cloud'-Error

W20190907-14:36:15.805(2)? (STDERR) { Error: ENOENT: no such file or directory, open 'C:\Users\NERV\Desktop\Projekte\DDCFull\.meteor\local\build\programs\server\bucketkey.json'
W20190907-14:36:15.807(2)? (STDERR)   errno: -4058,
W20190907-14:36:15.807(2)? (STDERR)   code: 'ENOENT',
W20190907-14:36:15.808(2)? (STDERR)   syscall: 'open',
W20190907-14:36:15.809(2)? (STDERR)   path: 'C:\\Users\\NERV\\Desktop\\Projekte\\DDCFull\\.meteor\\local\\build\\programs\\server\\bucketkey.json' }
=> Meteor server restarted
Unable to resolve some modules:

  "worker_threads" in /C/Users/NERV/Desktop/Projekte/DDCFull/node_modules/write-file-atomic/index.js (web.browser.legacy)

If you notice problems related to these missing modules, consider running:

  meteor npm install --save worker_threads

But this error occurs. If the json file is in the same folder as the code it should work with "./bucketkey.json" doesn't it?

[dr-dimitru]

@TomokoOG

But this error occurs. If the json file is in the same folder as the code it should work with "./bucketkey.json" doesn't it?

Take a look on meteor-root package, which would help you to get correct path to your static file.

I recommend to put bucketkey.json into your-project-directory/private/bucketkey.json and then:

  gcs = new Storage('google-cloud')({
    projectId: 'my-project-123',
    keyFilename:`${Meteor.rootPath}/assets/app/bucketkey.json`
  });

[TomokoOG]

Great, that package removed the error, thanks! This remains though:

Unable to resolve some modules:

  "worker_threads" in /C/Users/NERV/Desktop/Projekte/DDCFull/node_modules/write-file-atomic/index.js (web.browser.legacy)

If you notice problems related to these missing modules, consider running:

  meteor npm install --save worker_threads

Unresolved modules. Any thoughts about that?

Also, when I try to upload now while on production the image gets uploaded and displayed but not sent to the bucket. Error from the console

I20190911-09:24:00.890(2)? Exception in callback of async function: TypeError: Cannot read property 'upload' of undefined
I20190911-09:24:00.890(2)?     at _.each (imports/api/images/images.js:74:14)
I20190911-09:24:00.891(2)?     at Function._.each._.forEach (packages\underscore.js:147:22)
I20190911-09:24:00.891(2)?     at FilesCollection.onAfterUpload (imports/api/images/images.js:61:7)
I20190911-09:24:00.891(2)?     at _preCollection.update.preUpdateError (packages/ostrio:files/server.js:958:54)
I20190911-09:24:00.891(2)?     at packages/mongo/collection.js:729:7
I20190911-09:24:00.892(2)?     at runWithEnvironment (packages\meteor.js:1286:24)
I20190911-09:24:00.892(2)?     at packages\meteor.js:1299:14
I20190911-09:24:00.892(2)?     at packages/mongo/mongo_driver.js:331:7
I20190911-09:24:00.892(2)?     at packages/mongo/mongo_driver.js:618:15
I20190911-09:24:00.893(2)?     at runWithEnvironment (packages\meteor.js:1286:24)

Error in browser console

:3000/cdn/storage/Images/may47xDadMbgTckGp/original/may47xDadMbgTckGp.png:1 Failed to load resource: the server responded with a status of 404 (Not Found)

Also, I got one question about this part in the code:

Collections.files = new FilesCollection({ .../

Now in the old code I use, from the mainpage (https://github.com/VeliovGroup/Meteor-Files) I exported this FilesCollection. It's not happening in this code and I wonder why. This upload-function is supposed to be used by normal users and I kinda need to export it when I want to go to the clientside with it, don't I?

[dr-dimitru]

Hey @TomokoOG ,

Related to worker_threads issue (just google it, it isn't nice to ask every step on GitHub), here is first result from Google — nodejs/help#1498

TL;DR; find and downgrade @google-storage/cloud to a version where worker_threads not used, this is bleeding edge technology (at least for meteor environment).

And I didn't get your second question

[TomokoOG]

I apologize. I will improve my behavior in the future. I have googled this github post before but it didn't help me that much. I downgraded already to 2.1.0 and it's still showing me this error. And this version was from 2 years ago, I think? However, I'll keep downgrading and see.

I'm sorry about the second question. There was some confusion because I altered the original code of the api overview code https://github.com/VeliovGroup/Meteor-Files#api-overview-full-api. I exported it like so

export const Images = new FilesCollection({
  collectionName: 'Images',
  storagePath: '/Dev/',
  parentDirPermissions: 0774,
  permissions: 0774,
   allowClientCode: true, // Disallow remove files from Client
  onBeforeUpload(file) {
    // Allow upload files under 10MB, and only in png/jpg/jpeg formats
    if (file.size <= 10485760 && /png|jpg|jpeg/i.test(file.extension)) {
      return true;
    }
    return 'Please upload image, with size equal or less than 10MB';
  }

});

and wondered why in the google cloud guide the fileCollection is not exported.

[TomokoOG]

I think I will look for other solutions. This doesn't seem to work on any version. Thanks for the help though.

[dr-dimitru]

and wondered why in the google cloud guide the fileCollection is not exported.

Just not part of example, sure you got to export variables you want to use across the application in other files.

I apologize. I will improve my behavior in the future.

No worries

I have googled this github post before but it didn't help me that much. I downgraded already to 2.1.0 and it's still showing me this error. A

This is related to @google-cloud/storage package, right? So, it would be much better and efficient to ask for solution in their official repo. But again as mentioned here you simply need to run your node.js process with --experimental-worker

Hope that helps :)

Please support this project with:

• Star this package on GitHub
• Like at Facebook
• Follow on Twitter
• Support via patreon;
• Support via PayPal;
• Check and star our other open source projects on GitHub

[dr-dimitru]

@TomokoOG got an idea which may help you, could you post content of your package.json?

Idea is that you might be missing meteor-node-stubs package, make sure it's installed and updated to the latest available version:

meteor npm install --save meteor-node-stubs

[TomokoOG]

Thank you! But I switched to cloudinary for file-storage and upload, I'm sorry.

[dr-dimitru]

@TomokoOG no worries, if this tool better suits your needs, it's the right choice