Skip to content

Commit

Permalink
feat: add service upload json data and stream data
Browse files Browse the repository at this point in the history
  • Loading branch information
@pornchaitippawan
pornchaitippawan committed Mar 11, 2024
1 parent 23da285 commit f43c3ef
Show file tree
Hide file tree
Showing 4 changed files with 916 additions and 247 deletions.
176 changes: 175 additions & 1 deletion src/__tests__/cloud-storage.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import CloudStorageService from '../services/cloud-storage';
import { PassThrough, Readable } from 'stream';
import fs from 'fs';
import { v4 as uuidv4 } from 'uuid';
import { UploadStreamDescriptorWithPathType, UploadedJsonType } from '../types/upload-binary';

// Mock the entire uuid module
jest.mock('uuid');
Expand Down Expand Up @@ -102,7 +103,9 @@ describe('Cloud Storage', () => {

cloudStorageService.bucket_.upload = jest.fn().mockResolvedValue([
{
getSignedUrl: jest.fn().mockResolvedValue(['https://test.com/mock-bucket/uuid/mock-file.jpg']),
getSignedUrl: jest
.fn()
.mockResolvedValue(['https://test.com/mock-bucket/uuid/mock-file.jpg'])
}
]);

Expand Down Expand Up @@ -503,4 +506,175 @@ describe('Cloud Storage', () => {
expect(cloudStorageService.bucket_.file().exists).toHaveBeenCalled();
expect(cloudStorageService.bucket_.file().getSignedUrl).not.toHaveBeenCalled();
});

test('should success upload stream json with data (public)', async () => {
const dataMock = {
data: {
test: 'test-data-json'
},
path: 'import-product',
name: 'test02.json'
} as UploadedJsonType;
const mockExpectedValue = {
url: 'https://test.com/mock-bucket/uuid/test02.json',
key: 'import-product/test02.json'
};
const extension = '.json';
const fileName = dataMock.name.replace(/\.[^/.]+$/, '');
const destinationMock = `${dataMock.path}/${fileName}${extension}`;
// Mock the bucket and file objects
const mockObjFile = {
createWriteStream: jest.fn().mockReturnValue(new PassThrough()),
publicUrl: jest.fn().mockResolvedValue(mockExpectedValue.url),
cloudStorageURI: {
href: `gs://${rootBucketName}/${dataMock.path}/${dataMock.name}`
}
};
cloudStorageService.bucket_.file = jest.fn().mockReturnValue(mockObjFile);

const result = await cloudStorageService.uploadStreamJson(dataMock);

//Assertions
expect(result).toEqual(mockExpectedValue);
expect(cloudStorageService.bucket_.file).toHaveBeenCalledWith(destinationMock);
expect(cloudStorageService.bucket_.file().createWriteStream).toHaveBeenCalled();
expect(cloudStorageService.bucket_.file().publicUrl).toHaveBeenCalled();
});

test('should success upload stream json with data (private)', async () => {
const dataMock = {
data: {
test: 'test-data-json'
},
path: 'import-product',
name: 'test02.json',
isPrivate: true
} as UploadedJsonType;
const mockExpectedValue = {
url: 'https://test.com/mock-bucket/uuid/test02.json',
key: 'import-product/test02.json'
};
const extension = '.json';
const fileName = dataMock.name.replace(/\.[^/.]+$/, '');
const destinationMock = `${dataMock.path}/${fileName}${extension}`;
// Mock the bucket and file objects
const mockObjFile = {
createWriteStream: jest.fn().mockReturnValue(new PassThrough()),
publicUrl: jest.fn().mockResolvedValue(mockExpectedValue.url),
cloudStorageURI: {
href: `gs://${rootBucketName}/${dataMock.path}/${dataMock.name}`
}
};
cloudStorageService.bucket_.file = jest.fn().mockReturnValue(mockObjFile);

const result = await cloudStorageService.uploadStreamJson(dataMock);

//Assertions
expect(result).toEqual({
...mockExpectedValue,
url: `gs://${rootBucketName}/${dataMock.path}/${dataMock.name}`
});
expect(cloudStorageService.bucket_.file).toHaveBeenCalledWith(destinationMock);
expect(cloudStorageService.bucket_.file().createWriteStream).toHaveBeenCalled();
expect(cloudStorageService.bucket_.file().publicUrl).not.toHaveBeenCalled();
});

test('should success upload with stream data (public)', async () => {
const dataMock = {
path: 'import-product',
name: 'test.json',
ext: 'json'
} as UploadStreamDescriptorWithPathType;
const mockExpectedValue = {
url: 'https://test.com/mock-bucket/uuid/test.json',
key: 'import-product/test.json'
};
const extension = '.json';
const fileName = dataMock.name.replace(/\.[^/.]+$/, '');
const destinationMock = `${dataMock.path}/${fileName}${extension}`;
// Mock the bucket and file objects
const mockObjFile = {
createWriteStream: jest.fn().mockReturnValue(new PassThrough()),
publicUrl: jest.fn().mockResolvedValue(mockExpectedValue.url),
cloudStorageURI: {
href: `gs://${rootBucketName}/${dataMock.path}/${dataMock.name}`
}
};
cloudStorageService.bucket_.file = jest.fn().mockReturnValue(mockObjFile);

const result = await cloudStorageService.uploadStream(dataMock, Buffer.from('test'));

//Assertions
expect(result).toEqual(mockExpectedValue);
expect(cloudStorageService.bucket_.file).toHaveBeenCalledWith(destinationMock);
expect(cloudStorageService.bucket_.file().createWriteStream).toHaveBeenCalled();
expect(cloudStorageService.bucket_.file().publicUrl).toHaveBeenCalled();
});

test('should success upload with stream data (private)', async () => {
const dataMock = {
path: 'import-product',
name: 'test.json',
ext: 'json',
isPrivate: true
} as UploadStreamDescriptorWithPathType;
const mockExpectedValue = {
url: 'https://test.com/mock-bucket/uuid/test.json',
key: 'import-product/test.json'
};
const extension = '.json';
const fileName = dataMock.name.replace(/\.[^/.]+$/, '');
const destinationMock = `${dataMock.path}/${fileName}${extension}`;
// Mock the bucket and file objects
const mockObjFile = {
createWriteStream: jest.fn().mockReturnValue(new PassThrough()),
publicUrl: jest.fn().mockResolvedValue(mockExpectedValue.url),
cloudStorageURI: {
href: `gs://${rootBucketName}/${dataMock.path}/${dataMock.name}`
}
};
cloudStorageService.bucket_.file = jest.fn().mockReturnValue(mockObjFile);

const result = await cloudStorageService.uploadStream(dataMock, Buffer.from('test'));

//Assertions
expect(result).toEqual({
...mockExpectedValue,
url: `gs://${rootBucketName}/${dataMock.path}/${dataMock.name}`
});
expect(cloudStorageService.bucket_.file).toHaveBeenCalledWith(destinationMock);
expect(cloudStorageService.bucket_.file().createWriteStream).toHaveBeenCalled();
expect(cloudStorageService.bucket_.file().publicUrl).not.toHaveBeenCalled();
});

test('should throw an error when upload with stream data empty extension (public)', async () => {
const dataMock = {
path: 'import-product',
name: 'test'
} as UploadStreamDescriptorWithPathType;
const mockExpectedValue = {
url: 'https://test.com/mock-bucket/uuid/test.json',
key: 'import-product/test.json'
};
const extension = '.json';
const fileName = dataMock.name.replace(/\.[^/.]+$/, '');
const destinationMock = `${dataMock.path}/${fileName}${extension}`;
// Mock the bucket and file objects
const mockObjFile = {
createWriteStream: jest.fn().mockReturnValue(new PassThrough()),
publicUrl: jest.fn().mockResolvedValue(mockExpectedValue.url),
cloudStorageURI: {
href: `gs://${rootBucketName}/${dataMock.path}/${dataMock.name}`
}
};
cloudStorageService.bucket_.file = jest.fn().mockReturnValue(mockObjFile);

//Assertions
await expect(cloudStorageService.uploadStream(dataMock, Buffer.from('test'))).rejects.toThrow(
'File name must have extension.'
);
expect(cloudStorageService.bucket_.file).not.toHaveBeenCalled();
expect(cloudStorageService.bucket_.file().createWriteStream).not.toHaveBeenCalled();
expect(cloudStorageService.bucket_.file().publicUrl).not.toHaveBeenCalled();
});
});
103 changes: 93 additions & 10 deletions src/services/cloud-storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ import {
import { Storage, Bucket, GetSignedUrlConfig } from '@google-cloud/storage';
import { v4 as uuidv4 } from 'uuid';
import fs from 'fs';
import stream from 'stream';
import stream, { Readable, PassThrough } from 'stream';
import { UploadStreamDescriptorWithPathType, UploadedJsonType } from '../types/upload-binary';
class CloudStorageService extends AbstractFileService implements IFileService {
protected logger_: Logger;
protected storage_: Storage;
Expand Down Expand Up @@ -105,9 +106,7 @@ class CloudStorageService extends AbstractFileService implements IFileService {
return;
} else {
//not found file
throw new MedusaError(
MedusaError.Types.NOT_FOUND, 'Not found file.'
);
throw new MedusaError(MedusaError.Types.NOT_FOUND, 'Not found file.');
}
} catch (error) {
throw new MedusaError(
Expand Down Expand Up @@ -169,9 +168,7 @@ class CloudStorageService extends AbstractFileService implements IFileService {
const [isExist] = await file.exists();
if (!isExist) {
//Not found file
throw new MedusaError(
MedusaError.Types.NOT_FOUND, 'Not found file.'
);
throw new MedusaError(MedusaError.Types.NOT_FOUND, 'Not found file.');
}
return file.createReadStream();
} catch (error) {
Expand All @@ -189,9 +186,7 @@ class CloudStorageService extends AbstractFileService implements IFileService {
const [isExist] = await file.exists();
if (!isExist) {
//Not found file
throw new MedusaError(
MedusaError.Types.NOT_FOUND, 'Not found file.'
);
throw new MedusaError(MedusaError.Types.NOT_FOUND, 'Not found file.');
}
//config for generate url
const options: GetSignedUrlConfig = {
Expand All @@ -208,6 +203,94 @@ class CloudStorageService extends AbstractFileService implements IFileService {
);
}
}

async uploadStreamJson(fileData: UploadedJsonType): Promise<FileServiceUploadResult> {
const jsonString = JSON.stringify(fileData.data);
const buffer = Buffer.from(jsonString);
const stream = Readable.from(buffer);
//force extension to .json
const extension = '.json';
//force file name to be the same as the original name
const fileName = fileData.name.replace(/\.[^/.]+$/, '');
const destination = `${fileData.path}/${fileName}${extension}`;
const file = this.bucket_.file(destination);
const isPrivate = fileData?.isPrivate;
const options = {
metadata: {
predefinedAcl: isPrivate ? 'private' : 'publicRead'
},
private: isPrivate,
public: !isPrivate
};
//make file streaming
const pipe = stream.pipe(file.createWriteStream(options));
const pass = new PassThrough();
stream.pipe(pass);
const promise = new Promise((res, rej) => {
pipe.on('finish', res);
pipe.on('error', rej);
});
await promise;
//Get url of file
let url: string;
if (isPrivate) {
url = file.cloudStorageURI.href;
} else {
url = await file.publicUrl();
}
return {
url,
key: destination
};
}

async uploadStream(
fileDetail: UploadStreamDescriptorWithPathType,
arrayBuffer: WithImplicitCoercion<ArrayBuffer | SharedArrayBuffer>
): Promise<FileServiceUploadResult> {
const buffer = Buffer.from(arrayBuffer);
const stream = Readable.from(buffer);
let fileName = fileDetail.name.replace(/\.[^/.]+$/, '');
fileName = fileDetail.ext ? `${fileName}.${fileDetail.ext}` : fileName;
//check file name don't have extension
if (!fileDetail.ext && fileName.indexOf('.') === -1) {
throw new MedusaError(
MedusaError.Types.INVALID_DATA,
'File name must have extension.'
);
}
const destination = `${fileDetail.path}/${fileName}`;
//init file into the bucket *fileData.name include sub-bucket
const file = this.bucket_.file(destination);
const isPrivate = fileDetail?.isPrivate;
const options = {
metadata: {
predefinedAcl: isPrivate ? 'private' : 'publicRead'
},
private: isPrivate,
public: !isPrivate
};
//make file streaming
const pipe = stream.pipe(file.createWriteStream(options));
const pass = new PassThrough();
stream.pipe(pass);
const promise = new Promise((res, rej) => {
pipe.on('finish', res);
pipe.on('error', rej);
});
await promise;
//Get url of file
let url: string;
if (isPrivate) {
url = file.cloudStorageURI.href;
} else {
url = await file.publicUrl();
}
return {
url,
key: destination
};
}
}

export default CloudStorageService;
15 changes: 15 additions & 0 deletions src/types/upload-binary.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
export type UploadStreamDescriptorWithPathType = {
name: string;
ext?: string;
isPrivate?: boolean;
path: string;
};

export type UploadedJsonType = {
data: {
[x: string]: unknown;
};
path: string;
name: string;
isPrivate?: boolean;
};
Loading

0 comments on commit f43c3ef

Please sign in to comment.