Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Improved memory usage #21

Merged
merged 12 commits into from
Jun 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public void processGenerateReceipt(
boolean generateOnlyDebtor = payerCF == null || payerCF.equals(debtorCF);

//Generate and save PDF
PdfGeneration pdfGeneration = service.handlePdfsGeneration(generateOnlyDebtor, receipt, bizEvent, debtorCF, payerCF);
PdfGeneration pdfGeneration = service.handlePdfsGeneration(generateOnlyDebtor, receipt, bizEvent, debtorCF, payerCF, logger);

//Write PDF blob storage metadata on receipt
numberOfSavedPdfs = service.addPdfsMetadataToReceipt(receipt, pdfGeneration);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

import it.gov.pagopa.receipt.pdf.datastore.model.response.BlobStorageResponse;

import java.io.InputStream;

public interface ReceiptBlobClient {

BlobStorageResponse savePdfToBlobStorage(byte[] pdf, String fileName);
BlobStorageResponse savePdfToBlobStorage(InputStream pdf, String fileName);
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,24 @@
import it.gov.pagopa.receipt.pdf.datastore.model.request.PdfEngineRequest;
import it.gov.pagopa.receipt.pdf.datastore.model.response.PdfEngineResponse;
import it.gov.pagopa.receipt.pdf.datastore.utils.ObjectMapperUtils;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.ByteArrayBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;

/**
* Client for the PDF Engine
Expand Down Expand Up @@ -67,13 +69,13 @@ public PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest) {
//Generate client
try (CloseableHttpClient client = this.httpClientBuilder.build()) {
//Encode template and data
ByteArrayBody fileBody = new ByteArrayBody(pdfEngineRequest.getTemplate(), ZIP_FILE_NAME);

StringBody dataBody = new StringBody(pdfEngineRequest.getData(), ContentType.APPLICATION_JSON);

//Build the multipart request
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
builder.addPart(TEMPLATE_KEY, fileBody);
builder.addBinaryBody(TEMPLATE_KEY, pdfEngineRequest.getTemplate(), ContentType.create("application/zip"), ZIP_FILE_NAME);
builder.addPart(DATA_KEY, dataBody);
HttpEntity entity = builder.build();

Expand All @@ -82,9 +84,9 @@ public PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest) {
request.setHeader(HEADER_AUTH_KEY, ocpAimSubKey);
request.setEntity(entity);

handlePdfEngineResponse(pdfEngineResponse, client, request);
pdfEngineResponse = handlePdfEngineResponse(client, request);
} catch (IOException e) {
pdfEngineResponse.setStatusCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
handleExceptionErrorMessage(pdfEngineResponse, e);
}

return pdfEngineResponse;
Expand All @@ -93,31 +95,66 @@ public PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest) {
/**
* Calls the PDF Engine and handles its response, updating the PdfEngineResponse accordingly
*
* @param pdfEngineResponse Response output
* @param client The previously generated client
* @param request The request to the PDF engine
* @param client The previously generated client
* @param request The request to the PDF engine
* @return pdf engine response
*/
private static void handlePdfEngineResponse(PdfEngineResponse pdfEngineResponse, CloseableHttpClient client, HttpPost request) {
private static PdfEngineResponse handlePdfEngineResponse(CloseableHttpClient client, HttpPost request) {
PdfEngineResponse pdfEngineResponse = new PdfEngineResponse();
//Execute call
try (CloseableHttpResponse response = client.execute(request)) {
//Retrieve response
HttpEntity entityResponse = response.getEntity();

//Handles response
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && entityResponse != null) {
InputStream inputStream = entityResponse.getContent();
try (InputStream inputStream = entityResponse.getContent()) {
pdfEngineResponse.setStatusCode(HttpStatus.SC_OK);

pdfEngineResponse.setStatusCode(HttpStatus.SC_OK);
pdfEngineResponse.setPdf(inputStream.readAllBytes());
saveTempPdf(pdfEngineResponse, inputStream);
}
} else {
pdfEngineResponse.setStatusCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);

handleErrorResponse(pdfEngineResponse, response, entityResponse);
}
} catch (Exception e) {
handleExceptionErrorMessage(pdfEngineResponse, e);
}

} catch (IOException e) {
pdfEngineResponse.setStatusCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
return pdfEngineResponse;
}

/**
* Saves pdf as temporary file
*
* @param pdfEngineResponse Pdf engine response
* @param inputStream InputStream pdf
* @throws IOException In case of error to save
*/
private static void saveTempPdf(PdfEngineResponse pdfEngineResponse, InputStream inputStream) throws IOException {
File tempDirectory = new File("temp");
if (!tempDirectory.exists()) {
Files.createDirectory(tempDirectory.toPath());
}

File targetFile = File.createTempFile("tempFile", ".pdf", tempDirectory);

FileUtils.copyInputStreamToFile(inputStream, targetFile);

pdfEngineResponse.setTempPdfPath(targetFile.getAbsolutePath());
pdfEngineResponse.setTempDirectoryPath(tempDirectory.getAbsolutePath());
}

/**
* Handles error message in case of error thrown
*
* @param pdfEngineResponse Pdf engine respone
* @param e Error thrown
*/
private static void handleExceptionErrorMessage(PdfEngineResponse pdfEngineResponse, Exception e) {
pdfEngineResponse.setStatusCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
pdfEngineResponse.setErrorMessage(String.format("Exception thrown during pdf generation process: %s", e));
}

/**
Expand All @@ -128,9 +165,16 @@ private static void handlePdfEngineResponse(PdfEngineResponse pdfEngineResponse,
* @param entityResponse Response content from the PDF Engine
* @throws IOException in case of error encoding to string
*/
private static void handleErrorResponse(PdfEngineResponse pdfEngineResponse, CloseableHttpResponse response, HttpEntity entityResponse) throws IOException {
private static void handleErrorResponse(
PdfEngineResponse pdfEngineResponse,
CloseableHttpResponse response,
HttpEntity entityResponse
) throws IOException {
//Verify if unauthorized
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED) {
if (response != null &&
response.getStatusLine() != null &&
response.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED
) {
pdfEngineResponse.setErrorMessage("Unauthorized call to PDF engine function");

} else if (entityResponse != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import it.gov.pagopa.receipt.pdf.datastore.client.ReceiptBlobClient;
import it.gov.pagopa.receipt.pdf.datastore.model.response.BlobStorageResponse;

import java.io.ByteArrayInputStream;
import java.io.InputStream;

/**
* Client for the Blob Storage
Expand Down Expand Up @@ -55,7 +55,7 @@ public static ReceiptBlobClientImpl getInstance() {
* @param fileName Filename to save the PDF with
* @return blob storage response with PDF metadata or error message and status
*/
public BlobStorageResponse savePdfToBlobStorage(byte[] pdf, String fileName) {
public BlobStorageResponse savePdfToBlobStorage(InputStream pdf, String fileName) {

//Create the container and return a container client object
BlobContainerClient blobContainerClient = this.blobServiceClient.getBlobContainerClient(containerName);
Expand All @@ -67,7 +67,7 @@ public BlobStorageResponse savePdfToBlobStorage(byte[] pdf, String fileName) {
//Upload the blob
Response<BlockBlobItem> blockBlobItemResponse = blobClient.uploadWithResponse(
new BlobParallelUploadOptions(
new ByteArrayInputStream(pdf)
pdf
), null, null);

BlobStorageResponse blobStorageResponse = new BlobStorageResponse();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import lombok.NoArgsConstructor;
import lombok.Setter;

import java.io.InputStream;

/**
* Model class for PDF engine request
*/
Expand All @@ -12,7 +14,7 @@
@NoArgsConstructor
public class PdfEngineRequest {

byte[] template;
InputStream template;
String data;
boolean applySignature;
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
@NoArgsConstructor
public class PdfEngineResponse {

byte[] pdf;
String tempDirectoryPath;
String tempPdfPath;
int statusCode;
String errorMessage;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import lombok.NoArgsConstructor;
import org.apache.http.HttpStatus;

import java.io.*;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.util.logging.Logger;

Expand All @@ -38,7 +40,7 @@ public class GenerateReceiptPdfService {
* @param payerCF Payer fiscal code
* @return PdfGeneration object with the PDF metadata from the Blob Storage or relatives error messages
*/
public PdfGeneration handlePdfsGeneration(boolean generateOnlyDebtor, Receipt receipt, BizEvent bizEvent, String debtorCF, String payerCF) {
public PdfGeneration handlePdfsGeneration(boolean generateOnlyDebtor, Receipt receipt, BizEvent bizEvent, String debtorCF, String payerCF, Logger logger) {
PdfGeneration pdfGeneration = new PdfGeneration();

if (generateOnlyDebtor) {
Expand All @@ -48,7 +50,7 @@ public PdfGeneration handlePdfsGeneration(boolean generateOnlyDebtor, Receipt re
receipt.getMdAttach().getUrl() == null ||
receipt.getMdAttach().getUrl().isEmpty())
) {
pdfGeneration.setDebtorMetadata(generatePdf(bizEvent, debtorCF, true));
pdfGeneration.setDebtorMetadata(generatePdf(bizEvent, debtorCF, true, logger));
}
} else {
//Generate debtor's partial PDF
Expand All @@ -57,15 +59,15 @@ public PdfGeneration handlePdfsGeneration(boolean generateOnlyDebtor, Receipt re
receipt.getMdAttach().getUrl() == null ||
receipt.getMdAttach().getUrl().isEmpty())
) {
pdfGeneration.setDebtorMetadata(generatePdf(bizEvent, debtorCF, false));
pdfGeneration.setDebtorMetadata(generatePdf(bizEvent, debtorCF, false, logger));
}
//Generate payer's complete PDF
if (payerCF != null &&
(receipt.getMdAttachPayer() == null ||
receipt.getMdAttachPayer().getUrl() == null ||
receipt.getMdAttachPayer().getUrl().isEmpty())
) {
pdfGeneration.setPayerMetadata(generatePdf(bizEvent, payerCF, true));
pdfGeneration.setPayerMetadata(generatePdf(bizEvent, payerCF, true, logger));
}
}

Expand All @@ -80,7 +82,7 @@ public PdfGeneration handlePdfsGeneration(boolean generateOnlyDebtor, Receipt re
* @param completeTemplate Boolean that indicates what template to use
* @return PDF metadata retrieved from Blob Storage or relative error message
*/
private PdfMetadata generatePdf(BizEvent bizEvent, String fiscalCode, boolean completeTemplate) {
private PdfMetadata generatePdf(BizEvent bizEvent, String fiscalCode, boolean completeTemplate, Logger logger) {
PdfEngineRequest request = new PdfEngineRequest();
PdfMetadata response = new PdfMetadata();

Expand All @@ -90,12 +92,9 @@ private PdfMetadata generatePdf(BizEvent bizEvent, String fiscalCode, boolean co

String fileName = completeTemplate ? completeTemplateFileName : partialTemplateFileName;

try {
//File to byte[]
byte[] htmlTemplate = GenerateReceiptPdf.class.getClassLoader().getResourceAsStream(fileName).readAllBytes();

try (InputStream templateStream = GenerateReceiptPdf.class.getClassLoader().getResourceAsStream(fileName)) {
//Build the request
request.setTemplate(htmlTemplate);
request.setTemplate(templateStream);
request.setData(ObjectMapperUtils.writeValueAsString(TemplateMapperUtils.convertReceiptToPdfData(bizEvent)));

request.setApplySignature(false);
Expand All @@ -109,7 +108,7 @@ private PdfMetadata generatePdf(BizEvent bizEvent, String fiscalCode, boolean co
//Save the PDF
String pdfFileName = bizEvent.getId() + fiscalCode;

handleSaveToBlobStorage(response, pdfEngineResponse, pdfFileName);
handleSaveToBlobStorage(pdfEngineResponse, response, pdfFileName, logger);

} else {
//Handle PDF generation error
Expand All @@ -129,18 +128,20 @@ private PdfMetadata generatePdf(BizEvent bizEvent, String fiscalCode, boolean co
/**
* Handles saving PDF to Blob Storage
*
* @param response Pdf metadata containing response
* @param pdfEngineResponse Response from the pdf engine
* @param pdfFileName Filename composed of biz-event id and user fiscal code
* @param response Pdf metadata containing response
* @param pdfFileName Filename composed of biz-event id and user fiscal code
*/
private void handleSaveToBlobStorage(PdfMetadata response, PdfEngineResponse pdfEngineResponse, String pdfFileName) {
private void handleSaveToBlobStorage(PdfEngineResponse pdfEngineResponse, PdfMetadata response, String pdfFileName, Logger logger) {
BlobStorageResponse blobStorageResponse;

ReceiptBlobClientImpl blobClient = ReceiptBlobClientImpl.getInstance();

String tempPdfPath = pdfEngineResponse.getTempPdfPath();
String tempDirectoryPath = pdfEngineResponse.getTempDirectoryPath();

//Save to Blob Storage
try {
blobStorageResponse = blobClient.savePdfToBlobStorage(pdfEngineResponse.getPdf(), pdfFileName);
try (BufferedInputStream pdfStream = new BufferedInputStream(new FileInputStream(tempPdfPath))) {
blobStorageResponse = blobClient.savePdfToBlobStorage(pdfStream, pdfFileName);

if (blobStorageResponse.getStatusCode() == com.microsoft.azure.functions.HttpStatus.CREATED.value()) {
//Update PDF metadata
Expand All @@ -160,6 +161,39 @@ private void handleSaveToBlobStorage(PdfMetadata response, PdfEngineResponse pdf
response.setStatusCode(ReasonErrorCode.ERROR_BLOB_STORAGE.getCode());
response.setErrorMessage("Error saving pdf to blob storage : " + e);
}

deleteTempFolderAndFile(tempPdfPath, tempDirectoryPath, logger);
}

/**
* Delete temporary file and director created for the generated PDF
*
* @param tempPdfPath Path to the temp PDF file
* @param tempDirectoryPath Path to the temp directory containing the PDF file
* @param logger Logger
*/
private static void deleteTempFolderAndFile(String tempPdfPath, String tempDirectoryPath, Logger logger) {
String logMsg;

File tempFile = new File(tempPdfPath);
if (tempFile.exists()) {
try {
Files.delete(tempFile.toPath());
} catch (IOException e) {
logMsg = String.format("Error deleting temporary pdf file from file system: %s", e);
logger.warning(logMsg);
}
}

File tempDirectory = new File(tempDirectoryPath);
if (tempDirectory.exists()) {
try {
Files.delete(tempDirectory.toPath());
} catch (IOException e) {
logMsg = String.format("Error deleting temporary pdf directory from file system: %s", e);
logger.warning(logMsg);
}
}
}


Expand Down
Loading