Skip to content

Commit

Permalink
feat(SPDX): Use new SPDX library (eclipse-sw360#1496)
Browse files Browse the repository at this point in the history 
Signed-off-by: tuan99123 <tuan2.nguyennhu@toshiba.co.jp>
  • Loading branch information
@tuannn2
tuannn2 committed Jul 20, 2022
1 parent 7f9528d commit 64c43dc
Show file tree
Hide file tree
Showing 33 changed files with 789 additions and 169 deletions.
13 changes: 13 additions & 0 deletions backend/src-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,24 @@
</plugins>
</build>
<dependencies>
<!-- https://mvnrepository.com/artifact/com.googlecode.json-simple/json-simple -->
<dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<version>1.1.1</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.spdx</groupId>
<artifactId>spdx-tools</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.spdx</groupId>
<artifactId>tools-java</artifactId>
<version>1.0.2</version>
<scope>compile</scope>
</dependency>
<!-- Needed by spdx-tools -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
Expand Down
1 change: 0 additions & 1 deletion .../src-common/src/main/java/org/eclipse/sw360/datahandler/db/ChangeLogsDatabaseHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ public List<ChangeLogs> getChangeLogsByDocumentId(User user, String docId) {
changeLogsByDocId = changeLogsByDocId.stream().filter(Objects::nonNull).filter(changeLog -> isNotEmptyChangeLog(changeLog))
.collect(Collectors.toList());
Collections.sort(changeLogsByDocId, Comparator.comparing(ChangeLogs::getChangeTimestamp).reversed());
changeLogsByDocId.stream().forEach(cl -> cl.setChangeTimestamp(cl.getChangeTimestamp().split(" ")[0]));
return changeLogsByDocId;
}

Expand Down
124 changes: 119 additions & 5 deletions ...d/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,21 @@
import org.eclipse.sw360.spdx.SpdxBOMImporterSink;
import org.jetbrains.annotations.NotNull;
import org.spdx.rdfparser.InvalidSPDXAnalysisException;

import org.spdx.tools.SpdxConverter;
import org.spdx.tools.SpdxConverterException;
import org.spdx.tools.TagToRDF;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
Expand Down Expand Up @@ -160,6 +171,7 @@ public ComponentDatabaseHandler(Supplier<CloudantClient> httpClient, String dbNa
attachmentConnector = new AttachmentConnector(httpClient, attachmentDbName, durationOf(30, TimeUnit.SECONDS));
DatabaseConnectorCloudant dbChangeLogs = new DatabaseConnectorCloudant(httpClient, DatabaseSettings.COUCH_DB_CHANGE_LOGS);
this.dbHandlerUtil = new DatabaseHandlerUtil(dbChangeLogs);

}

public ComponentDatabaseHandler(Supplier<CloudantClient> httpClient, String dbName, String changeLogsDbName, String attachmentDbName, ComponentModerator moderator, ReleaseModerator releaseModerator, ProjectModerator projectModerator) throws MalformedURLException {
Expand Down Expand Up @@ -513,12 +525,26 @@ public AddDocumentRequestSummary addRelease(Release release, User user) throws S
}

private boolean isDuplicate(Component component, boolean caseInsenstive){
Set<String> duplicates = componentRepository.getComponentIdsByName(component.getName(), caseInsenstive);
return duplicates.size()>0;
return isDuplicate(component.getName(), caseInsenstive);
}

private boolean isDuplicate(Release release){
List<Release> duplicates = releaseRepository.searchByNameAndVersion(release.getName(), release.getVersion());
return isDuplicate(release.getName(), release.getVersion());
}

private boolean isDuplicate(String componentName, boolean caseInsenstive) {
if (isNullEmptyOrWhitespace(componentName)) {
return false;
}
Set<String> duplicates = componentRepository.getComponentIdsByName(componentName, caseInsenstive);
return duplicates.size()>0;
}

private boolean isDuplicate(String releaseName, String releaseVersion) {
if (isNullEmptyOrWhitespace(releaseName)) {
return false;
}
List<Release> duplicates = releaseRepository.searchByNameAndVersion(releaseName, releaseVersion);
return duplicates.size()>0;
}

Expand Down Expand Up @@ -1680,6 +1706,7 @@ public RequestStatus deleteRelease(String id, User user) throws SW360Exception {
// Remove release id from component
removeReleaseId(id, release.componentId);
Component componentAfter=removeReleaseAndCleanUp(release, user);

dbHandlerUtil.addChangeLogs(null, release, user.getEmail(), Operation.DELETE, attachmentConnector,
Lists.newArrayList(), null, null);
dbHandlerUtil.addChangeLogs(componentAfter, componentBefore, user.getEmail(), Operation.UPDATE,
Expand Down Expand Up @@ -2363,6 +2390,75 @@ private void sendMailNotificationsForReleaseUpdate(Release release, String user)
release.getName(), release.getVersion());
}

public ImportBomRequestPreparation prepareImportBom(User user, String attachmentContentId) throws SW360Exception {
final AttachmentContent attachmentContent = attachmentConnector.getAttachmentContent(attachmentContentId);
final Duration timeout = Duration.durationOf(30, TimeUnit.SECONDS);
String sourceFilePath = null;
String targetFilePath = null;
try {
final AttachmentStreamConnector attachmentStreamConnector = new AttachmentStreamConnector(timeout);
try (final InputStream inputStream = attachmentStreamConnector.unsafeGetAttachmentStream(attachmentContent)) {
final SpdxBOMImporterSink spdxBOMImporterSink = new SpdxBOMImporterSink(user, null, this);
final SpdxBOMImporter spdxBOMImporter = new SpdxBOMImporter(spdxBOMImporterSink);

InputStream spdxInputStream = null;
String fileType = getFileType(attachmentContent.getFilename());

final String ext = "." + fileType;
final File sourceFile = DatabaseHandlerUtil.saveAsTempFile(user, inputStream, attachmentContentId, ext);
sourceFilePath = sourceFile.getAbsolutePath();
cutFileInformation(sourceFilePath);
File targetFile = new File (sourceFilePath);
spdxInputStream = new FileInputStream(targetFile);
targetFilePath = sourceFilePath;

ImportBomRequestPreparation importBomRequestPreparation = spdxBOMImporter.prepareImportSpdxBOMAsRelease(spdxInputStream, attachmentContent);
if (RequestStatus.SUCCESS.equals(importBomRequestPreparation.getRequestStatus())) {
String name = importBomRequestPreparation.getName();
String version = importBomRequestPreparation.getVersion();
if (!isDuplicate(name, true)) {
importBomRequestPreparation.setIsComponentDuplicate(false);
importBomRequestPreparation.setIsReleaseDuplicate(false);
} else if (!isDuplicate(name, version)) {
importBomRequestPreparation.setIsComponentDuplicate(true);
importBomRequestPreparation.setIsReleaseDuplicate(false);
} else {
importBomRequestPreparation.setIsComponentDuplicate(true);
importBomRequestPreparation.setIsReleaseDuplicate(true);
}
importBomRequestPreparation.setMessage(targetFilePath);
}
return importBomRequestPreparation;
}
} catch (InvalidSPDXAnalysisException | IOException e) {
throw new SW360Exception(e.getMessage());
}
}

private void cutFileInformation(String pathFile) {
try {
log.info("Run command cut File information from RDF file from line");
String command = "file=\"" + pathFile + "\" " +
"&& start=$(cat $file | grep -nF \"spdx:hasFile>\" | head -1 | cut -d \":\" -f1) " +
"&& end=$(cat $file | grep -nF \"/spdx:hasFile>\" | tail -1 | cut -d \":\" -f1) " +
"&& echo $start to $end " +
"&& sed -i \"${start},${end}d\" $file ";
Process process = Runtime.getRuntime().exec(new String[] { "/bin/bash", "-c", command });
printResults(process);
} catch (IOException e) {
log.error("Error when cut File information");
e.printStackTrace();
}
}

public static void printResults(Process process) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line = "";
while ((line = reader.readLine()) != null) {
log.info(line);
}
}

public RequestSummary importBomFromAttachmentContent(User user, String attachmentContentId) throws SW360Exception {
final AttachmentContent attachmentContent = attachmentConnector.getAttachmentContent(attachmentContentId);
final Duration timeout = Duration.durationOf(30, TimeUnit.SECONDS);
Expand All @@ -2373,11 +2469,29 @@ public RequestSummary importBomFromAttachmentContent(User user, String attachmen
final SpdxBOMImporter spdxBOMImporter = new SpdxBOMImporter(spdxBOMImporterSink);
return spdxBOMImporter.importSpdxBOMAsRelease(inputStream, attachmentContent);
}
} catch (InvalidSPDXAnalysisException | IOException e) {
} catch (IOException e) {
throw new SW360Exception(e.getMessage());
}
}

private String getFileType(String fileName) {
if (isNullEmptyOrWhitespace(fileName) || !fileName.contains(".")) {
log.error("Can not get file type from file name - no file extension");
return null;
}
String ext = fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase();
if ("xml".equals(ext)) {
if (fileName.endsWith("rdf.xml")) {
ext = "rdf";
}
}
return ext;
}

private boolean isJSONFile(String fileType) {
return (!isNullEmptyOrWhitespace(fileType) && fileType.equals("json"));
}

private void removeLeadingTrailingWhitespace(Release release) {
DatabaseHandlerUtil.trimStringFields(release, listOfStringFieldsInReleaseToTrim);

Expand Down
18 changes: 15 additions & 3 deletions backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/DatabaseHandlerUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
package org.eclipse.sw360.datahandler.db;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
Expand Down Expand Up @@ -44,6 +45,7 @@
import java.util.stream.Collectors;

import org.apache.logging.log4j.Level;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
Expand Down Expand Up @@ -75,6 +77,7 @@
import org.eclipse.sw360.datahandler.couchdb.DatabaseMixInForChangeLog.RepositoryMixin;
import org.eclipse.sw360.datahandler.couchdb.DatabaseMixInForChangeLog.VendorMixin;
import org.eclipse.sw360.datahandler.couchdb.DatabaseMixInForChangeLog.ObligationMixin;
import org.eclipse.sw360.datahandler.couchdb.DatabaseMixInForChangeLog.*;
import org.eclipse.sw360.datahandler.thrift.ProjectReleaseRelationship;
import org.eclipse.sw360.datahandler.thrift.RequestStatus;
import org.eclipse.sw360.datahandler.thrift.SW360Exception;
Expand Down Expand Up @@ -124,6 +127,7 @@ public class DatabaseHandlerUtil {
private static final boolean IS_STORE_ATTACHMENT_TO_FILE_SYSTEM_ENABLED;
private static final String ATTACHMENT_STORE_FILE_SYSTEM_LOCATION;
private static final String ATTACHMENT_STORE_FILE_SYSTEM_PERMISSION;
// private static final String TEMPLE_FILE_LOCATION;
private static ExecutorService ATTACHMENT_FILE_SYSTEM_STORE_THREAD_POOL = Executors.newFixedThreadPool(5);
private static final String ATTACHMENT_DELETE_NO_OF_DAYS;
private static final boolean IS_SW360CHANGELOG_ENABLED;
Expand All @@ -139,6 +143,7 @@ public class DatabaseHandlerUtil {
"/opt/sw360tempattachments");
ATTACHMENT_STORE_FILE_SYSTEM_PERMISSION = props.getProperty("attachment.store.file.system.permission",
"rwx------");
// TEMPLE_FILE_LOCATION = props.getProperty("temp.dir", "../temp")
IS_STORE_ATTACHMENT_TO_FILE_SYSTEM_ENABLED = Boolean.parseBoolean(props.getProperty("enable.attachment.store.to.file.system", "false"));
ATTACHMENT_DELETE_NO_OF_DAYS = props.getProperty("attachemnt.delete.no.of.days",
"30");
Expand Down Expand Up @@ -411,8 +416,6 @@ public static <T, R> void trimStringFields(T obj, List<R> listOfStrFields) {
changeLog.setDbName(DatabaseSettings.COUCH_DB_DATABASE);
}

log.info("Initialize ChangeLogs for Document Id : " + changeLog.getDocumentId());

if (parentOperation != null)
info.put("PARENT_OPERATION", parentOperation.name());
if (!info.isEmpty())
Expand Down Expand Up @@ -690,7 +693,7 @@ private static boolean isTwoCollectionSame(Collection<?> col1, Collection<?> col
}

private static String getTimeStamp() {
SimpleDateFormat timestampPattern = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat timestampPattern = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
Date timeNow = new Date(System.currentTimeMillis());
return timestampPattern.format(timeNow);
}
Expand Down Expand Up @@ -979,5 +982,14 @@ private static void configureLog4J(String outputpath, String liferayhome) {
.add( builder.newAppenderRef("ChangeLogFile")));
Configurator.reconfigure(builder.build());
}
public static File saveAsTempFile(User user, InputStream inputStream, String prefix, String suffix) throws IOException {
final File tempFile = File.createTempFile(prefix, suffix);
tempFile.deleteOnExit();
// Set append to false, overwrite if file existed
try (FileOutputStream outputStream = new FileOutputStream(tempFile, false)) {
IOUtils.copy(inputStream, outputStream);
}
return tempFile;
}
}

Loading

0 comments on commit 64c43dc

Please sign in to comment.