Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Log warning when dependency graph is missing the root node #3990

Merged
merged 1 commit into from
Jul 20, 2024
Merged

Log warning when dependency graph is missing the root node #3990

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 56 additions & 53 deletions src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@
import static org.apache.commons.lang3.time.DurationFormatUtils.formatDurationHMS;
import static org.datanucleus.PropertyNames.PROPERTY_FLUSH_MODE;
import static org.datanucleus.PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT;
import static org.datanucleus.PropertyNames.PROPERTY_RETAIN_VALUES;
import static org.dependencytrack.common.MdcKeys.MDC_BOM_FORMAT;
import static org.dependencytrack.common.MdcKeys.MDC_BOM_SERIAL_NUMBER;
import static org.dependencytrack.common.MdcKeys.MDC_BOM_SPEC_VERSION;
Expand Down Expand Up @@ -259,14 +258,6 @@ private void processBom(final Context ctx, final org.cyclonedx.model.Bom cdxBom)
// BomUploadProcessingTaskTest#informWithBloatedBomTest can be used to profile the impact on large BOMs.
qm.getPersistenceManager().setProperty(PROPERTY_FLUSH_MODE, FlushMode.MANUAL.name());

// Prevent object fields from being unloaded upon commit.
//
// DataNucleus transitions objects into the "hollow" state after the transaction is committed.
// In hollow state, all fields except the ID are unloaded. Accessing fields afterward triggers
// one or more database queries to load them again.
// See https://www.datanucleus.org/products/accessplatform_6_0/jdo/persistence.html#lifecycle
qm.getPersistenceManager().setProperty(PROPERTY_RETAIN_VALUES, "true");

qm.getPersistenceManager().addInstanceLifecycleListener(new IndexingInstanceLifecycleListener(eventsToDispatch::add),
Component.class, Project.class, ProjectMetadata.class, ServiceComponent.class);
qm.getPersistenceManager().addInstanceLifecycleListener(new L2CacheEvictingInstanceLifecycleListener(qm),
Expand Down Expand Up @@ -310,8 +301,12 @@ private void processBom(final Context ctx, final org.cyclonedx.model.Bom cdxBom)
dispatchBomProcessedNotification(ctx);
}

private Project processProject(final Context ctx, final QueryManager qm,
final Project project, final ProjectMetadata projectMetadata) {
private Project processProject(
final Context ctx,
final QueryManager qm,
final Project project,
final ProjectMetadata projectMetadata
) {
final Query<Project> query = qm.getPersistenceManager().newQuery(Project.class);
query.setFilter("uuid == :uuid");
query.setParameters(ctx.project.getUuid());
Expand Down Expand Up @@ -347,7 +342,7 @@ private Project processProject(final Context ctx, final QueryManager qm,
if (projectMetadata != null) {
if (persistentProject.getMetadata() == null) {
projectMetadata.setProject(persistentProject);
qm.getPersistenceManager().makePersistent(projectMetadata);
qm.persist(projectMetadata);
hasChanged = true;
} else {
hasChanged |= applyIfChanged(persistentProject.getMetadata(), projectMetadata, ProjectMetadata::getAuthors, persistentProject.getMetadata()::setAuthors);
Expand All @@ -362,11 +357,13 @@ private Project processProject(final Context ctx, final QueryManager qm,
return persistentProject;
}

private Map<ComponentIdentity, Component> processComponents(final QueryManager qm,
final Project project,
final List<Component> components,
final Map<String, ComponentIdentity> identitiesByBomRef,
final MultiValuedMap<ComponentIdentity, String> bomRefsByIdentity) {
private Map<ComponentIdentity, Component> processComponents(
final QueryManager qm,
final Project project,
final List<Component> components,
final Map<String, ComponentIdentity> identitiesByBomRef,
final MultiValuedMap<ComponentIdentity, String> bomRefsByIdentity
) {
assertPersistent(project, "Project mut be persistent");

// Fetch IDs of all components that exist in the project already.
Expand Down Expand Up @@ -405,7 +402,7 @@ private Map<ComponentIdentity, Component> processComponents(final QueryManager q
}
if (persistentComponent == null) {
component.setProject(project);
persistentComponent = qm.getPersistenceManager().makePersistent(component);
persistentComponent = qm.persist(component);
component.setNew(true); // Transient
} else {
persistentComponent.setBomRef(component.getBomRef()); // Transient
Expand Down Expand Up @@ -464,11 +461,13 @@ private Map<ComponentIdentity, Component> processComponents(final QueryManager q
return persistentComponents;
}

private Map<ComponentIdentity, ServiceComponent> processServices(final QueryManager qm,
final Project project,
final List<ServiceComponent> services,
final Map<String, ComponentIdentity> identitiesByBomRef,
final MultiValuedMap<ComponentIdentity, String> bomRefsByIdentity) {
private Map<ComponentIdentity, ServiceComponent> processServices(
final QueryManager qm,
final Project project,
final List<ServiceComponent> services,
final Map<String, ComponentIdentity> identitiesByBomRef,
final MultiValuedMap<ComponentIdentity, String> bomRefsByIdentity
) {
assertPersistent(project, "Project must be persistent");

final PersistenceManager pm = qm.getPersistenceManager();
Expand All @@ -484,7 +483,7 @@ private Map<ComponentIdentity, ServiceComponent> processServices(final QueryMana
ServiceComponent persistentService = qm.matchServiceIdentity(project, componentIdentity);
if (persistentService == null) {
service.setProject(project);
persistentService = pm.makePersistent(service);
persistentService = qm.persist(service);
} else {
persistentService.setBomRef(service.getBomRef()); // Transient
applyIfChanged(persistentService, service, ServiceComponent::getGroup, persistentService::setGroup);
Expand Down Expand Up @@ -534,22 +533,30 @@ private void recordBomImport(final Context ctx, final QueryManager qm, final Pro
bom.setSerialNumber(ctx.bomSerialNumber);
bom.setBomVersion(ctx.bomVersion);
bom.setImported(bomImportDate);
qm.getPersistenceManager().makePersistent(bom);
qm.persist(bom);

project.setLastBomImport(bomImportDate);
project.setLastBomImportFormat("%s %s".formatted(ctx.bomFormat.getFormatShortName(), ctx.bomSpecVersion));
qm.getPersistenceManager().flush();
}

private void processDependencyGraph(final QueryManager qm,
final Project project,
final MultiValuedMap<String, String> dependencyGraph,
final Map<ComponentIdentity, Component> componentsByIdentity,
final Map<String, ComponentIdentity> identitiesByBomRef) {
private void processDependencyGraph(
final QueryManager qm,
final Project project,
final MultiValuedMap<String, String> dependencyGraph,
final Map<ComponentIdentity, Component> componentsByIdentity,
final Map<String, ComponentIdentity> identitiesByBomRef
) {
assertPersistent(project, "Project must be persistent");

if (project.getBomRef() != null) {
final Collection<String> directDependencyBomRefs = dependencyGraph.get(project.getBomRef());
if (directDependencyBomRefs == null || directDependencyBomRefs.isEmpty()) {
LOGGER.warn("""
The dependency graph has %d entries, but the project (metadata.component node of the BOM) \
is not one of them; Graph will be incomplete because it is not possible to determine its root\
""".formatted(dependencyGraph.size()));
}
final String directDependenciesJson = resolveDirectDependenciesJson(project.getBomRef(), directDependencyBomRefs, identitiesByBomRef);
if (!Objects.equals(directDependenciesJson, project.getDirectDependencies())) {
project.setDirectDependencies(directDependenciesJson);
Expand Down Expand Up @@ -588,8 +595,10 @@ private void processDependencyGraph(final QueryManager qm,
qm.getPersistenceManager().flush();
}

private static Predicate<Component> distinctComponentsByIdentity(final Map<String, ComponentIdentity> identitiesByBomRef,
final MultiValuedMap<ComponentIdentity, String> bomRefsByIdentity) {
private static Predicate<Component> distinctComponentsByIdentity(
final Map<String, ComponentIdentity> identitiesByBomRef,
final MultiValuedMap<ComponentIdentity, String> bomRefsByIdentity
) {
final var identitiesSeen = new HashSet<ComponentIdentity>();
return component -> {
final var componentIdentity = new ComponentIdentity(component);
Expand All @@ -614,8 +623,10 @@ private static Predicate<Component> distinctComponentsByIdentity(final Map<Strin
};
}

private static Predicate<ServiceComponent> distinctServicesByIdentity(final Map<String, ComponentIdentity> identitiesByBomRef,
final MultiValuedMap<ComponentIdentity, String> bomRefsByIdentity) {
private static Predicate<ServiceComponent> distinctServicesByIdentity(
final Map<String, ComponentIdentity> identitiesByBomRef,
final MultiValuedMap<ComponentIdentity, String> bomRefsByIdentity
) {
final var identitiesSeen = new HashSet<ComponentIdentity>();
return service -> {
final var componentIdentity = new ComponentIdentity(service);
Expand All @@ -631,9 +642,11 @@ private static Predicate<ServiceComponent> distinctServicesByIdentity(final Map<
};
}

private String resolveDirectDependenciesJson(final String dependencyBomRef,
final Collection<String> directDependencyBomRefs,
final Map<String, ComponentIdentity> identitiesByBomRef) {
private String resolveDirectDependenciesJson(
final String dependencyBomRef,
final Collection<String> directDependencyBomRefs,
final Map<String, ComponentIdentity> identitiesByBomRef
) {
if (directDependencyBomRefs == null || directDependencyBomRefs.isEmpty()) {
return null;
}
Expand All @@ -654,10 +667,12 @@ private String resolveDirectDependenciesJson(final String dependencyBomRef,
return jsonDependencies.isEmpty() ? null : jsonDependencies.toString();
}

private static void resolveAndApplyLicense(final QueryManager qm,
final Component component,
final Map<String, License> licenseCache,
final Map<String, License> customLicenseCache) {
private static void resolveAndApplyLicense(
final QueryManager qm,
final Component component,
final Map<String, License> licenseCache,
final Map<String, License> customLicenseCache
) {
// CycloneDX components can declare multiple licenses, but we currently
// only support one. We assume that the licenseCandidates list is ordered
// by priority, and simply take the first resolvable candidate.
Expand Down Expand Up @@ -701,18 +716,6 @@ private static void resolveAndApplyLicense(final QueryManager qm,
}
}

private static License resolveCustomLicense(final QueryManager qm, final String licenseName) {
final Query<License> query = qm.getPersistenceManager().newQuery(License.class);
query.setFilter("name == :name && customLicense == true");
query.setParameters(licenseName);
try {
final License license = query.executeUnique();
return license != null ? license : License.UNRESOLVED;
} finally {
query.closeAll();
}
}

private static <T> Set<Long> getAllComponentIds(final QueryManager qm, final Project project, final Class<T> clazz) {
final Query<T> query = qm.getPersistenceManager().newQuery(clazz);
query.setFilter("project == :project");
Expand Down