Skip to content

Commit

Permalink
Merge branch 'develop' into gh-3355-delete-elements-to-return-count
Browse files Browse the repository at this point in the history
  • Loading branch information
@j69772
j69772 authored Jan 21, 2025
2 parents b263383 + 07694ea commit 4531628
Show file tree
Hide file tree
Showing 12 changed files with 259 additions and 267 deletions.
3 changes: 2 additions & 1 deletion core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2024 Crown Copyright
* Copyright 2024-2025 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -25,6 +25,7 @@ public final class OtelUtil {
public static final String JOB_ID_ATTRIBUTE = "gaffer.jobId";
public static final String GRAPH_ID_ATTRIBUTE = "gaffer.graphId";
public static final String VIEW_ATTRIBUTE = "gaffer.view";
public static final String OP_OPTIONS_ATTRIBUTE = "gaffer.operation.options";
public static final String GREMLIN_QUERY_ATTRIBUTE = "gaffer.gremlin.query";

private static boolean openTelemetryActive = false;
Expand Down
5 changes: 3 additions & 2 deletions core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2024 Crown Copyright
* Copyright 2017-2025 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -320,10 +320,11 @@ private <O> GraphResult<O> _execute(final StoreExecuter<O> storeExecuter, final
// OpenTelemetry hooks
Span span = OtelUtil.startSpan(
this.getClass().getName(),
"Graph Request: " + clonedOpChain.toOverviewString());
"Graph Request: " + clonedContext.getJobId());
span.setAttribute(OtelUtil.GRAPH_ID_ATTRIBUTE, getGraphId());
span.setAttribute(OtelUtil.JOB_ID_ATTRIBUTE, clonedContext.getJobId());
span.setAttribute(OtelUtil.USER_ATTRIBUTE, clonedContext.getUser().getUserId());
span.setAttribute(OtelUtil.OP_OPTIONS_ATTRIBUTE, clonedOpChain.getOptions().toString());

O result = null;
// Sets the span to current so parent child spans are auto linked
Expand Down
10 changes: 9 additions & 1 deletion ...store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,16 @@ public OUT doOperation(final OperationChain<OUT> operationChain, final Context c
// OpenTelemetry hooks
Span span = OtelUtil.startSpan(this.getClass().getName(), op.getClass().getName());
span.setAttribute(OtelUtil.JOB_ID_ATTRIBUTE, context.getJobId());
span.setAttribute(OtelUtil.OP_OPTIONS_ATTRIBUTE, (op.getOptions() != null) ? op.getOptions().toString() : "[]");
// Extract the view
if (op instanceof OperationView && ((OperationView) op).getView() != null) {
span.setAttribute(OtelUtil.VIEW_ATTRIBUTE, ((OperationView) op).getView().toString());
String strView = ((OperationView) op).getView().toString();
// Truncate the view if its too long
if (strView.length() > 2048) {
span.setAttribute(OtelUtil.VIEW_ATTRIBUTE, strView.substring(0, 2048));
} else {
span.setAttribute(OtelUtil.VIEW_ATTRIBUTE, strView);
}
}

// Sets the span to current so parent child spans are auto linked
Expand Down
202 changes: 102 additions & 100 deletions library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2016-2024 Crown Copyright
* Copyright 2016-2025 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -253,11 +253,11 @@ public enum DefaultIdManager {

private final Graph graph;
private final Configuration configuration;
private final GafferPopGraphVariables variables;
private final GafferPopGraphFeatures features;
private final Map<String, String> opOptions;
private final User defaultUser;
private final ServiceRegistry serviceRegistry;
private final GafferPopGraphVariables variables = new GafferPopGraphVariables();
private final GafferPopGraphFeatures features = new GafferPopGraphFeatures();
private final Map<String, String> opOptions = new HashMap<>();
private final ServiceRegistry serviceRegistry = new ServiceRegistry();

private static final Logger LOGGER = LoggerFactory.getLogger(GafferPopGraph.class);
private static final String GET_DEBUG_MSG = "Requested a GetElements, results will be truncated to: {}.";
Expand All @@ -270,8 +270,6 @@ public GafferPopGraph(final Configuration configuration) {
public GafferPopGraph(final Configuration configuration, final Graph graph) {
this.configuration = configuration;
this.graph = graph;
features = new GafferPopGraphFeatures();
opOptions = new HashMap<>();
if (configuration().containsKey(OP_OPTIONS)) {
for (final String option : configuration().getStringArray(OP_OPTIONS)) {
final String[] parts = option.split(":");
Expand All @@ -285,10 +283,8 @@ public GafferPopGraph(final Configuration configuration, final Graph graph) {
.build();

// Set the graph variables to current config
variables = new GafferPopGraphVariables();
setDefaultVariables(variables);
setDefaultVariables();

serviceRegistry = new ServiceRegistry();
serviceRegistry.registerService(new GafferPopNamedOperationServiceFactory(this));

// Add and register custom traversals
Expand All @@ -299,6 +295,16 @@ public GafferPopGraph(final Configuration configuration, final Graph graph) {
GlobalCache.registerStrategies(this.getClass(), traversalStrategies);
}

/**
* Return a new instance of the graph usually so a different set
* of graph variables can be used for a query.
*
* @return Identical instance this graph.
*/
public GafferPopGraph newInstance() {
return new GafferPopGraph(configuration, graph);
}

private static Graph createGraph(final Configuration configuration) {
final String graphId = configuration.getString(GRAPH_ID);
if (null == graphId) {
Expand Down Expand Up @@ -354,21 +360,6 @@ public Vertex addVertex(final Object... keyValues) {
idValue = ElementHelper.getIdValue(keyValues).orElseThrow(() -> new IllegalArgumentException("ID is required"));
}

/*
* TODO: Check the ID type is relevant for the group (a.k.a label) in the schema and auto convert
* as the some Standard tinkerpop tests add data for the same group but with a different
* Object type for the ID. Using a String ID manager might be the most flexible for these
* tests.
* Basic idea of auto converting the type is below:
*
* String idSchemaType = graph.getSchema().getEntity(label).getVertex();
* String idTypeName = graph.getSchema().getType(idSchemaType).getFullClassString();
* if (!idTypeName.equals(idValue.getClass().getName())) {
* LOGGER.warn("Vertex ID is not the correct type for the schema: " + idValue);
* idValue = graph.getSchema().getType(idSchemaType).getClazz().cast(idValue);
* }
*/

final GafferPopVertex vertex = new GafferPopVertex(label, idValue, this);
ElementHelper.attachProperties(vertex, VertexProperty.Cardinality.list, keyValues);
addVertex(vertex);
Expand Down Expand Up @@ -438,7 +429,7 @@ public Iterator<Vertex> vertices(final Object... vertexIds) {
.first(getOperation)
.then(new Limit<>(variables.getElementsLimit(), true))
.build();
final Set<Element> result = new HashSet<>(IterableUtils.toList(execute(chain)));
final List<? extends Element> result = IterableUtils.toList(execute(chain));

// Warn of truncation
if (result.size() >= variables.getElementsLimit()) {
Expand All @@ -447,13 +438,13 @@ public Iterator<Vertex> vertices(final Object... vertexIds) {
variables.getElementsLimit());
}

// Translate results to Gafferpop elements
// Translate results to GafferPop elements
final GafferPopElementGenerator generator = new GafferPopElementGenerator(this);
final Set<Vertex> translatedResults = StreamSupport.stream(result.spliterator(), false)
final List<Vertex> translatedResults = StreamSupport.stream(result.spliterator(), false)
.map(generator::_apply)
.filter(Vertex.class::isInstance)
.map(e -> (Vertex) e)
.collect(Collectors.toSet());
.collect(Collectors.toList());

// Check for seeds that are not entities but are vertices on an edge (orphan vertices)
if (variables.getIncludeOrphanedVertices()) {
Expand Down Expand Up @@ -606,16 +597,16 @@ public Iterator<Edge> edges(final Object... elementIds) {
}

// Run requested chain on the graph and buffer to set to avoid reusing iterator
final Set<Element> result = new HashSet<>(IterableUtils.toList(execute(getOperation)));
final List<? extends Element> result = IterableUtils.toList(execute(getOperation));

// Translate results to Gafferpop elements
final GafferPopElementGenerator generator = new GafferPopElementGenerator(this);
final Set<Edge> translatedResults = StreamSupport.stream(result.spliterator(), false)
final List<Edge> translatedResults = StreamSupport.stream(result.spliterator(), false)
.map(generator::_apply)
.filter(Edge.class::isInstance)
.map(e -> (Edge) e)
.limit(variables.getElementsLimit())
.collect(Collectors.toSet());
.collect(Collectors.toList());

if (translatedResults.size() >= variables.getElementsLimit()) {
LOGGER.warn(
Expand Down Expand Up @@ -667,60 +658,6 @@ public Iterator<Edge> edgesWithView(final Object id, final Direction direction,
return edgesWithView(Collections.singletonList(id), direction, view);
}

/**
* This performs a GetElements operation filtering edges by direction and view.
*
* @param ids vertex IDs or edge IDs to be queried for.
* Supports input as a {@link Vertex}, {@link Edge}, List of Edge IDs or individual Vertex IDs.
* @param direction {@link Direction} of edges to return.
* @param view Gaffer {@link View} to filter edges by
* @return iterator of {@link GafferPopEdge}s.
* @see #edges(Object...)
*/
public Iterator<Edge> edgesWithView(final Iterable<Object> ids, final Direction direction, final View view) {
return edgesWithSeedsAndView(getElementSeeds(ids), direction, view);
}

@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) throws IllegalArgumentException {
throw Exceptions.graphComputerNotSupported();
}

@Override
public GraphComputer compute() throws IllegalArgumentException {
throw Exceptions.graphComputerNotSupported();
}

@Override
public Transaction tx() {
throw Exceptions.transactionsNotSupported();
}

@Override
public Variables variables() {
return this.variables;
}

@Override
public Configuration configuration() {
return configuration;
}

@Override
public void close() throws Exception {
serviceRegistry.close();
}

@Override
public ServiceRegistry getServiceRegistry() {
return serviceRegistry;
}

@Override
public Features features() {
return features;
}

public <T> T execute(final OperationChain<T> opChain) {
// Set options at opChain level
opChain.setOptions(variables.getOperationOptions());
Expand Down Expand Up @@ -754,14 +691,25 @@ public <T> T execute(final OperationChain<T> opChain) {

/**
* Sets the {@link GafferPopGraphVariables} to default values for this
* graph
* graph.
*/
public void setDefaultVariables() {
setDefaultVariables(false);
}

/**
* Sets the {@link GafferPopGraphVariables} to default values for this
* graph optionally preserving the current user.
*
* @param variables The variables
* @param preserveUser keep the current set user.
*/
public void setDefaultVariables(final GafferPopGraphVariables variables) {
public void setDefaultVariables(final boolean preserveUser) {
LOGGER.debug("Resetting graph variables to defaults");
if (!preserveUser) {
LOGGER.debug("Resetting graph user to default");
variables.set(GafferPopGraphVariables.USER, defaultUser);
}
variables.set(GafferPopGraphVariables.OP_OPTIONS, Collections.unmodifiableMap(opOptions));
variables.set(GafferPopGraphVariables.USER, defaultUser);
variables.set(GafferPopGraphVariables.GET_ELEMENTS_LIMIT,
configuration().getInteger(GET_ELEMENTS_LIMIT, DEFAULT_GET_ELEMENTS_LIMIT));
variables.set(GafferPopGraphVariables.HAS_STEP_FILTER_STAGE,
Expand All @@ -780,6 +728,60 @@ public Graph getGafferGraph() {
return graph;
}

/**
* This performs a GetElements operation filtering edges by direction and view.
*
* @param ids vertex IDs or edge IDs to be queried for.
* Supports input as a {@link Vertex}, {@link Edge}, List of Edge IDs or individual Vertex IDs.
* @param direction {@link Direction} of edges to return.
* @param view Gaffer {@link View} to filter edges by
* @return iterator of {@link GafferPopEdge}s.
* @see #edges(Object...)
*/
public Iterator<Edge> edgesWithView(final Iterable<Object> ids, final Direction direction, final View view) {
return edgesWithSeedsAndView(getElementSeeds(ids), direction, view);
}

@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) throws IllegalArgumentException {
throw Exceptions.graphComputerNotSupported();
}

@Override
public GraphComputer compute() throws IllegalArgumentException {
throw Exceptions.graphComputerNotSupported();
}

@Override
public Transaction tx() {
throw Exceptions.transactionsNotSupported();
}

@Override
public Variables variables() {
return this.variables;
}

@Override
public Configuration configuration() {
return configuration;
}

@Override
public void close() throws Exception {
serviceRegistry.close();
}

@Override
public ServiceRegistry getServiceRegistry() {
return serviceRegistry;
}

@Override
public Features features() {
return features;
}

private Iterator<GafferPopVertex> verticesWithSeedsAndView(final List<ElementSeed> seeds, final View view) {
final boolean getAll = null == seeds || seeds.isEmpty();
final LinkedList<GafferPopVertex> idVertices = new LinkedList<>();
Expand Down Expand Up @@ -817,15 +819,15 @@ private Iterator<GafferPopVertex> verticesWithSeedsAndView(final List<ElementSee
.first(getOperation)
.then(new Limit<>(variables.getElementsLimit(), true))
.build();
final Set<Element> result = new HashSet<>(IterableUtils.toList(execute(chain)));
final List<? extends Element> result = IterableUtils.toList(execute(chain));

// Translate results to Gafferpop elements
final GafferPopElementGenerator generator = new GafferPopElementGenerator(this);
final Set<GafferPopVertex> translatedResults = StreamSupport.stream(result.spliterator(), false)
final List<GafferPopVertex> translatedResults = StreamSupport.stream(result.spliterator(), false)
.map(generator::_apply)
.filter(GafferPopVertex.class::isInstance)
.map(e -> (GafferPopVertex) e)
.collect(Collectors.toSet());
.collect(Collectors.toList());

return translatedResults.iterator();
}
Expand Down Expand Up @@ -853,16 +855,16 @@ private Iterator<Vertex> adjVerticesWithSeedsAndView(final List<ElementSeed> see
}

// GetAdjacentIds provides list of entity seeds so run a GetElements to get the actual Entities
final Set<Element> result = new HashSet<>(IterableUtils.toList(
execute(new OperationChain.Builder().first(builder.build()).build())));
final List<? extends Element> result = IterableUtils.toList(
execute(new OperationChain.Builder().first(builder.build()).build()));

// Translate results to Gafferpop elements
final GafferPopElementGenerator generator = new GafferPopElementGenerator(this);
final Set<Vertex> translatedResults = StreamSupport.stream(result.spliterator(), false)
final List<Vertex> translatedResults = StreamSupport.stream(result.spliterator(), false)
.map(generator::_apply)
.filter(Vertex.class::isInstance)
.map(e -> (Vertex) e)
.collect(Collectors.toSet());
.collect(Collectors.toList());

// Check for seeds that are not entities but are vertices on an edge (orphan vertices)
if (variables.getIncludeOrphanedVertices()) {
Expand Down Expand Up @@ -907,16 +909,16 @@ private Iterator<Edge> edgesWithSeedsAndView(final List<ElementSeed> seeds, fina
}

// Run requested chain on the graph
final Set<Element> result = new HashSet<>(IterableUtils.toList(execute(getOperation)));
final List<? extends Element> result = IterableUtils.toList(execute(getOperation));

// Translate results to Gafferpop elements
final GafferPopElementGenerator generator = new GafferPopElementGenerator(this, true);
final Set<Edge> translatedResults = StreamSupport.stream(result.spliterator(), false)
final List<Edge> translatedResults = StreamSupport.stream(result.spliterator(), false)
.map(generator::_apply)
.filter(Edge.class::isInstance)
.map(e -> (Edge) e)
.limit(variables.getElementsLimit())
.collect(Collectors.toSet());
.collect(Collectors.toList());

return translatedResults.iterator();
}
Expand Down
Loading

0 comments on commit 4531628

Please sign in to comment.