Merge remote-tracking branch 'origin/v2-alpha' into gh-2457-double-ca…

…ching-issue

.gitignore

@@ -27,3 +27,6 @@ __pycache__
 parquet_data/
 **/graphLibrary/
 **/.vscode
+example/real-federated-store/*.properties
+example/real-federated-store/*.json
+example/real-federated-store/*.jar

example/federated-demo/basic/scripts/addMapEdgesClashingGraph.sh

@@ -36,7 +36,6 @@ curl -X POST --header 'Content-Type: application/json' --header 'Accept: applica
              }
            ]
          }
-       }
      }
     },
     "isPublic": true

example/federated-demo/src/main/resources/federatedStore.properties

@@ -15,3 +15,4 @@
 #
 gaffer.store.class=uk.gov.gchq.gaffer.federatedstore.FederatedStore
 gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.JcsCacheService
+gaffer.store.operation.declarations=./operationDeclarations.json

example/federated-demo/src/main/resources/operationDeclarations.json

@@ -0,0 +1,4 @@
+{
+  "operations": [
+  ]
+}

example/real-federated-store/README.md

@@ -0,0 +1,42 @@
+Copyright 2022 Crown Copyright
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Real FederatedStore Example
+=============
+This example uses a simple shell script that can build the Gaffer project, gathers
+default config files (federatedStore.properties, graphConfig.json, OperationDeclarations.json, schema.json)
+and an executable Gaffer Spring REST API. The REST API is then started backed by a real FederatedStore from the current config files.
+Users can change these local config files, target/indexed-disk-cache and then reboot the REST API. 
+The script will keep users changes to config files, it will only re-gather deleted/missing files. 
+This should give a good foundation for users to learn how to configure, interact and launch a real FederatedStore.
+
+## Deployment
+This example does not start or configure a real accumulo cluster. Without you having access to an Accumulo cluster you will not be able to use a Gaffer Accumulo Store.
+
+Assuming you have Java 8, Maven and Git installed, you can build and run the latest version of Gaffer FederatedStore locally by doing the following:
+```bash
+# Clone the Gaffer repository, to reduce the amount you need to download this will only clone the master branch with a depth of 1 so there won't be any history.
+git clone --depth 1 https://github.com/gchq/Gaffer.git #--branch master
+cd Gaffer/example/real-federated-store/
+
+# Run this script from this directory.
+# This will download several maven dependencies such as tomcat.
+./startRealFederatedStore.sh
+```
+
+The rest api will be deployed to localhost:8080/rest.
+
+
+## Example Operations
+Examples can be found at `have-a-go-at-operations`.

example/real-federated-store/have-a-go-at-operations/README.md

@@ -0,0 +1,35 @@
+Copyright 2022-2022 Crown Copyright
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Have A Go Operations
+=============
+
+Your FederatedStore  will be empty the first time, so a graph followed by elements will need to be added. Example operations can be found here.
+
+## JSON
+Example json operations can be found at `json-operation-examples`. 
+Copy & paste the files contents into the Swaggers execute panel.
+
+## Curl Command
+Example curl commands within shell scripts can be found at `curl-opertion-examples`. 
+Execute these shell scripts or copy & paste the curl command into your terminal.
+Curl commands with the "alternative" in the name are duplicate examples that contain inline JSON
+instead of reference to a .json file. This allows users to have a go at changing and editing operations in the command line.
+
+```bash
+cd curl-operation-examples/
+./curl_getAllGraphIds.sh
+```
+
+The rest api will be deployed to localhost:8080/rest.

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_add1BasicEdge.sh

@@ -0,0 +1 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d @../json-operation-examples/operation_add1BasicEdge.json 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_add1BasicEdge_alternative.sh

@@ -0,0 +1,16 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{
+    "class" : "uk.gov.gchq.gaffer.operation.impl.add.AddElements",
+    "input" : [ {
+     "group" : "BasicEdge",
+     "source" : "1",
+     "destination" : "2",
+     "directed" : true,
+     "properties" : {
+       "count" : 1
+     },
+     "class" : "uk.gov.gchq.gaffer.data.element.Edge"
+    } ],
+     "options": {
+      "gaffer.federatedstore.operation.graphIds": "mapEdges"
+      }
+ }' 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_addGraphForBasicEdges.sh

@@ -0,0 +1 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d @../json-operation-examples/operation_addGraphMapBasicEdges.json 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_addGraphForBasicEdges_alternative.sh

@@ -0,0 +1,41 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{
+    "class": "AddGraph",
+    "graphId": "mapEdges",
+    "storeProperties": {
+      "gaffer.store.class":"uk.gov.gchq.gaffer.mapstore.MapStore"
+    },
+    "schema": {
+         "edges": {
+             "BasicEdge": {
+               "source": "vertex",
+               "destination": "vertex",
+               "directed": "true",
+               "properties": {
+                 "count": "count"
+               }
+             }
+           },
+
+       "types": {
+         "vertex": {
+           "class": "java.lang.String"
+         },
+         "count": {
+           "class": "java.lang.Integer",
+           "aggregateFunction": {
+             "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Sum"
+           }
+         },
+         "true": {
+           "description": "A simple boolean that must always be true.",
+           "class": "java.lang.Boolean",
+           "validateFunctions": [
+             {
+               "class": "uk.gov.gchq.koryphe.impl.predicate.IsTrue"
+             }
+           ]
+         }
+       }
+    },
+    "isPublic": true
+ }' 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_getAllElements.sh

@@ -0,0 +1 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d @../json-operation-examples/operation_getAllElements.json 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_getAllGraphIds.sh

@@ -0,0 +1 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d @../json-operation-examples/operation_getAllGraphIds.json 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_getSchema.sh

@@ -0,0 +1 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d @../json-operation-examples/operation_getSchema.json 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_getSchema_alternative.sh

@@ -0,0 +1,4 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '
+{
+"class":"getSchema"
+}' 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_removeBasicEdgeGraph.sh

@@ -0,0 +1 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d @../json-operation-examples/operation_removeBasicEdgeGraph.json 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/curl-operation-examples/curl_removeBasicEdgeGraph_alternative.sh

@@ -0,0 +1,4 @@
+curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{
+"class": "removeGraph",
+"graphId": "mapEdges"
+}' 'http://localhost:8080/rest/graph/operations/execute'

example/real-federated-store/have-a-go-at-operations/json-operation-examples/operation_add1BasicEdge.json

@@ -0,0 +1,18 @@
+{
+  "class": "uk.gov.gchq.gaffer.operation.impl.add.AddElements",
+  "input": [
+    {
+      "group": "BasicEdge",
+      "source": "1",
+      "destination": "2",
+      "directed": true,
+      "properties": {
+        "count": 1
+      },
+      "class": "uk.gov.gchq.gaffer.data.element.Edge"
+    }
+  ],
+  "options": {
+    "gaffer.federatedstore.operation.graphIds": "mapEdges"
+  }
+}

example/real-federated-store/have-a-go-at-operations/json-operation-examples/operation_addGraphAccumuloBasicEdges.json

@@ -0,0 +1,26 @@
+{
+  "class" : "uk.gov.gchq.gaffer.federatedstore.operation.AddGraph",
+  "graphId" : "ToAccumulo",
+  "isPublic" : true,
+  "schema" : {
+    "edges" : {
+      "simple" : {
+        "source" : "String",
+        "destination" : "String"
+      }
+    },
+    "types" : {
+      "String" : {
+        "class" : "java.lang.String"
+      }
+    }
+  },
+  "storeProperties" : {
+    "gaffer.store.class" : "uk.gov.gchq.gaffer.accumulostore.AccumuloStore",
+    "accumulo.user" : "root",
+    "accumulo.instance" : "accumulo",
+    "gaffer.store.properties.class" : "uk.gov.gchq.gaffer.accumulostore.AccumuloProperties",
+    "accumulo.zookeepers" : "localhost:2181",
+    "accumulo.password" : "secret"
+  }
+}

example/real-federated-store/have-a-go-at-operations/json-operation-examples/operation_addGraphMapBasicEdges.json

@@ -0,0 +1,40 @@
+{
+  "class": "AddGraph",
+  "graphId": "mapEdges",
+  "storeProperties": {
+    "gaffer.store.class": "uk.gov.gchq.gaffer.mapstore.MapStore"
+  },
+  "schema": {
+    "edges": {
+      "BasicEdge": {
+        "source": "vertex",
+        "destination": "vertex",
+        "directed": "true",
+        "properties": {
+          "count": "count"
+        }
+      }
+    },
+    "types": {
+      "vertex": {
+        "class": "java.lang.String"
+      },
+      "count": {
+        "class": "java.lang.Integer",
+        "aggregateFunction": {
+          "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Sum"
+        }
+      },
+      "true": {
+        "description": "A simple boolean that must always be true.",
+        "class": "java.lang.Boolean",
+        "validateFunctions": [
+          {
+            "class": "uk.gov.gchq.koryphe.impl.predicate.IsTrue"
+          }
+        ]
+      }
+    }
+  },
+  "isPublic": true
+}

example/real-federated-store/have-a-go-at-operations/json-operation-examples/operation_getAllElements.json

@@ -0,0 +1,3 @@
+{
+  "class": "GetAllElements"
+}

example/real-federated-store/have-a-go-at-operations/json-operation-examples/operation_getAllGraphIds.json

@@ -0,0 +1,3 @@
+{
+  "class": "GetAllGraphIds"
+}

example/real-federated-store/have-a-go-at-operations/json-operation-examples/operation_getSchema.json

@@ -0,0 +1,3 @@
+{
+  "class": "GetSchema"
+}

example/real-federated-store/have-a-go-at-operations/json-operation-examples/operation_removeBasicEdgeGraph.json

@@ -0,0 +1,4 @@
+{
+  "class": "removeGraph",
+  "graphId": "mapEdges"
+}

example/real-federated-store/startRealFederatedStore.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Run this script from this repository.
+# Usage: ./realFederatedStore.sh
+
+SPRING_V=$(cat ../pom.xml | grep "^        <version>.*</version>$" | awk -F'[><]' '{print $3}')
+SPRING_JAR=spring-rest-$SPRING_V-exec.jar
+SPRING_TARGET=../../rest-api/spring-rest/target/$SPRING_JAR
+PROP=federatedStore.properties
+CONF=graphConfig.json
+SCHEMA=schema.json
+OP_DEC=operationDeclarations.json
+PROP_RESOURCE=../federated-demo/src/main/resources/$PROP
+CONF_RESOURCE=../federated-demo/src/main/resources/$CONF
+SCHEMA_RESOURCE=../federated-demo/src/main/resources/$SCHEMA
+OP_DEC_RESOURCE=../federated-demo/src/main/resources/$OP_DEC
+
+if [[ ! -f $CONF ]] 
+then
+	echo "Getting graph config $CONF_RESOURCE"
+	cp $CONF_RESOURCE ./$CONF
+fi
+
+if [[ ! -f $PROP ]] 
+then
+	echo "Getting properties file: $PROP_RESOURCE"
+	cp $PROP_RESOURCE ./$PROP
+fi
+
+if [[ ! -f $OP_DEC ]]
+then
+	echo "Getting operationDeclarations file: $OP_DEC_RESOURCE"
+	cp $OP_DEC_RESOURCE ./$OP_DEC
+fi
+
+if [[ ! -f $SCHEMA ]]
+then
+	echo "Making empty schema"
+	echo "{}" > $SCHEMA
+fi
+
+if [[ ! -f $SPRING_JAR ]]
+then
+	echo "Getting Spring Jar $SPRING_TARGET"
+	if [[ ! -f $SPRING_TARGET ]] 
+	then
+		echo "Spring Jar not found, so building project"
+		mvn clean install -Pquick -f ../../
+	fi 
+	cp $SPRING_TARGET  ./$SPRING_JAR
+	# echo "mvn clean"
+	# mvn clean -f ../../
+fi
+
+java -Dgaffer.storeProperties=$PROP -Dgaffer.graph.config=$CONF -Dgaffer.schemas=$SCHEMA -jar $SPRING_JAR