Skip to content

Commit

Permalink
Standard / ISO / DCAT formatters / A bit closer to what EU-DCAT-AP SH…
Browse files Browse the repository at this point in the history 
…ACL rule expect. Tested with https://www.itb.ec.europa.eu/shacl/dcat-ap/upload. Difficulties to reproduce SHACL errors similar the the online validator. Could be related to not using the same SHACL rules or a difference with Jena validator?
  • Loading branch information
@fxprunayre
fxprunayre committed Apr 11, 2024
1 parent a5623a3 commit 40aeb69
Show file tree
Hide file tree
Showing 21 changed files with 2,596 additions and 1,780 deletions.
11 changes: 10 additions & 1 deletion schemas/iso19115-3.2018/src/main/plugin/iso19115-3.2018/formatter/dcat/dcat-core-contact.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
xmlns:mcc="http://standards.iso.org/iso/19115/-3/mcc/1.0"
xmlns:gcx="http://standards.iso.org/iso/19115/-3/gcx/1.0"
xmlns:gco="http://standards.iso.org/iso/19115/-3/gco/1.0"
xmlns:skos="http://www.w3.org/2004/02/skos/core#"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:prov="http://www.w3.org/ns/prov#"
xmlns:dcat="http://www.w3.org/ns/dcat#"
Expand Down Expand Up @@ -75,7 +76,15 @@
<xsl:call-template name="rdf-contact-foaf"/>
</prov:agent>
<dcat:hadRole>
<dcat:Role rdf:about="{concat($isoCodeListBaseUri, $role)}"/>
<dcat:Role rdf:about="{concat($isoCodeListBaseUri, $role)}">
<!--
Property needs to have at least 1 value
Location:
[Focus node] - [http://standards.iso.org/iso/19115/resources/Codelists/cat/codelists.xml#custodian] -
[Result path] - [http://www.w3.org/2004/02/skos/core#prefLabel]
-->
<skos:prefLabel><xsl:value-of select="$role"/></skos:prefLabel>
</dcat:Role>
</dcat:hadRole>
</prov:Attribution>
</prov:qualifiedAttribution>
Expand Down
10 changes: 8 additions & 2 deletions ...mas/iso19115-3.2018/src/main/plugin/iso19115-3.2018/formatter/dcat/dcat-core-keywords.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,24 @@
</xsl:for-each>
</xsl:template>

<!--
dcat:keyword is a rdfs:Literal and not a skos:Concept.
Main drawback is that the keyword is not linked to a concept in a concept scheme
which is often the case in ISO encoding using Anchor.
Using dcat:theme when an Anchor is present.
-->
<xsl:template mode="iso19115-3-to-dcat"
match="mdb:identificationInfo/*/mri:descriptiveKeywords/*/mri:keyword[gcx:Anchor/@xlink:href != '']"
priority="2">
<dcat:keyword>
<dcat:theme>
<skos:Concept>
<xsl:call-template name="rdf-object-ref-attribute"/>
<xsl:call-template name="rdf-localised">
<xsl:with-param name="nodeName"
select="'skos:prefLabel'"/>
</xsl:call-template>
</skos:Concept>
</dcat:keyword>
</dcat:theme>
</xsl:template>

<xsl:template mode="iso19115-3-to-dcat"
Expand Down
21 changes: 19 additions & 2 deletions schemas/iso19115-3.2018/src/main/plugin/iso19115-3.2018/formatter/dcat/dcat-core.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,11 +325,28 @@
</xsl:otherwise>
</xsl:choose>

<!--
SHACL rule
Value must be an instance of skos:Concept
Location:
[Focus node] - [https://xyz/geonetwork/srv/api/records/7fe2f305] -
[Result path] - [http://purl.org/dc/terms/type]
Test:
[Value] - [http://purl.org/dc/dcmitype/Dataset]
-->
<xsl:if test="$dcmiType">
<dct:type rdf:resource="http://purl.org/dc/dcmitype/{$dcmiType}"/>
<dct:type>
<skos:Concept rdf:about="http://purl.org/dc/dcmitype/{$dcmiType}">
<skos:prefLabel><xsl:value-of select="$dcmiType"/></skos:prefLabel>
</skos:Concept>
</dct:type>
</xsl:if>
<xsl:if test="$isPreservingIsoType and current() != ''">
<dct:type rdf:resource="{concat($isoCodeListBaseUri, current())}"/>
<dct:type>
<skos:Concept rdf:about="{concat($isoCodeListBaseUri, current())}">
<skos:prefLabel><xsl:value-of select="current()"/></skos:prefLabel>
</skos:Concept>
</dct:type>
</xsl:if>
<!-- TODO: Add mapping to Datacite https://schema.datacite.org/meta/kernel-4.1/include/datacite-resourceType-v4.1.xsd ?-->
</xsl:template>
Expand Down
122 changes: 69 additions & 53 deletions services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,23 +111,23 @@ public void checkFormatter() throws Exception {
String schema = testParameter[3];
String checkfile = testParameter[4];
String url = "/srv/api/records/"
+ testDataUuidBySchema.get(testFile)
+ "/formatters/" + formatter + urlParams;
+ testDataUuidBySchema.get(testFile)
+ "/formatters/" + formatter + urlParams;
try {
MvcResult result = mockMvc.perform(get(url)
.session(mockHttpSession)
.accept(MediaType.ALL_VALUE))
.andExpect(status().isOk())
.andReturn();
.session(mockHttpSession)
.accept(MediaType.ALL_VALUE))
.andExpect(status().isOk())
.andReturn();

String expected = StreamUtils.copyToString(
FormatterApiTest.class.getResourceAsStream(
String.format("%s-%s-%s",
schema, formatter, checkfile)
),
StandardCharsets.UTF_8)
.trim()
.replace("{uuid}", testDataUuidBySchema.get(testFile));
FormatterApiTest.class.getResourceAsStream(
String.format("%s-%s-%s",
schema, formatter, checkfile)
),
StandardCharsets.UTF_8)
.trim()
.replace("{uuid}", testDataUuidBySchema.get(testFile));

String actual = result.getResponse().getContentAsString();

Expand All @@ -147,38 +147,43 @@ public void checkFormatter() throws Exception {
}
}


// FileUtils.writeStringToFile(new File("/tmp/" + String.format("%s-%s-%s",
// schema, formatter, checkfile)), actual, StandardCharsets.UTF_8);

Diff diff = DiffBuilder
.compare(Input.fromString(actual))
.withTest(Input.fromString(expected))
.withNodeMatcher(new DefaultNodeMatcher(ElementSelectors.byName))
.normalizeWhitespace()
.ignoreComments()
.checkForSimilar()
.build();
.compare(Input.fromString(actual))
.withTest(Input.fromString(expected))
.withNodeMatcher(new DefaultNodeMatcher(ElementSelectors.byName))
.normalizeWhitespace()
.ignoreComments()
.checkForSimilar()
.build();
assertFalse(
String.format("%s. Checked with %s. Differences: %s", url, checkfile, diff.toString()),
diff.hasDifferences());
String.format("%s. Checked with %s. Differences: %s", url, checkfile, diff.toString()),
diff.hasDifferences());

if (isRdf) {
String[] shaclValidation = {};
// if("eu-dcat-ap".equalsIgnoreCase(formatter)){
//// shaclValidation = new String[]{"dcat-ap-2.1.1-base-SHACL.ttl"};
// shaclValidation = new String[]{"dcat-ap-3-SHACL.ttl"};
// // TODO: Failure with v3 shaclValidation = new String[]{"dcat-ap-2.1.1-base-SHACL.ttl", "dcat-ap-3-SHACL.ttl"};
if ("eu-dcat-ap".equalsIgnoreCase(formatter)) {
// shaclValidation = new String[]{"dcat-ap-2.1.1-base-SHACL.ttl"};
// https://github.com/ISAITB/validator-resources-dcat-ap/blob/master/resources/config.properties#L117-L128
//+ shapes.ttl + range.ttl + shapes_recommended.ttl + imports.ttl + deprecateduris.ttl+
// shaclValidation = new String[]{"shacl/eu-dcat-ap-3.0.0/shapes.ttl"};
// } else if("eu-dcat-ap-hvd".equalsIgnoreCase(formatter)){
// shaclValidation = new String[]{"dcat-ap-hvd-2.2.0-SHACL.ttl"};
// } else if("eu-geodcat-ap".equalsIgnoreCase(formatter)){
// shaclValidation = new String[]{"geodcat-ap-2.0.1-SHACL.ttl"};
// }
// for(String shaclShapes : shaclValidation) {
// applyShaclValidation(formatter, schema, checkfile, url, shaclShapes);
// }
}
for (String shaclShapes : shaclValidation) {
applyShaclValidation(formatter, schema, checkfile, url, shaclShapes);
}
}
} else {
assertEquals(
url,
expected,
actual.replaceAll("\\r\\n?", "\n")
url,
expected,
actual.replaceAll("\\r\\n?", "\n")
);
}
} catch (Exception e) {
Expand All @@ -196,55 +201,66 @@ public void quickTestToValidateRdfModelAndShaclRules() throws IOException {
String checkfile = "dataset-core.rdf";
String file = String.format("%s-%s-%s", schema, formatter, checkfile);
String expected = StreamUtils.copyToString(
FormatterApiTest.class.getResourceAsStream(file),
StandardCharsets.UTF_8);
FormatterApiTest.class.getResourceAsStream(file),
StandardCharsets.UTF_8);
try {
Model model = ModelFactory.createMemModelMaker().createDefaultModel();
RDFDataMgr.read(model,
IOUtils.toInputStream(expected, StandardCharsets.UTF_8),
Lang.RDFXML);
IOUtils.toInputStream(expected, StandardCharsets.UTF_8),
Lang.RDFXML);
} catch (Exception rdfException) {
fail(String.format("%s. RDF model error. %s.",
file, rdfException.getMessage()));
file, rdfException.getMessage()));
}
String[] shaclValidation = new String[]{"dcat-ap-2.1.1-base-SHACL.ttl"};
// String[] shaclValidation = new String[]{"dcat-ap-3-SHACL.ttl"};
// String[] shaclValidation = new String[]{"dcat-ap-hvd-2.2.0-SHACL.ttl"};
// String[] shaclValidation = new String[]{"geodcat-ap-2.0.1-SHACL.ttl"};
for(String shaclShapes : shaclValidation) {
for (String shaclShapes : shaclValidation) {
applyShaclValidation(formatter, schema, checkfile, "", shaclShapes);
}
}

private static void applyShaclValidation(String formatter, String schema, String checkfile, String url, String shaclShapes) {
String SHAPES = FormatterApiTest.class.getResource(shaclShapes).getFile();
if(SHAPES.startsWith("/")){ SHAPES.replaceFirst("/","");}
if (SHAPES.startsWith("/")) {
SHAPES.replaceFirst("/", "");
}

//Load document to validate.
String DATA = FormatterApiTest.class.getResource(
String.format("%s-%s-%s",
schema, formatter, checkfile)
String.format("%s-%s-%s",
schema, formatter, checkfile)
).getFile();
if(DATA.startsWith("/")){
DATA.replaceFirst("/","");
if (DATA.startsWith("/")) {
DATA.replaceFirst("/", "");
}
Graph shapesGraph;
Shapes shapes;
try {
shapesGraph = RDFDataMgr.loadGraph(SHAPES);
shapes = Shapes.parse(shapesGraph);
} catch (Exception e) {
fail(String.format(
"%s. Checked with %s [%s]. SHACL graph error. Error is: %s",
url, checkfile, shaclShapes, e.getMessage()));
return;
}

Graph shapesGraph = RDFDataMgr.loadGraph(SHAPES);
Graph dataGraph = RDFDataMgr.loadGraph(DATA);

Shapes shapes = Shapes.parse(shapesGraph);

ValidationReport report = ShaclValidator.get().validate(shapes, dataGraph);

if(!report.conforms()){
if (!report.conforms()) {
long count = report.getEntries().stream()
.filter(e -> e.severity().level().getURI().equals("http://www.w3.org/ns/shacl#Violation"))
.count();
.filter(e -> e.severity().level().getURI().equals("http://www.w3.org/ns/shacl#Violation"))
.count();

ShLib.printReport(report);
System.out.println();
RDFDataMgr.write(System.out, report.getModel(), Lang.TTL);
fail(String.format("%s. Checked with %s [%s]. Invalid DCAT-AP document. %d violations found. See report in the test console output.",
url, checkfile, shaclShapes, count));
url, checkfile, shaclShapes, count));
}
}

Expand All @@ -262,8 +278,8 @@ private void createTestData() throws Exception {
loadFile("iso19115-3.2018", getSampleISO19115MetadataXml());
} else {
loadFile(file,
Xml.loadStream(
FormatterApiTest.class.getResourceAsStream(file)));
Xml.loadStream(
FormatterApiTest.class.getResourceAsStream(file)));
}
}
}
Expand Down
Loading

0 comments on commit 40aeb69

Please sign in to comment.