From bba03237fecee3cfccc1678bc722bcca0969cd51 Mon Sep 17 00:00:00 2001 From: "aliaksei.bykau" Date: Mon, 20 Nov 2023 08:59:19 +0100 Subject: [PATCH 1/5] [DSC-1355] removed config for dc.relation.ispartof from authority.cfg --- dspace/config/modules/authority.cfg | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dspace/config/modules/authority.cfg b/dspace/config/modules/authority.cfg index 037cf01e5d79..e1814c1669c5 100644 --- a/dspace/config/modules/authority.cfg +++ b/dspace/config/modules/authority.cfg @@ -271,10 +271,6 @@ choices.plugin.dc.identifier.issn = ZDBAuthority choices.presentation.dc.identifier.issn = suggest authority.controlled.dc.identifier.issn = true -choices.plugin.dc.relation.ispartof = SherpaAuthority -choices.presentation.dc.relation.ispartof = suggest -authority.controlled.dc.relation.ispartof = true - authority.controlled.dc.type = true choices.plugin.dc.type = ControlledVocabularyAuthority From e4f3cf4e8c5df10fb38010522ffaa43684c47e38 Mon Sep 17 00:00:00 2001 From: "aliaksei.bykau" Date: Fri, 5 Jan 2024 09:17:05 +0100 Subject: [PATCH 2/5] [DSC-1457] Updated logic of MetadataImporter, RegistryUpdater and RegistryLoader. Now you can load all registries with one script. --- .../dspace/administer/MetadataImporter.java | 37 +++++++++++++++++-- .../org/dspace/administer/RegistryLoader.java | 10 ++++- .../dspace/storage/rdbms/RegistryUpdater.java | 5 ++- dspace/config/dspace.cfg | 2 + 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java b/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java index 2677cb20501f..501d86af45f8 100644 --- a/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java +++ b/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java @@ -7,8 +7,13 @@ */ package org.dspace.administer; +import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.xpath.XPath; @@ -100,18 +105,39 @@ public static void main(String[] args) Options options = new Options(); options.addOption("f", "file", true, "source xml file for DC fields"); options.addOption("u", "update", false, "update an existing schema"); + options.addOption("h", "help", false, "help message"); CommandLine line = parser.parse(options, args); - if (line.hasOption('f')) { + if (line.hasOption('h')) { + usage(); + System.exit(1); + } else if (line.hasOption('f')) { String file = line.getOptionValue('f'); boolean forceUpdate = line.hasOption('u'); loadRegistry(file, forceUpdate); } else { - usage(); - System.exit(1); + boolean forceUpdate = line.hasOption('u'); + for (String file : getAllRegistryFiles()) { + loadRegistry(file, forceUpdate); + } } } + public static List getAllRegistryFiles() { + File folder = new File("config/registries"); + + if (folder.exists() && folder.isDirectory()) { + File[] files = folder.listFiles((dir, name) -> name.toLowerCase().endsWith(".xml")); + + if (files != null) { + return Arrays.stream(files) + .map(file -> "config/registries/" + file.getName()) + .collect(Collectors.toList()); + } + } + return Collections.emptyList(); + } + /** * Load the data from the specified file path into the database * @@ -285,7 +311,10 @@ private static void loadType(Context context, Node node) public static void usage() { String usage = "Use this class with the following option:\n" + " -f : specify which xml source file " + - "contains the DC fields to import.\n"; + "contains the DC fields to import.\n" + + "If you use the script without the -f parameter, then all" + + " registries will be loaded from the config/registries folder\n"; + System.out.println(usage); } } diff --git a/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java b/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java index bbf320a0d5e5..ba156cafc89e 100644 --- a/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java +++ b/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java @@ -33,6 +33,8 @@ import org.w3c.dom.NodeList; import org.xml.sax.SAXException; + + /** * Loads the bitstream format and Dublin Core type registries into the database. * Intended for use as a command-line tool. @@ -84,7 +86,13 @@ public static void main(String[] argv) throws Exception { RegistryLoader.loadBitstreamFormats(context, argv[1]); } else if (argv[0].equalsIgnoreCase("-metadata")) { // Call MetadataImporter, as it handles Metadata schema updates - MetadataImporter.loadRegistry(argv[1], true); + if (argv.length == 1) { + for (String file : MetadataImporter.getAllRegistryFiles()) { + MetadataImporter.loadRegistry(file, true); + } + } else { + MetadataImporter.loadRegistry(argv[1], true); + } } else { System.err.println(usage); } diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java index 7debf3ba449b..6a1d71b9e656 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java @@ -69,8 +69,9 @@ private void updateRegistries() { + "registries" + File.separator; // Load updates to Bitstream format registry (if any) - log.info("Updating Bitstream Format Registry based on {}bitstream-formats.xml", base); - RegistryLoader.loadBitstreamFormats(context, base + "bitstream-formats.xml"); + String bitstreamFormat = config.getProperty("registry.bitstream-formats.load"); + log.info("Updating Bitstream Format Registry based on {}{}", base, bitstreamFormat); + RegistryLoader.loadBitstreamFormats(context, base + bitstreamFormat); // Load updates to Metadata schema registries (if any) log.info("Updating Metadata Registries based on metadata type configs in {}", base); diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 05ebdd660679..dcba8455fdd4 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -1012,6 +1012,8 @@ registry.metadata.load = dspace-types.xml registry.metadata.load = iiif-types.xml registry.metadata.load = bitstream-types.xml +registry.bitstream-formats.load = bitstream-formats.xml + #---------------------------------------------------------------# #-----------------UI-Related CONFIGURATIONS---------------------# #---------------------------------------------------------------# From db85f99920876aa8dcd815d300f81de20d47444c Mon Sep 17 00:00:00 2001 From: "aliaksei.bykau" Date: Fri, 9 Feb 2024 13:55:31 +0100 Subject: [PATCH 3/5] =?UTF-8?q?[CST-13510]=20Update=20the=20registry-loade?= =?UTF-8?q?r=20script=20and=20related=20classes.=20Added=20new=20script=20?= =?UTF-8?q?argument=20=E2=80=9C-all=E2=80=9D=20that=20will=20run=20import?= =?UTF-8?q?=20of=20all=20metadata=20types=20and=20bitstream=20formats.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dspace/administer/MetadataImporter.java | 35 ++++++++++------- .../org/dspace/administer/RegistryLoader.java | 39 ++++++++++++++++--- .../dspace/storage/rdbms/RegistryUpdater.java | 30 +++++++------- 3 files changed, 68 insertions(+), 36 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java b/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java index 501d86af45f8..85f084b03cd0 100644 --- a/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java +++ b/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java @@ -11,7 +11,6 @@ import java.io.IOException; import java.sql.SQLException; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import javax.xml.parsers.ParserConfigurationException; @@ -35,6 +34,8 @@ import org.dspace.content.service.MetadataFieldService; import org.dspace.content.service.MetadataSchemaService; import org.dspace.core.Context; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -66,10 +67,18 @@ * } */ public class MetadataImporter { + public static final String BASE = DSpaceServicesFactory.getInstance() + .getConfigurationService().getProperty("dspace.dir") + File.separator + "config" + File.separator + + "registries" + File.separator; + public static final String REGISTRY_METADATA_PROPERTY = "registry.metadata.load"; + public static final String REGISTRY_BITSTREAM_FORMAT_PROPERTY = "registry.bitstream-formats.load"; + protected static MetadataSchemaService metadataSchemaService = ContentServiceFactory.getInstance() .getMetadataSchemaService(); protected static MetadataFieldService metadataFieldService = ContentServiceFactory.getInstance() .getMetadataFieldService(); + protected static ConfigurationService configurationService = DSpaceServicesFactory.getInstance() + .getConfigurationService(); /** * logging category @@ -117,25 +126,21 @@ public static void main(String[] args) loadRegistry(file, forceUpdate); } else { boolean forceUpdate = line.hasOption('u'); - for (String file : getAllRegistryFiles()) { + for (String file : getAllRegistryFiles(REGISTRY_METADATA_PROPERTY)) { loadRegistry(file, forceUpdate); } } } - public static List getAllRegistryFiles() { - File folder = new File("config/registries"); - - if (folder.exists() && folder.isDirectory()) { - File[] files = folder.listFiles((dir, name) -> name.toLowerCase().endsWith(".xml")); - - if (files != null) { - return Arrays.stream(files) - .map(file -> "config/registries/" + file.getName()) - .collect(Collectors.toList()); - } - } - return Collections.emptyList(); + /** + * Load all registry file names from config + * + * @param propertyName + * @return list of all registry files + */ + public static List getAllRegistryFiles(String propertyName) { + List files = Arrays.asList(configurationService.getArrayProperty(propertyName)); + return files.stream().map(file -> BASE + file).collect(Collectors.toList()); } /** diff --git a/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java b/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java index ba156cafc89e..7db0cc8d8936 100644 --- a/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java +++ b/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java @@ -70,7 +70,7 @@ private RegistryLoader() { } */ public static void main(String[] argv) throws Exception { String usage = "Usage: " + RegistryLoader.class.getName() - + " (-bitstream | -metadata) registry-file.xml"; + + " (-bitstream | -metadata | -all) registry-file.xml"; Context context = null; @@ -83,16 +83,21 @@ public static void main(String[] argv) throws Exception { // Work out what we're loading if (argv[0].equalsIgnoreCase("-bitstream")) { - RegistryLoader.loadBitstreamFormats(context, argv[1]); + if (argv.length == 1) { + loadAllBitstreamFormats(context); + } else { + RegistryLoader.loadBitstreamFormats(context, argv[1]); + } } else if (argv[0].equalsIgnoreCase("-metadata")) { // Call MetadataImporter, as it handles Metadata schema updates if (argv.length == 1) { - for (String file : MetadataImporter.getAllRegistryFiles()) { - MetadataImporter.loadRegistry(file, true); - } + loadAllRegistry(); } else { MetadataImporter.loadRegistry(argv[1], true); } + } else if (argv[0].equalsIgnoreCase("-all")) { + loadAllBitstreamFormats(context); + loadAllRegistry(); } else { System.err.println(usage); } @@ -119,6 +124,30 @@ public static void main(String[] argv) throws Exception { } } + + /** + * Load all bitstream formats from configuration properties + * + * @param context + * @throws Exception + */ + private static void loadAllBitstreamFormats(Context context) throws Exception { + for (String file : MetadataImporter.getAllRegistryFiles(MetadataImporter.REGISTRY_BITSTREAM_FORMAT_PROPERTY)) { + RegistryLoader.loadBitstreamFormats(context, file); + } + } + + /** + * Load all metadata registry from configuration properties + * + * @throws Exception + */ + private static void loadAllRegistry() throws Exception { + for (String file : MetadataImporter.getAllRegistryFiles(MetadataImporter.REGISTRY_METADATA_PROPERTY)) { + MetadataImporter.loadRegistry(file, true); + } + } + /** * Load Bitstream Format metadata * diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java index 6a1d71b9e656..1d9948690d86 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java @@ -7,9 +7,9 @@ */ package org.dspace.storage.rdbms; -import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.util.List; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.xpath.XPathExpressionException; @@ -20,8 +20,6 @@ import org.dspace.authorize.AuthorizeException; import org.dspace.content.NonUniqueMetadataException; import org.dspace.core.Context; -import org.dspace.services.ConfigurationService; -import org.dspace.services.factory.DSpaceServicesFactory; import org.flywaydb.core.api.callback.Callback; import org.flywaydb.core.api.callback.Event; import org.slf4j.Logger; @@ -58,31 +56,31 @@ public class RegistryUpdater implements Callback { * Method to actually update our registries from latest configuration files. */ private void updateRegistries() { - ConfigurationService config = DSpaceServicesFactory.getInstance().getConfigurationService(); Context context = null; try { context = new Context(); context.turnOffAuthorisationSystem(); - String base = config.getProperty("dspace.dir") - + File.separator + "config" + File.separator - + "registries" + File.separator; - - // Load updates to Bitstream format registry (if any) - String bitstreamFormat = config.getProperty("registry.bitstream-formats.load"); - log.info("Updating Bitstream Format Registry based on {}{}", base, bitstreamFormat); - RegistryLoader.loadBitstreamFormats(context, base + bitstreamFormat); + // Load updates to Bitstream formats registries (if any) + List RegistryMetadataFiles = + MetadataImporter.getAllRegistryFiles(MetadataImporter.REGISTRY_METADATA_PROPERTY); + for (String bitstreamFormat : RegistryMetadataFiles) { + log.info("Updating Bitstream Format Registry based on {}", bitstreamFormat); + RegistryLoader.loadBitstreamFormats(context, bitstreamFormat); + } // Load updates to Metadata schema registries (if any) - log.info("Updating Metadata Registries based on metadata type configs in {}", base); - for (String namespaceFile: config.getArrayProperty("registry.metadata.load")) { + List RegistryBitstreamFormatFiles = + MetadataImporter.getAllRegistryFiles(MetadataImporter.REGISTRY_BITSTREAM_FORMAT_PROPERTY); + log.info("Updating Metadata Registries based on metadata type configs in {}", MetadataImporter.BASE); + for (String namespaceFile: RegistryBitstreamFormatFiles) { log.info("Reading {}", namespaceFile); - MetadataImporter.loadRegistry(base + namespaceFile, true); + MetadataImporter.loadRegistry(namespaceFile, true); } String workflowTypes = "workflow-types.xml"; log.info("Reading {}", workflowTypes); - MetadataImporter.loadRegistry(base + workflowTypes, true); + MetadataImporter.loadRegistry( MetadataImporter.BASE + workflowTypes, true); context.restoreAuthSystemState(); // Commit changes and close context From 13aa12074f6d881adc3d00ec417ee0dfd9836862 Mon Sep 17 00:00:00 2001 From: "aliaksei.bykau" Date: Fri, 9 Feb 2024 13:57:07 +0100 Subject: [PATCH 4/5] [CST-13510] checkstyle --- .../src/main/java/org/dspace/administer/MetadataImporter.java | 2 +- .../src/main/java/org/dspace/administer/RegistryLoader.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java b/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java index 85f084b03cd0..7a1aaa782787 100644 --- a/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java +++ b/dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java @@ -135,7 +135,7 @@ public static void main(String[] args) /** * Load all registry file names from config * - * @param propertyName + * @param propertyName name of the property that used in config * @return list of all registry files */ public static List getAllRegistryFiles(String propertyName) { diff --git a/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java b/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java index 7db0cc8d8936..37876c587ee7 100644 --- a/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java +++ b/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java @@ -128,7 +128,7 @@ public static void main(String[] argv) throws Exception { /** * Load all bitstream formats from configuration properties * - * @param context + * @param context DSpace context object * @throws Exception */ private static void loadAllBitstreamFormats(Context context) throws Exception { From e666ff85b2aeb2baf0bfc05882310415807bf59e Mon Sep 17 00:00:00 2001 From: Andrea Bollini Date: Fri, 9 Feb 2024 16:53:25 +0100 Subject: [PATCH 5/5] DSC-1457 fix inverted list of registries (metadata, bitstream) --- .../dspace/storage/rdbms/RegistryUpdater.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java index 1d9948690d86..d6577dc7e19d 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java @@ -62,20 +62,20 @@ private void updateRegistries() { context.turnOffAuthorisationSystem(); // Load updates to Bitstream formats registries (if any) - List RegistryMetadataFiles = - MetadataImporter.getAllRegistryFiles(MetadataImporter.REGISTRY_METADATA_PROPERTY); - for (String bitstreamFormat : RegistryMetadataFiles) { + List registryBitstreamFormatFiles = + MetadataImporter.getAllRegistryFiles(MetadataImporter.REGISTRY_BITSTREAM_FORMAT_PROPERTY); + for (String bitstreamFormat : registryBitstreamFormatFiles) { log.info("Updating Bitstream Format Registry based on {}", bitstreamFormat); RegistryLoader.loadBitstreamFormats(context, bitstreamFormat); } // Load updates to Metadata schema registries (if any) - List RegistryBitstreamFormatFiles = - MetadataImporter.getAllRegistryFiles(MetadataImporter.REGISTRY_BITSTREAM_FORMAT_PROPERTY); + List registryMetadataFiles = + MetadataImporter.getAllRegistryFiles(MetadataImporter.REGISTRY_METADATA_PROPERTY); log.info("Updating Metadata Registries based on metadata type configs in {}", MetadataImporter.BASE); - for (String namespaceFile: RegistryBitstreamFormatFiles) { - log.info("Reading {}", namespaceFile); - MetadataImporter.loadRegistry(namespaceFile, true); + for (String metadataFile : registryMetadataFiles) { + log.info("Reading {}", metadataFile); + MetadataImporter.loadRegistry(metadataFile, true); } String workflowTypes = "workflow-types.xml";