From affa4b00ff59cffa69d47de6a3a7d4137008dd40 Mon Sep 17 00:00:00 2001 From: Vincenzo Mecca Date: Wed, 24 Jul 2024 19:07:44 +0200 Subject: [PATCH 001/178] [CST-14905] Orcid revoke token feature --- .../org/dspace/orcid/client/OrcidClient.java | 8 + .../dspace/orcid/client/OrcidClientImpl.java | 37 +++++ .../orcid/client/OrcidConfiguration.java | 9 ++ .../impl/OrcidSynchronizationServiceImpl.java | 14 ++ .../ResearcherProfileRestRepositoryIT.java | 146 ++++++++++++++++-- dspace/config/modules/orcid.cfg | 1 + dspace/config/spring/api/orcid-services.xml | 1 + 7 files changed, 200 insertions(+), 16 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/orcid/client/OrcidClient.java b/dspace-api/src/main/java/org/dspace/orcid/client/OrcidClient.java index 99d1920aa53a..d21f61a922f5 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/client/OrcidClient.java +++ b/dspace-api/src/main/java/org/dspace/orcid/client/OrcidClient.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Optional; +import org.dspace.orcid.OrcidToken; import org.dspace.orcid.exception.OrcidClientException; import org.dspace.orcid.model.OrcidTokenResponseDTO; import org.orcid.jaxb.model.v3.release.record.Person; @@ -161,4 +162,11 @@ public interface OrcidClient { */ OrcidResponse deleteByPutCode(String accessToken, String orcid, String putCode, String path); + /** + * Revokes the given {@param accessToken} with a POST method. + * @param orcidToken the access token to revoke + * @throws OrcidClientException if some error occurs during the search + */ + void revokeToken(OrcidToken orcidToken); + } diff --git a/dspace-api/src/main/java/org/dspace/orcid/client/OrcidClientImpl.java b/dspace-api/src/main/java/org/dspace/orcid/client/OrcidClientImpl.java index 3e7ca7b21029..1532abe63412 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/client/OrcidClientImpl.java +++ b/dspace-api/src/main/java/org/dspace/orcid/client/OrcidClientImpl.java @@ -42,6 +42,7 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicNameValuePair; +import org.dspace.orcid.OrcidToken; import org.dspace.orcid.exception.OrcidClientException; import org.dspace.orcid.model.OrcidEntityType; import org.dspace.orcid.model.OrcidProfileSectionType; @@ -178,6 +179,16 @@ public OrcidResponse deleteByPutCode(String accessToken, String orcid, String pu return execute(buildDeleteUriRequest(accessToken, "/" + orcid + path + "/" + putCode), true); } + @Override + public void revokeToken(OrcidToken orcidToken) { + List params = new ArrayList<>(); + params.add(new BasicNameValuePair("client_id", orcidConfiguration.getClientId())); + params.add(new BasicNameValuePair("client_secret", orcidConfiguration.getClientSecret())); + params.add(new BasicNameValuePair("token", orcidToken.getAccessToken())); + + executeSuccessful(buildPostForRevokeToken(new UrlEncodedFormEntity(params, Charset.defaultCharset()))); + } + @Override public OrcidTokenResponseDTO getReadPublicAccessToken() { return getClientCredentialsAccessToken("/read-public"); @@ -220,6 +231,14 @@ private HttpUriRequest buildPostUriRequest(String accessToken, String relativePa .build(); } + private HttpUriRequest buildPostForRevokeToken(HttpEntity entity) { + return post(orcidConfiguration.getRevokeUrl()) + .addHeader("Accept", "application/json") + .addHeader("Content-Type", "application/x-www-form-urlencoded") + .setEntity(entity) + .build(); + } + private HttpUriRequest buildPutUriRequest(String accessToken, String relativePath, Object object) { return put(orcidConfiguration.getApiUrl() + relativePath.trim()) .addHeader("Content-Type", "application/vnd.orcid+xml") @@ -234,6 +253,24 @@ private HttpUriRequest buildDeleteUriRequest(String accessToken, String relative .build(); } + private void executeSuccessful(HttpUriRequest httpUriRequest) { + try { + HttpClient client = HttpClientBuilder.create().build(); + HttpResponse response = client.execute(httpUriRequest); + + if (isNotSuccessfull(response)) { + throw new OrcidClientException( + getStatusCode(response), + "Operation " + httpUriRequest.getMethod() + " for the resource " + httpUriRequest.getURI() + + " was not successful: " + new String(response.getEntity().getContent().readAllBytes(), + StandardCharsets.UTF_8) + ); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private T executeAndParseJson(HttpUriRequest httpUriRequest, Class clazz) { HttpClient client = HttpClientBuilder.create().build(); diff --git a/dspace-api/src/main/java/org/dspace/orcid/client/OrcidConfiguration.java b/dspace-api/src/main/java/org/dspace/orcid/client/OrcidConfiguration.java index 550b0215c435..dfa90fcae03a 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/client/OrcidConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/orcid/client/OrcidConfiguration.java @@ -35,6 +35,8 @@ public final class OrcidConfiguration { private String scopes; + private String revokeUrl; + public String getApiUrl() { return apiUrl; } @@ -111,4 +113,11 @@ public boolean isApiConfigured() { return !StringUtils.isAnyBlank(clientId, clientSecret); } + public String getRevokeUrl() { + return revokeUrl; + } + + public void setRevokeUrl(String revokeUrl) { + this.revokeUrl = revokeUrl; + } } diff --git a/dspace-api/src/main/java/org/dspace/orcid/service/impl/OrcidSynchronizationServiceImpl.java b/dspace-api/src/main/java/org/dspace/orcid/service/impl/OrcidSynchronizationServiceImpl.java index 97d832d3de82..a302c58f2c5e 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/service/impl/OrcidSynchronizationServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/orcid/service/impl/OrcidSynchronizationServiceImpl.java @@ -37,6 +37,7 @@ import org.dspace.eperson.EPerson; import org.dspace.eperson.service.EPersonService; import org.dspace.orcid.OrcidToken; +import org.dspace.orcid.client.OrcidClient; import org.dspace.orcid.model.OrcidEntityType; import org.dspace.orcid.model.OrcidTokenResponseDTO; import org.dspace.orcid.service.OrcidSynchronizationService; @@ -47,6 +48,8 @@ import org.dspace.profile.OrcidSynchronizationMode; import org.dspace.profile.service.ResearcherProfileService; import org.dspace.services.ConfigurationService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; /** @@ -57,6 +60,7 @@ */ public class OrcidSynchronizationServiceImpl implements OrcidSynchronizationService { + private static final Logger log = LoggerFactory.getLogger(OrcidSynchronizationServiceImpl.class); @Autowired private ItemService itemService; @@ -75,6 +79,9 @@ public class OrcidSynchronizationServiceImpl implements OrcidSynchronizationServ @Autowired private ResearcherProfileService researcherProfileService; + @Autowired + private OrcidClient orcidClient; + @Override public void linkProfile(Context context, Item profile, OrcidTokenResponseDTO token) throws SQLException { @@ -118,7 +125,14 @@ public void unlinkProfile(Context context, Item profile) throws SQLException { itemService.clearMetadata(context, profile, "dspace", "orcid", "scope", Item.ANY); itemService.clearMetadata(context, profile, "dspace", "orcid", "authenticated", Item.ANY); + OrcidToken profileToken = orcidTokenService.findByProfileItem(context, profile); + if (profileToken == null) { + log.warn("Cannot find any token related to the user profile: {}", profile.getID()); + return; + } + orcidTokenService.deleteByProfileItem(context, profile); + orcidClient.revokeToken(profileToken); updateItem(context, profile); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ResearcherProfileRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ResearcherProfileRestRepositoryIT.java index 70009d049fc5..0d46f4268cae 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ResearcherProfileRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ResearcherProfileRestRepositoryIT.java @@ -30,7 +30,10 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -44,6 +47,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; import java.sql.SQLException; import java.util.List; import java.util.UUID; @@ -79,11 +83,13 @@ import org.dspace.orcid.exception.OrcidClientException; import org.dspace.orcid.model.OrcidTokenResponseDTO; import org.dspace.orcid.service.OrcidQueueService; +import org.dspace.orcid.service.OrcidSynchronizationService; import org.dspace.orcid.service.OrcidTokenService; import org.dspace.services.ConfigurationService; import org.dspace.util.UUIDUtils; import org.junit.After; import org.junit.Test; +import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MvcResult; @@ -114,7 +120,11 @@ public class ResearcherProfileRestRepositoryIT extends AbstractControllerIntegra @Autowired private OrcidClient orcidClient; - private OrcidClient orcidClientMock = mock(OrcidClient.class); + @Mock + private OrcidClient orcidClientMock; + + @Autowired + private OrcidSynchronizationService orcidSynchronizationService; private EPerson user; @@ -158,16 +168,36 @@ public void setUp() throws Exception { context.restoreAuthSystemState(); - researcherProfileAddOrcidOperation.setOrcidClient(orcidClientMock); - + useInstanceForBean(orcidSynchronizationService, orcidClientMock); + useInstanceForBean(researcherProfileAddOrcidOperation, orcidClientMock); } @After public void after() { orcidTokenService.deleteAll(context); - researcherProfileAddOrcidOperation.setOrcidClient(orcidClient); + useInstanceForBean(orcidSynchronizationService, orcidClient); + useInstanceForBean(researcherProfileAddOrcidOperation, orcidClient); + } + + private void useInstanceForBean(B bean, I instance) { + Field[] fields = bean.getClass().getDeclaredFields(); + + for (Field field : fields) { + if (field.getType().isAssignableFrom(instance.getClass())) { + boolean accessible = field.isAccessible(); + try { + field.setAccessible(true); + field.set(bean, instance); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } finally { + field.setAccessible(accessible); + } + } + } } + /** * Verify that the findById endpoint returns the own profile. * @@ -608,30 +638,38 @@ public void testDeleteWithProfileLinkedWithOrcid() throws Exception { .withOrcidAuthenticated("authenticated") .build(); + context.restoreAuthSystemState(); + String id = user.getID().toString(); String authToken = getAuthToken(user.getEmail(), password); + OrcidToken orcidToken = orcidTokenService.findByProfileItem(context, profileItem); - context.restoreAuthSystemState(); - - getClient(authToken).perform(get("/api/eperson/profiles/{id}", id)) + getClient(authToken) + .perform(get("/api/eperson/profiles/{id}", id)) .andExpect(status().isOk()); assertThat(profileItem.getMetadata(), hasItem(with("person.identifier.orcid", "0000-1111-2222-3333"))); assertThat(profileItem.getMetadata(), hasItem(with("dspace.orcid.authenticated", "authenticated"))); - assertThat(getOrcidAccessToken(profileItem), notNullValue()); + assertThat(orcidToken.getAccessToken(), notNullValue()); getClient(authToken).perform(get("/api/eperson/profiles/{id}/item", id)) .andExpect(status().isOk()) .andExpect(jsonPath("$", hasJsonPath("$.metadata", matchMetadataNotEmpty("dspace.object.owner")))); + getClient(authToken).perform(delete("/api/eperson/profiles/{id}", id)) .andExpect(status().isNoContent()); + verify(orcidClientMock, times(1)).revokeToken(matchesToken(orcidToken)); + verifyNoMoreInteractions(orcidClientMock); + profileItem = context.reloadEntity(profileItem); + orcidToken = orcidTokenService.findByProfileItem(context, profileItem); + assertThat(profileItem.getMetadata(), not(hasItem(with("person.identifier.orcid", "0000-1111-2222-3333")))); assertThat(profileItem.getMetadata(), not(hasItem(with("dspace.orcid.authenticated", "authenticated")))); - assertThat(getOrcidAccessToken(profileItem), nullValue()); + assertThat(orcidToken, nullValue()); } @@ -1850,7 +1888,8 @@ public void testOwnerPatchToDisconnectProfileFromOrcidWithOnlyOwnerConfiguration .withNameInMetadata("Test", "User") .build(); - OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); + OrcidToken orcidToken = + OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); Item profile = createProfile(ePerson); @@ -1872,6 +1911,9 @@ public void testOwnerPatchToDisconnectProfileFromOrcidWithOnlyOwnerConfiguration .andExpect(jsonPath("$.orcid").doesNotExist()) .andExpect(jsonPath("$.orcidSynchronization").doesNotExist()); + verify(orcidClientMock, times(1)).revokeToken(matchesToken(orcidToken)); + verifyNoMoreInteractions(orcidClientMock); + profile = context.reloadEntity(profile); assertThat(getMetadataValues(profile, "person.identifier.orcid"), empty()); @@ -1880,6 +1922,54 @@ public void testOwnerPatchToDisconnectProfileFromOrcidWithOnlyOwnerConfiguration assertThat(getOrcidAccessToken(profile), nullValue()); } + @Test + public void testPatchToDisconnectProfileFromOrcidDoesntRevokeOrcidToken() throws Exception { + + configurationService.setProperty("orcid.disconnection.allowed-users", "admin_and_owner"); + + context.turnOffAuthorisationSystem(); + + EPerson ePerson = EPersonBuilder.createEPerson(context) + .withCanLogin(true) + .withOrcid("0000-1111-2222-3333") + .withOrcidScope("/read") + .withOrcidScope("/write") + .withEmail("test@email.it") + .withPassword(password) + .withNameInMetadata("Test", "User") + .build(); + + OrcidToken orcidToken = + OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); + + Item profile = createProfile(ePerson); + + assertThat(getMetadataValues(profile, "person.identifier.orcid"), not(empty())); + assertThat(getMetadataValues(profile, "dspace.orcid.scope"), not(empty())); + assertThat(getMetadataValues(profile, "dspace.orcid.authenticated"), not(empty())); + assertThat(getOrcidAccessToken(profile), is("3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4")); + + context.restoreAuthSystemState(); + + doThrow(new OrcidClientException(403, "")).when(orcidClientMock).revokeToken(any(OrcidToken.class)); + + getClient(getAuthToken(ePerson.getEmail(), password)) + .perform(patch("/api/eperson/profiles/{id}", ePerson.getID().toString()) + .content(getPatchContent(asList(new RemoveOperation("/orcid")))) + .contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(status().isInternalServerError()); + + verify(orcidClientMock, times(1)).revokeToken(matchesToken(orcidToken)); + verifyNoMoreInteractions(orcidClientMock); + + profile = context.reloadEntity(profile); + + assertThat(getMetadataValues(profile, "person.identifier.orcid"), not(empty())); + assertThat(getMetadataValues(profile, "dspace.orcid.scope"), not(empty())); + assertThat(getMetadataValues(profile, "dspace.orcid.authenticated"), not(empty())); + assertThat(getOrcidAccessToken(profile), is("3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4")); + } + @Test public void testAdminPatchToDisconnectProfileFromOrcidWithOnlyOwnerConfiguration() throws Exception { @@ -2023,7 +2113,8 @@ public void testAdminPatchToDisconnectProfileFromOrcidWithOnlyAdminConfiguration .withNameInMetadata("Test", "User") .build(); - OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); + OrcidToken orcidToken = + OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); Item profile = createProfile(ePerson); @@ -2034,6 +2125,7 @@ public void testAdminPatchToDisconnectProfileFromOrcidWithOnlyAdminConfiguration context.restoreAuthSystemState(); + getClient(getAuthToken(admin.getEmail(), password)) .perform(patch("/api/eperson/profiles/{id}", ePerson.getID().toString()) .content(getPatchContent(asList(new RemoveOperation("/orcid")))) @@ -2045,6 +2137,9 @@ public void testAdminPatchToDisconnectProfileFromOrcidWithOnlyAdminConfiguration .andExpect(jsonPath("$.orcid").doesNotExist()) .andExpect(jsonPath("$.orcidSynchronization").doesNotExist()); + verify(orcidClientMock, times(1)).revokeToken(matchesToken(orcidToken)); + verifyNoMoreInteractions(orcidClientMock); + profile = context.reloadEntity(profile); assertThat(getMetadataValues(profile, "person.identifier.orcid"), empty()); @@ -2111,17 +2206,18 @@ public void testOwnerPatchToDisconnectProfileFromOrcidWithAdminAndOwnerConfigura .withPassword(password) .withNameInMetadata("Test", "User") .build(); - - OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); + OrcidToken orcidToken = + OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); Item profile = createProfile(ePerson); + context.restoreAuthSystemState(); + assertThat(getMetadataValues(profile, "person.identifier.orcid"), not(empty())); assertThat(getMetadataValues(profile, "dspace.orcid.scope"), not(empty())); assertThat(getMetadataValues(profile, "dspace.orcid.authenticated"), not(empty())); assertThat(getOrcidAccessToken(profile), is("3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4")); - context.restoreAuthSystemState(); getClient(getAuthToken(ePerson.getEmail(), password)) .perform(patch("/api/eperson/profiles/{id}", ePerson.getID().toString()) @@ -2134,6 +2230,9 @@ public void testOwnerPatchToDisconnectProfileFromOrcidWithAdminAndOwnerConfigura .andExpect(jsonPath("$.orcid").doesNotExist()) .andExpect(jsonPath("$.orcidSynchronization").doesNotExist()); + verify(orcidClientMock, times(1)).revokeToken(matchesToken(orcidToken)); + verifyNoMoreInteractions(orcidClientMock); + profile = context.reloadEntity(profile); assertThat(getMetadataValues(profile, "person.identifier.orcid"), empty()); @@ -2159,7 +2258,8 @@ public void testAdminPatchToDisconnectProfileFromOrcidWithAdminAndOwnerConfigura .withNameInMetadata("Test", "User") .build(); - OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); + OrcidToken orcidToken = + OrcidTokenBuilder.create(context, ePerson, "3de2e370-8aa9-4bbe-8d7e-f5b1577bdad4").build(); Item profile = createProfile(ePerson); @@ -2170,6 +2270,7 @@ public void testAdminPatchToDisconnectProfileFromOrcidWithAdminAndOwnerConfigura context.restoreAuthSystemState(); + getClient(getAuthToken(admin.getEmail(), password)) .perform(patch("/api/eperson/profiles/{id}", ePerson.getID().toString()) .content(getPatchContent(asList(new RemoveOperation("/orcid")))) @@ -2181,6 +2282,10 @@ public void testAdminPatchToDisconnectProfileFromOrcidWithAdminAndOwnerConfigura .andExpect(jsonPath("$.orcid").doesNotExist()) .andExpect(jsonPath("$.orcidSynchronization").doesNotExist()); + + verify(orcidClientMock, times(1)).revokeToken(matchesToken(orcidToken)); + verifyNoMoreInteractions(orcidClientMock); + profile = context.reloadEntity(profile); assertThat(getMetadataValues(profile, "person.identifier.orcid"), empty()); @@ -2627,4 +2732,13 @@ private OrcidTokenResponseDTO buildOrcidTokenResponse(String orcid, String acces return token; } + private static OrcidToken matchesToken(OrcidToken orcidToken) { + return argThat( + token -> + token != null && + orcidToken.getAccessToken().equals(token.getAccessToken()) && + orcidToken.getID().equals(token.getID()) + ); + } + } diff --git a/dspace/config/modules/orcid.cfg b/dspace/config/modules/orcid.cfg index cde819677447..93da0a5f4cb9 100644 --- a/dspace/config/modules/orcid.cfg +++ b/dspace/config/modules/orcid.cfg @@ -14,6 +14,7 @@ orcid.disconnection.allowed-users = admin_and_owner # ORCID API (https://github.com/ORCID/ORCID-Source/tree/master/orcid-api-web#endpoints) orcid.domain-url= https://sandbox.orcid.org orcid.authorize-url = ${orcid.domain-url}/oauth/authorize +orcid.revoke-url = ${orcid.domain-url}/oauth/revoke orcid.token-url = ${orcid.domain-url}/oauth/token orcid.api-url = https://api.sandbox.orcid.org/v3.0 orcid.public-url = https://pub.sandbox.orcid.org/v3.0 diff --git a/dspace/config/spring/api/orcid-services.xml b/dspace/config/spring/api/orcid-services.xml index e5fb002c314f..eb31acb29c4d 100644 --- a/dspace/config/spring/api/orcid-services.xml +++ b/dspace/config/spring/api/orcid-services.xml @@ -30,6 +30,7 @@ + From 26905b7d45f058f1bd8476f9e915af9a5be1547f Mon Sep 17 00:00:00 2001 From: Vincenzo Mecca Date: Tue, 21 Nov 2023 11:51:25 +0100 Subject: [PATCH 002/178] [CST-14901][DSC-1357][#8662] Handles versioning for ORCID publications. feat: - ORCID publications waiting to be published are removed whenever a new version is created - ORCID publications already published will be updated with the ref to the last item version - ORCID consumer will process only latest item versions, ignoring all the other ones --- .../org/dspace/content/ItemServiceImpl.java | 42 ++++ .../dspace/content/service/ItemService.java | 10 + .../orcid/consumer/OrcidQueueConsumer.java | 51 +++-- .../org/dspace/orcid/dao/OrcidQueueDAO.java | 10 + .../orcid/dao/impl/OrcidQueueDAOImpl.java | 7 + .../orcid/service/OrcidQueueService.java | 10 + .../service/impl/OrcidQueueServiceImpl.java | 5 + .../dspace/versioning/VersioningConsumer.java | 35 +++- .../dspace/orcid/OrcidQueueConsumerIT.java | 185 ++++++++++++++++++ 9 files changed, 337 insertions(+), 18 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java index ac6c0d43d710..ae7081f73a37 100644 --- a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java @@ -79,6 +79,9 @@ import org.dspace.orcid.service.OrcidTokenService; import org.dspace.profile.service.ResearcherProfileService; import org.dspace.services.ConfigurationService; +import org.dspace.versioning.Version; +import org.dspace.versioning.VersionHistory; +import org.dspace.versioning.service.VersionHistoryService; import org.dspace.versioning.service.VersioningService; import org.dspace.workflow.WorkflowItemService; import org.dspace.workflow.factory.WorkflowServiceFactory; @@ -171,6 +174,9 @@ public class ItemServiceImpl extends DSpaceObjectServiceImpl implements It @Autowired(required = true) protected SubscribeService subscribeService; + @Autowired + private VersionHistoryService versionHistoryService; + protected ItemServiceImpl() { super(); } @@ -1920,4 +1926,40 @@ private void deleteOrcidQueueRecords(Context context, Item item) throws SQLExcep } } + @Override + public boolean isLatestVersion(Context context, Item item) throws SQLException { + + VersionHistory history = versionHistoryService.findByItem(context, item); + if (history == null) { + // not all items have a version history + // if an item does not have a version history, it is by definition the latest + // version + return true; + } + + // start with the very latest version of the given item (may still be in + // workspace) + Version latestVersion = versionHistoryService.getLatestVersion(context, history); + + // find the latest version of the given item that is archived + while (latestVersion != null && !latestVersion.getItem().isArchived()) { + latestVersion = versionHistoryService.getPrevious(context, history, latestVersion); + } + + // could not find an archived version of the given item + if (latestVersion == null) { + // this scenario should never happen, but let's err on the side of showing too + // many items vs. to little + // (see discovery.xml, a lot of discovery configs filter out all items that are + // not the latest version) + return true; + } + + // sanity check + assert latestVersion.getItem().isArchived(); + + return item.equals(latestVersion.getItem()); + + } + } diff --git a/dspace-api/src/main/java/org/dspace/content/service/ItemService.java b/dspace-api/src/main/java/org/dspace/content/service/ItemService.java index 12867ad18c3f..a307e7ceffad 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/ItemService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/ItemService.java @@ -983,4 +983,14 @@ public List getMetadata(Item item, String schema, String element, */ public EntityType getEntityType(Context context, Item item) throws SQLException; + + /** + * Check whether the given item is the latest version. If the latest item cannot + * be determined, because either the version history or the latest version is + * not present, assume the item is latest. + * @param context the DSpace context. + * @param item the item that should be checked. + * @return true if the item is the latest version, false otherwise. + */ + public boolean isLatestVersion(Context context, Item item) throws SQLException; } diff --git a/dspace-api/src/main/java/org/dspace/orcid/consumer/OrcidQueueConsumer.java b/dspace-api/src/main/java/org/dspace/orcid/consumer/OrcidQueueConsumer.java index d177e61607f1..ae989e1dd8f8 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/consumer/OrcidQueueConsumer.java +++ b/dspace-api/src/main/java/org/dspace/orcid/consumer/OrcidQueueConsumer.java @@ -14,9 +14,10 @@ import static org.apache.commons.collections.CollectionUtils.isNotEmpty; import java.sql.SQLException; -import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -56,7 +57,7 @@ * be synchronized (based on the preferences set by the user) *
  • are publications/fundings related to profile items linked to orcid (based * on the preferences set by the user)
    • - * + * * * * @author Luca Giamminonni (luca.giamminonni at 4science.it) @@ -82,7 +83,7 @@ public class OrcidQueueConsumer implements Consumer { private RelationshipService relationshipService; - private List alreadyConsumedItems = new ArrayList<>(); + private final Set itemsToConsume = new HashSet<>(); @Override public void initialize() throws Exception { @@ -117,17 +118,26 @@ public void consume(Context context, Event event) throws Exception { return; } - if (alreadyConsumedItems.contains(item.getID())) { - return; - } + itemsToConsume.add(item.getID()); + } + + @Override + public void end(Context context) throws Exception { + + for (UUID itemId : itemsToConsume) { + + Item item = itemService.find(context, itemId); + + context.turnOffAuthorisationSystem(); + try { + consumeItem(context, item); + } finally { + context.restoreAuthSystemState(); + } - context.turnOffAuthorisationSystem(); - try { - consumeItem(context, item); - } finally { - context.restoreAuthSystemState(); } + itemsToConsume.clear(); } /** @@ -146,7 +156,7 @@ private void consumeItem(Context context, Item item) throws SQLException { consumeProfile(context, item); } - alreadyConsumedItems.add(item.getID()); + itemsToConsume.add(item.getID()); } @@ -169,6 +179,10 @@ private void consumeEntity(Context context, Item entity) throws SQLException { continue; } + if (isNotLatestVersion(context, entity)) { + continue; + } + orcidQueueService.create(context, relatedItem, entity); } @@ -329,6 +343,14 @@ private boolean isNotProfileItem(Item profileItemItem) { return !getProfileType().equals(itemService.getEntityTypeLabel(profileItemItem)); } + private boolean isNotLatestVersion(Context context, Item entity) { + try { + return !itemService.isLatestVersion(context, entity); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + private String getMetadataValue(Item item, String metadataField) { return itemService.getMetadataFirstValue(item, new MetadataFieldName(metadataField), Item.ANY); } @@ -345,11 +367,6 @@ private boolean isOrcidSynchronizationDisabled() { return !configurationService.getBooleanProperty("orcid.synchronization-enabled", true); } - @Override - public void end(Context context) throws Exception { - alreadyConsumedItems.clear(); - } - @Override public void finish(Context context) throws Exception { // nothing to do diff --git a/dspace-api/src/main/java/org/dspace/orcid/dao/OrcidQueueDAO.java b/dspace-api/src/main/java/org/dspace/orcid/dao/OrcidQueueDAO.java index 235443b15033..b7e0b1ed2a85 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/dao/OrcidQueueDAO.java +++ b/dspace-api/src/main/java/org/dspace/orcid/dao/OrcidQueueDAO.java @@ -74,6 +74,16 @@ public List findByProfileItemAndEntity(Context context, Item profile */ public List findByProfileItemOrEntity(Context context, Item item) throws SQLException; + /** + * Get the OrcidQueue records where the given item is the entity. + * + * @param context DSpace context object + * @param item the item to search for + * @return the found OrcidQueue entities + * @throws SQLException if database error + */ + public List findByEntity(Context context, Item item) throws SQLException; + /** * Find all the OrcidQueue records with the given entity and record type. * diff --git a/dspace-api/src/main/java/org/dspace/orcid/dao/impl/OrcidQueueDAOImpl.java b/dspace-api/src/main/java/org/dspace/orcid/dao/impl/OrcidQueueDAOImpl.java index 2114b2535759..8e941b056535 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/dao/impl/OrcidQueueDAOImpl.java +++ b/dspace-api/src/main/java/org/dspace/orcid/dao/impl/OrcidQueueDAOImpl.java @@ -63,6 +63,13 @@ public List findByProfileItemOrEntity(Context context, Item item) th return query.getResultList(); } + @Override + public List findByEntity(Context context, Item item) throws SQLException { + Query query = createQuery(context, "FROM OrcidQueue WHERE entity.id = :itemId"); + query.setParameter("itemId", item.getID()); + return query.getResultList(); + } + @Override public List findByEntityAndRecordType(Context context, Item entity, String type) throws SQLException { Query query = createQuery(context, "FROM OrcidQueue WHERE entity = :entity AND recordType = :type"); diff --git a/dspace-api/src/main/java/org/dspace/orcid/service/OrcidQueueService.java b/dspace-api/src/main/java/org/dspace/orcid/service/OrcidQueueService.java index 8de25e9caf1e..b667088eabb4 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/service/OrcidQueueService.java +++ b/dspace-api/src/main/java/org/dspace/orcid/service/OrcidQueueService.java @@ -164,6 +164,16 @@ public List findByProfileItemAndEntity(Context context, Item profile */ public List findByProfileItemOrEntity(Context context, Item item) throws SQLException; + /** + * Get the OrcidQueue records where the given item is the entity. + * + * @param context DSpace context object + * @param item the item to search for + * @return the found OrcidQueue records + * @throws SQLException if database error + */ + public List findByEntity(Context context, Item item) throws SQLException; + /** * Get all the OrcidQueue records with attempts less than the given attempts. * diff --git a/dspace-api/src/main/java/org/dspace/orcid/service/impl/OrcidQueueServiceImpl.java b/dspace-api/src/main/java/org/dspace/orcid/service/impl/OrcidQueueServiceImpl.java index d3300fea6606..261f8ef9a9f7 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/service/impl/OrcidQueueServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/orcid/service/impl/OrcidQueueServiceImpl.java @@ -70,6 +70,11 @@ public List findByProfileItemOrEntity(Context context, Item item) th return orcidQueueDAO.findByProfileItemOrEntity(context, item); } + @Override + public List findByEntity(Context context, Item item) throws SQLException { + return orcidQueueDAO.findByEntity(context, item); + } + @Override public long countByProfileItemId(Context context, UUID profileItemId) throws SQLException { return orcidQueueDAO.countByProfileItemId(context, profileItemId); diff --git a/dspace-api/src/main/java/org/dspace/versioning/VersioningConsumer.java b/dspace-api/src/main/java/org/dspace/versioning/VersioningConsumer.java index 63b5391d0a28..27a81a157917 100644 --- a/dspace-api/src/main/java/org/dspace/versioning/VersioningConsumer.java +++ b/dspace-api/src/main/java/org/dspace/versioning/VersioningConsumer.java @@ -33,6 +33,11 @@ import org.dspace.discovery.IndexEventConsumer; import org.dspace.event.Consumer; import org.dspace.event.Event; +import org.dspace.orcid.OrcidHistory; +import org.dspace.orcid.OrcidQueue; +import org.dspace.orcid.factory.OrcidServiceFactory; +import org.dspace.orcid.service.OrcidHistoryService; +import org.dspace.orcid.service.OrcidQueueService; import org.dspace.versioning.factory.VersionServiceFactory; import org.dspace.versioning.service.VersionHistoryService; import org.dspace.versioning.utils.RelationshipVersioningUtils; @@ -58,6 +63,8 @@ public class VersioningConsumer implements Consumer { private RelationshipTypeService relationshipTypeService; private RelationshipService relationshipService; private RelationshipVersioningUtils relationshipVersioningUtils; + private OrcidQueueService orcidQueueService; + private OrcidHistoryService orcidHistoryService; @Override public void initialize() throws Exception { @@ -67,6 +74,8 @@ public void initialize() throws Exception { relationshipTypeService = ContentServiceFactory.getInstance().getRelationshipTypeService(); relationshipService = ContentServiceFactory.getInstance().getRelationshipService(); relationshipVersioningUtils = VersionServiceFactory.getInstance().getRelationshipVersioningUtils(); + this.orcidQueueService = OrcidServiceFactory.getInstance().getOrcidQueueService(); + this.orcidHistoryService = OrcidServiceFactory.getInstance().getOrcidHistoryService(); } @Override @@ -132,7 +141,8 @@ public void consume(Context ctx, Event event) throws Exception { // unarchive previous item unarchiveItem(ctx, previousItem); - + // handles versions for ORCID publications waiting to be shipped, or already published (history-queue). + handleOrcidSynchronization(ctx, previousItem, latestItem); // update relationships updateRelationships(ctx, latestItem, previousItem); } @@ -148,6 +158,29 @@ protected void unarchiveItem(Context ctx, Item item) { )); } + private void handleOrcidSynchronization(Context ctx, Item previousItem, Item latestItem) { + try { + replaceOrcidHistoryEntities(ctx, previousItem, latestItem); + removeOrcidQueueEntries(ctx, previousItem); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + private void removeOrcidQueueEntries(Context ctx, Item previousItem) throws SQLException { + List queueEntries = orcidQueueService.findByEntity(ctx, previousItem); + for (OrcidQueue queueEntry : queueEntries) { + orcidQueueService.delete(ctx, queueEntry); + } + } + + private void replaceOrcidHistoryEntities(Context ctx, Item previousItem, Item latestItem) throws SQLException { + List entries = orcidHistoryService.findByEntity(ctx, previousItem); + for (OrcidHistory entry : entries) { + entry.setEntity(latestItem); + } + } + /** * Update {@link Relationship#latestVersionStatus} of the relationships of both the old version and the new version * of the item. diff --git a/dspace-api/src/test/java/org/dspace/orcid/OrcidQueueConsumerIT.java b/dspace-api/src/test/java/org/dspace/orcid/OrcidQueueConsumerIT.java index f2e528d78cd6..e17fd0072efa 100644 --- a/dspace-api/src/test/java/org/dspace/orcid/OrcidQueueConsumerIT.java +++ b/dspace-api/src/test/java/org/dspace/orcid/OrcidQueueConsumerIT.java @@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import java.sql.SQLException; import java.time.Instant; @@ -41,13 +42,19 @@ import org.dspace.content.Item; import org.dspace.content.MetadataValue; import org.dspace.content.RelationshipType; +import org.dspace.content.WorkspaceItem; import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.InstallItemService; import org.dspace.content.service.ItemService; +import org.dspace.content.service.WorkspaceItemService; import org.dspace.orcid.consumer.OrcidQueueConsumer; import org.dspace.orcid.factory.OrcidServiceFactory; import org.dspace.orcid.service.OrcidQueueService; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; +import org.dspace.utils.DSpace; +import org.dspace.versioning.Version; +import org.dspace.versioning.service.VersioningService; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -64,8 +71,15 @@ public class OrcidQueueConsumerIT extends AbstractIntegrationTestWithDatabase { private ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + private WorkspaceItemService workspaceItemService = ContentServiceFactory.getInstance().getWorkspaceItemService(); + + private InstallItemService installItemService = ContentServiceFactory.getInstance().getInstallItemService(); + private ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + private VersioningService versioningService = new DSpace().getServiceManager() + .getServicesByType(VersioningService.class).get(0); + private Collection profileCollection; @Before @@ -763,6 +777,177 @@ public void testWithManyInsertionAndDeletionOfSameMetadataValue() throws Excepti } + @Test + public void testOrcidQueueRecordCreationForPublicationWithNotFoundAuthority() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item profile = ItemBuilder.createItem(context, profileCollection) + .withTitle("Test User") + .withOrcidIdentifier("0000-1111-2222-3333") + .withOrcidAccessToken("ab4d18a0-8d9a-40f1-b601-a417255c8d20", eperson) + .withOrcidSynchronizationPublicationsPreference(ALL) + .build(); + + Collection publicationCollection = createCollection("Publications", "Publication"); + + Item publication = ItemBuilder.createItem(context, publicationCollection) + .withTitle("Test publication") + .withAuthor("First User") + .withAuthor("Test User") + .build(); + + EntityType publicationType = EntityTypeBuilder.createEntityTypeBuilder(context, "Publication").build(); + EntityType personType = EntityTypeBuilder.createEntityTypeBuilder(context, "Person").build(); + + RelationshipType isAuthorOfPublication = createRelationshipTypeBuilder(context, personType, publicationType, + "isAuthorOfPublication", + "isPublicationOfAuthor", 0, null, 0, + null).build(); + + RelationshipBuilder.createRelationshipBuilder(context, profile, publication, isAuthorOfPublication).build(); + + context.restoreAuthSystemState(); + context.commit(); + + List orcidQueueRecords = orcidQueueService.findAll(context); + assertThat(orcidQueueRecords, hasSize(1)); + assertThat(orcidQueueRecords.get(0), matches(profile, publication, "Publication", INSERT)); + } + + @Test + public void testOrcidQueueWithItemVersioning() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item profile = ItemBuilder.createItem(context, profileCollection) + .withTitle("Test User") + .withOrcidIdentifier("0000-1111-2222-3333") + .withOrcidAccessToken("ab4d18a0-8d9a-40f1-b601-a417255c8d20", eperson) + .withOrcidSynchronizationPublicationsPreference(ALL) + .build(); + + Collection publicationCollection = createCollection("Publications", "Publication"); + + Item publication = ItemBuilder.createItem(context, publicationCollection) + .withTitle("Test publication") + .withAuthor("Test User") + .build(); + + EntityType publicationType = EntityTypeBuilder.createEntityTypeBuilder(context, "Publication").build(); + EntityType personType = EntityTypeBuilder.createEntityTypeBuilder(context, "Person").build(); + + RelationshipType isAuthorOfPublication = createRelationshipTypeBuilder(context, personType, publicationType, + "isAuthorOfPublication", + "isPublicationOfAuthor", 0, null, 0, + null).build(); + + RelationshipBuilder.createRelationshipBuilder(context, profile, publication, isAuthorOfPublication).build(); + + context.restoreAuthSystemState(); + context.commit(); + + List orcidQueueRecords = orcidQueueService.findAll(context); + assertThat(orcidQueueRecords, hasSize(1)); + assertThat(orcidQueueRecords.get(0), matches(profile, publication, "Publication", INSERT)); + + context.turnOffAuthorisationSystem(); + Version newVersion = versioningService.createNewVersion(context, publication); + context.restoreAuthSystemState(); + Item newPublication = newVersion.getItem(); + assertThat(newPublication.isArchived(), is(false)); + + context.commit(); + + orcidQueueRecords = orcidQueueService.findAll(context); + assertThat(orcidQueueRecords, hasSize(1)); + assertThat(orcidQueueRecords.get(0), matches(profile, publication, "Publication", INSERT)); + + WorkspaceItem workspaceItem = workspaceItemService.findByItem(context, newVersion.getItem()); + context.turnOffAuthorisationSystem(); + + installItemService.installItem(context, workspaceItem); + + context.restoreAuthSystemState(); + context.commit(); + + orcidQueueRecords = orcidQueueService.findAll(context); + assertThat(orcidQueueRecords, hasSize(1)); + assertThat(orcidQueueRecords.get(0), matches(profile, newPublication, "Publication", INSERT)); + } + + @Test + public void testOrcidQueueUpdateWithItemVersioning() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item profile = ItemBuilder.createItem(context, profileCollection) + .withTitle("Test User") + .withOrcidIdentifier("0000-1111-2222-3333") + .withOrcidAccessToken("ab4d18a0-8d9a-40f1-b601-a417255c8d20", eperson) + .withOrcidSynchronizationPublicationsPreference(ALL) + .build(); + + Collection publicationCollection = createCollection("Publications", "Publication"); + + Item publication = ItemBuilder.createItem(context, publicationCollection) + .withTitle("Test publication") + .build(); + + OrcidHistory orcidHistory = OrcidHistoryBuilder.createOrcidHistory(context, profile, publication) + .withDescription("Test publication") + .withOperation(OrcidOperation.INSERT) + .withPutCode("12345") + .withStatus(201) + .build(); + + addMetadata(publication, "dc", "contributor", "author", "Test User", null); + + EntityType publicationType = EntityTypeBuilder.createEntityTypeBuilder(context, "Publication").build(); + EntityType personType = EntityTypeBuilder.createEntityTypeBuilder(context, "Person").build(); + + RelationshipType isAuthorOfPublication = + createRelationshipTypeBuilder( + context, personType, publicationType, + "isAuthorOfPublication", + "isPublicationOfAuthor", 0, null, 0, + null + ).build(); + + RelationshipBuilder.createRelationshipBuilder(context, profile, publication, isAuthorOfPublication).build(); + + context.commit(); + + List orcidQueueRecords = orcidQueueService.findAll(context); + assertThat(orcidQueueRecords, hasSize(1)); + assertThat(orcidQueueRecords.get(0), matches(profile, publication, "Publication", "12345", UPDATE)); + + Version newVersion = versioningService.createNewVersion(context, publication); + Item newPublication = newVersion.getItem(); + assertThat(newPublication.isArchived(), is(false)); + + context.commit(); + + orcidQueueRecords = orcidQueueService.findAll(context); + assertThat(orcidQueueRecords, hasSize(1)); + assertThat(orcidQueueRecords.get(0), matches(profile, publication, "Publication", "12345", UPDATE)); + + WorkspaceItem workspaceItem = workspaceItemService.findByItem(context, newVersion.getItem()); + installItemService.installItem(context, workspaceItem); + + context.commit(); + + context.restoreAuthSystemState(); + + orcidQueueRecords = orcidQueueService.findAll(context); + assertThat(orcidQueueRecords, hasSize(1)); + assertThat(orcidQueueRecords.get(0), matches(profile, newPublication, "Publication", "12345", UPDATE)); + + orcidHistory = context.reloadEntity(orcidHistory); + assertThat(orcidHistory.getEntity(), is(newPublication)); + + } + private void addMetadata(Item item, String schema, String element, String qualifier, String value, String authority) throws Exception { context.turnOffAuthorisationSystem(); From 405397b8b061205253768db73924aeffa77c846d Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Wed, 3 Jul 2024 13:23:32 +0200 Subject: [PATCH 003/178] update eperson's attributes right after successful login (cherry picked from commit 428489ca5258ea7ac6942a722621e22c3371bfcf) --- .../authenticate/LDAPAuthentication.java | 69 +++++++++++-------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java b/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java index 585eaf9cd8b1..2203937e75d6 100644 --- a/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java +++ b/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java @@ -68,12 +68,8 @@ * @author Ivan Masár * @author Michael Plate */ -public class LDAPAuthentication - implements AuthenticationMethod { +public class LDAPAuthentication implements AuthenticationMethod { - /** - * log4j category - */ private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(LDAPAuthentication.class); @@ -130,7 +126,7 @@ public boolean allowSetPassword(Context context, return false; } - /* + /** * This is an explicit method. */ @Override @@ -138,7 +134,7 @@ public boolean isImplicit() { return false; } - /* + /** * Add authenticated users to the group defined in dspace.cfg by * the login.specialgroup key. */ @@ -177,7 +173,7 @@ public List getSpecialGroups(Context context, HttpServletRequest request) return Collections.EMPTY_LIST; } - /* + /** * Authenticate the given credentials. * This is the heart of the authentication method: test the * credentials for authenticity, and if accepted, attempt to match @@ -250,7 +246,7 @@ public int authenticate(Context context, } // Check a DN was found - if ((dn == null) || (dn.trim().equals(""))) { + if (StringUtils.isBlank(dn)) { log.info(LogHelper .getHeader(context, "failed_login", "no DN found for user " + netid)); return BAD_CREDENTIALS; @@ -269,6 +265,18 @@ public int authenticate(Context context, context.setCurrentUser(eperson); request.setAttribute(LDAP_AUTHENTICATED, true); + // update eperson's attributes + context.turnOffAuthorisationSystem(); + setEpersonAttributes(context, eperson, ldap, Optional.empty()); + try { + ePersonService.update(context, eperson); + context.dispatchEvents(); + } catch (AuthorizeException e) { + log.warn("update of eperson " + eperson.getID() + " failed", e); + } finally { + context.restoreAuthSystemState(); + } + // assign user to groups based on ldap dn assignGroups(dn, ldap.ldapGroup, context); @@ -313,14 +321,13 @@ public int authenticate(Context context, log.info(LogHelper.getHeader(context, "type=ldap-login", "type=ldap_but_already_email")); context.turnOffAuthorisationSystem(); - eperson.setNetid(netid.toLowerCase()); + setEpersonAttributes(context, eperson, ldap, Optional.of(netid)); ePersonService.update(context, eperson); context.dispatchEvents(); context.restoreAuthSystemState(); context.setCurrentUser(eperson); request.setAttribute(LDAP_AUTHENTICATED, true); - // assign user to groups based on ldap dn assignGroups(dn, ldap.ldapGroup, context); @@ -331,20 +338,7 @@ public int authenticate(Context context, try { context.turnOffAuthorisationSystem(); eperson = ePersonService.create(context); - if (StringUtils.isNotEmpty(email)) { - eperson.setEmail(email); - } - if (StringUtils.isNotEmpty(ldap.ldapGivenName)) { - eperson.setFirstName(context, ldap.ldapGivenName); - } - if (StringUtils.isNotEmpty(ldap.ldapSurname)) { - eperson.setLastName(context, ldap.ldapSurname); - } - if (StringUtils.isNotEmpty(ldap.ldapPhone)) { - ePersonService.setMetadataSingleValue(context, eperson, - MD_PHONE, ldap.ldapPhone, null); - } - eperson.setNetid(netid.toLowerCase()); + setEpersonAttributes(context, eperson, ldap, Optional.of(netid)); eperson.setCanLogIn(true); authenticationService.initEPerson(context, request, eperson); ePersonService.update(context, eperson); @@ -382,6 +376,27 @@ public int authenticate(Context context, return BAD_ARGS; } + /** + * Update eperson's attributes + */ + private void setEpersonAttributes(Context context, EPerson eperson, SpeakerToLDAP ldap, Optional netid) throws SQLException { + if (StringUtils.isNotEmpty(ldap.ldapEmail)) { + eperson.setEmail(ldap.ldapEmail); + } + if (StringUtils.isNotEmpty(ldap.ldapGivenName)) { + eperson.setFirstName(context, ldap.ldapGivenName); + } + if (StringUtils.isNotEmpty(ldap.ldapSurname)) { + eperson.setLastName(context, ldap.ldapSurname); + } + if (StringUtils.isNotEmpty(ldap.ldapPhone)) { + ePersonService.setMetadataSingleValue(context, eperson, MD_PHONE, ldap.ldapPhone, null); + } + if (netid.isPresent()) { + eperson.setNetid(netid.get().toLowerCase()); + } + } + /** * Internal class to manage LDAP query and results, mainly * because there are multiple values to return. @@ -671,7 +686,7 @@ protected boolean ldapAuthenticate(String netid, String password, } } - /* + /** * Returns the URL of an external login page which is not applicable for this authn method. * * Note: Prior to DSpace 7, this method return the page of login servlet. @@ -699,7 +714,7 @@ public String getName() { return "ldap"; } - /* + /** * Add authenticated users to the group defined in dspace.cfg by * the authentication-ldap.login.groupmap.* key. * From b0370a064b843db6d77efb58c5d54788e3872253 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Wed, 3 Jul 2024 13:36:45 +0200 Subject: [PATCH 004/178] add missing import (cherry picked from commit c5ad32a9b3ece7e64043f9f39d22b594e913737f) --- .../main/java/org/dspace/authenticate/LDAPAuthentication.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java b/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java index 2203937e75d6..06bd6ae603d9 100644 --- a/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java +++ b/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java @@ -17,6 +17,7 @@ import java.util.Hashtable; import java.util.Iterator; import java.util.List; +import java.util.Optional; import javax.naming.NamingEnumeration; import javax.naming.NamingException; import javax.naming.directory.Attribute; From 31af49ea725b5658f408e519cc577805b8f8c72d Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Wed, 3 Jul 2024 13:54:53 +0200 Subject: [PATCH 005/178] fix Checkstyle violations (cherry picked from commit aaa74b88c99af8ece67d43fa412a39a7406fe10c) --- .../org/dspace/authenticate/LDAPAuthentication.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java b/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java index 06bd6ae603d9..93f0feb384c0 100644 --- a/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java +++ b/dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java @@ -184,7 +184,7 @@ public List getSpecialGroups(Context context, HttpServletRequest request) * @param context * DSpace context, will be modified (ePerson set) upon success. * - * @param username + * @param netid * Username (or email address) when method is explicit. Use null for * implicit method. * @@ -322,7 +322,7 @@ public int authenticate(Context context, log.info(LogHelper.getHeader(context, "type=ldap-login", "type=ldap_but_already_email")); context.turnOffAuthorisationSystem(); - setEpersonAttributes(context, eperson, ldap, Optional.of(netid)); + setEpersonAttributes(context, eperson, ldap, Optional.of(netid)); ePersonService.update(context, eperson); context.dispatchEvents(); context.restoreAuthSystemState(); @@ -380,7 +380,9 @@ public int authenticate(Context context, /** * Update eperson's attributes */ - private void setEpersonAttributes(Context context, EPerson eperson, SpeakerToLDAP ldap, Optional netid) throws SQLException { + private void setEpersonAttributes(Context context, EPerson eperson, SpeakerToLDAP ldap, Optional netid) + throws SQLException { + if (StringUtils.isNotEmpty(ldap.ldapEmail)) { eperson.setEmail(ldap.ldapEmail); } @@ -389,7 +391,7 @@ private void setEpersonAttributes(Context context, EPerson eperson, SpeakerToLDA } if (StringUtils.isNotEmpty(ldap.ldapSurname)) { eperson.setLastName(context, ldap.ldapSurname); - } + } if (StringUtils.isNotEmpty(ldap.ldapPhone)) { ePersonService.setMetadataSingleValue(context, eperson, MD_PHONE, ldap.ldapPhone, null); } From 702ff9cf4c0e9e2afeb1bab820087e6fb7584154 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Fri, 27 Sep 2024 12:07:52 +0200 Subject: [PATCH 006/178] minor fix in parameter description (cherry picked from commit 5758d9e90302d4da33ec869b6ec656e5c25b865a) --- dspace/config/emails/subscriptions_content | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace/config/emails/subscriptions_content b/dspace/config/emails/subscriptions_content index 9b8c91e559df..a1b107e8fc50 100644 --- a/dspace/config/emails/subscriptions_content +++ b/dspace/config/emails/subscriptions_content @@ -1,7 +1,7 @@ ## E-mail sent to designated address about updates on subscribed items ## -## Parameters: {0} Collections updates -## {1} Communities updates +## Parameters: {0} Communities updates +## {1} Collections updates #set($subject = "${config.get('dspace.name')} Subscriptions") This email is sent from ${config.get('dspace.name')} based on the chosen subscription preferences. From a721f8c41e92762d6d1af831b337b111a5ea9991 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Tue, 27 Aug 2024 10:33:01 +0200 Subject: [PATCH 007/178] fix failed first login attempt in HAL browser (cherry picked from commit 002e637d4fc69b31c2a2321230d96534e49ee33e) --- .../src/main/webapp/login.html | 128 ++++++++++-------- 1 file changed, 68 insertions(+), 60 deletions(-) diff --git a/dspace-server-webapp/src/main/webapp/login.html b/dspace-server-webapp/src/main/webapp/login.html index 50d95b80472f..0597a66bfe96 100644 --- a/dspace-server-webapp/src/main/webapp/login.html +++ b/dspace-server-webapp/src/main/webapp/login.html @@ -32,7 +32,7 @@ border-radius: 5px; box-shadow: 0 1px 2px rgba(0, 0, 0, .05); } - .form-signin .form-signin-heading, .form-signin .checkbox { + .form-signin .form-signin-heading, .form-signin { margin-bottom: 10px; } .form-signin input[type="text"], .form-signin input[type="password"] { @@ -94,63 +94,71 @@

    Other login methods:

    "onclick" : function() { toastr.remove(); } } + // retrieves a valid CSRF token (please note that this method works both in DS 7 and DS 8) + // HTTP response code 403 is expected at this point (the response contains the DSPACE-XSRF-TOKEN header) + $.ajax({ + url : window.location.href.replace("login.html", "") + 'api/authn/login', + type : 'POST', + error : function(xhr) { + // Check for an update to the CSRF Token & save to a MyHalBrowserCsrfToken cookie (if found) + checkForUpdatedCSRFTokenInResponse(xhr); + } + }); + // When the login page loads, we do *two* AJAX requests. - // (1) Call GET /api/authn/status. This call has two purposes. First, it checks to see if you are logged in, - // (if not, WWW-Authenticate will return login options). Second, it retrieves the CSRF token, if a - // new one has been assigned (as a valid CSRF token is required for the POST call). + // (1) Call GET /api/authn/status. This call checks to see if you are logged in + // (if not, WWW-Authenticate will return login options). // (2) If that /api/authn/status call finds authentication data, call POST /api/authn/login. - // This scenario occurs when you login via an external authentication system (e.g. Shibboleth)... + // This scenario occurs when you log in via an external authentication system (e.g. Shibboleth) // in which case the main role of /api/authn/login is to simply ensure the "Authorization" header // is sent back to the client (based on your authentication data). $.ajax({ - url : window.location.href.replace("login.html", "") + 'api/authn/status', - type : 'GET', - success : function(result, status, xhr) { - // Check for an update to the CSRF Token & save to a MyHalBrowserCsrfToken cookie (if found) - checkForUpdatedCSRFTokenInResponse(xhr); + url : window.location.href.replace("login.html", "") + 'api/authn/status', + type : 'GET', + success : function(result, status, xhr) { - // Check for WWW-Authenticate header. If found, this means we are not yet authenticated, and - // therefore we need to display available authentication options. - var authenticate = xhr.getResponseHeader("WWW-Authenticate"); - if (authenticate !== null) { - var element = $('div.other-login-methods'); - var realms = authenticate.match(/(\w+ (\w+=((".*?")|[^,]*)(, )?)*)/g); - if (realms.length == 1){ - var loc = /location="([^,]*)"/.exec(authenticate); - if (loc !== null && loc.length === 2) { - document.location = loc[1]; - } - } else if (realms.length > 1){ - for (var i = 0; i < realms.length; i++){ - addLocationButton(realms[i], element); + // Check for WWW-Authenticate header. If found, this means we are not yet authenticated, and + // therefore we need to display available authentication options. + var authenticate = xhr.getResponseHeader("WWW-Authenticate"); + if (authenticate !== null && authenticate.contains('location=')) { + var element = $('div.other-login-methods'); + var realms = authenticate.match(/(\w+ (\w+=((".*?")|[^,]*)(, )?)*)/g); + if (realms.length === 1){ + var loc = /location="([^,]*)"/.exec(authenticate); + if (loc !== null && loc.length === 2) { + document.location = loc[1]; + } + } else if (realms.length > 1){ + for (var i = 0; i < realms.length; i++){ + addLocationButton(realms[i], element); + } } + } else { + // If Authentication data was found, do a POST /api/authn/login to ensure that data's JWT + // is sent back in the "Authorization" header. This simply completes an external authentication + // process (e.g. Shibboleth) + $.ajax({ + url : window.location.href.replace("login.html", "") + 'api/authn/login', + type : 'POST', + beforeSend: function (xhr) { + // If CSRF token found in cookie, send it back as X-XSRF-Token header + var csrfToken = getCSRFToken(); + if (csrfToken != null) { + xhr.setRequestHeader('X-XSRF-Token', csrfToken); + } + }, + success : successHandler, + error : function(xhr) { + // Check for an update to the CSRF Token & save to a MyHalBrowserCsrfToken cookie (if found) + checkForUpdatedCSRFTokenInResponse(xhr); + toastr.error('Failed to logged in. Please check for errors in Javascript console.', 'Login Failed'); + } + }); } - } else { - // If Authentication data was found, do a POST /api/authn/login to ensure that data's JWT - // is sent back in the "Authorization" header. This simply completes an external authentication - // process (e.g. Shibboleth) - $.ajax({ - url : window.location.href.replace("login.html", "") + 'api/authn/login', - type : 'POST', - beforeSend: function (xhr, settings) { - // If CSRF token found in cookie, send it back as X-XSRF-Token header - var csrfToken = getCSRFToken(); - if (csrfToken != null) { - xhr.setRequestHeader('X-XSRF-Token', csrfToken); - } - }, - success : successHandler, - error : function(xhr, textStatus, errorThrown) { - // Check for an update to the CSRF Token & save to a MyHalBrowserCsrfToken cookie (if found) - checkForUpdatedCSRFTokenInResponse(xhr); - toastr.error('Failed to logged in. Please check for errors in Javascript console.', 'Login Failed'); - } - }); + }, + error : function() { + toastr.error('Failed to connect with backend. Please check for errors in Javascript console.', 'Could Not Load'); } - }, - error : function(xhr, textStatus, errorThrown) { - toastr.error('Failed to connect with backend. Please check for errors in Javascript console.', 'Could Not Load'); - } }); function addLocationButton(realm, element){ @@ -166,22 +174,22 @@

    Other login methods:

    return string.charAt(0).toUpperCase() + string.slice(1); } - /** - * Check current response headers to see if the CSRF Token has changed. If a new value is found in headers, - * save the new value into our "MyHalBrowserCsrfToken" cookie. - **/ + /** + * Check current response headers to see if the CSRF Token has changed. If a new value is found in headers, + * save the new value into our "MyHalBrowserCsrfToken" cookie. + **/ function checkForUpdatedCSRFTokenInResponse(jqxhr) { // look for DSpace-XSRF-TOKEN header & save to our MyHalBrowserCsrfToken cookie (if found) var updatedCsrfToken = jqxhr.getResponseHeader('DSPACE-XSRF-TOKEN'); if (updatedCsrfToken != null) { - document.cookie = "MyHalBrowserCsrfToken=" + updatedCsrfToken; + document.cookie = "MyHalBrowserCsrfToken=" + updatedCsrfToken; } } - /** - * Get CSRF Token by parsing it out of the "MyHalBrowserCsrfToken" cookie. - * This cookie is set in login.html after a successful login occurs. - **/ + /** + * Get CSRF Token by parsing it out of the "MyHalBrowserCsrfToken" cookie. + * This cookie is set in login.html after a successful login occurs. + **/ function getCSRFToken() { var cookie = document.cookie.match('(^|;)\\s*' + 'MyHalBrowserCsrfToken' + '\\s*=\\s*([^;]+)'); if (cookie != null) { @@ -204,11 +212,11 @@

    Other login methods:

    user: $("#username").val(), password: $("#password").val() }, - beforeSend: function (xhr, settings) { + beforeSend: function (xhr) { // If CSRF token found in cookie, send it back as X-XSRF-Token header var csrfToken = getCSRFToken(); if (csrfToken != null) { - xhr.setRequestHeader('X-XSRF-Token', csrfToken); + xhr.setRequestHeader('X-XSRF-Token', csrfToken); } }, success : successHandler, From 4669d8f7605a980a3f038c6f10835f89b7d81bee Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Thu, 29 Aug 2024 16:24:38 +0200 Subject: [PATCH 008/178] applied change suggested by reviewer: use String.prototype.includes (cherry picked from commit 546afb189e1df8bdfbc9948e790004baa81ec894) --- dspace-server-webapp/src/main/webapp/login.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/main/webapp/login.html b/dspace-server-webapp/src/main/webapp/login.html index 0597a66bfe96..959190020a13 100644 --- a/dspace-server-webapp/src/main/webapp/login.html +++ b/dspace-server-webapp/src/main/webapp/login.html @@ -120,7 +120,7 @@

    Other login methods:

    // Check for WWW-Authenticate header. If found, this means we are not yet authenticated, and // therefore we need to display available authentication options. var authenticate = xhr.getResponseHeader("WWW-Authenticate"); - if (authenticate !== null && authenticate.contains('location=')) { + if (authenticate !== null && authenticate.includes('location=')) { var element = $('div.other-login-methods'); var realms = authenticate.match(/(\w+ (\w+=((".*?")|[^,]*)(, )?)*)/g); if (realms.length === 1){ From 1e47fa61d92c9c87aeabbb4d1c074cc360c721af Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 11 Oct 2024 14:38:47 -0500 Subject: [PATCH 009/178] Bump to Spring 5.3.39 and Spring Security 5.8.14 --- dspace-server-webapp/pom.xml | 21 +++++++++++++++++++++ pom.xml | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/pom.xml b/dspace-server-webapp/pom.xml index e0ec7ef5ed76..97bdaa241a4f 100644 --- a/dspace-server-webapp/pom.xml +++ b/dspace-server-webapp/pom.xml @@ -362,6 +362,27 @@ org.springframework.boot spring-boot-starter-security ${spring-boot.version} + + + + org.springframework.security + spring-security-core + + + org.springframework.security + spring-security-web + + + + + org.springframework.security + spring-security-core + ${spring-security.version} + + + org.springframework.security + spring-security-web + ${spring-security.version} diff --git a/pom.xml b/pom.xml index b662e333225d..6433b6f227de 100644 --- a/pom.xml +++ b/pom.xml @@ -19,9 +19,9 @@ 11 - 5.3.34 + 5.3.39 2.7.18 - 5.7.11 + 5.8.14 5.6.15.Final 6.2.5.Final 42.7.3 From 70dd8477599abf1e0b4c8933b45c80ddd49e7561 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 11 Oct 2024 16:41:40 -0500 Subject: [PATCH 010/178] Revert to Spring Security 5.7.12. Spring Security 5.8.x changes behavior of CSRF tokens --- dspace-server-webapp/pom.xml | 9 +++++++++ pom.xml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dspace-server-webapp/pom.xml b/dspace-server-webapp/pom.xml index 97bdaa241a4f..5a4e1b32123d 100644 --- a/dspace-server-webapp/pom.xml +++ b/dspace-server-webapp/pom.xml @@ -364,6 +364,10 @@ ${spring-boot.version} + + org.springframework.security + spring-security-config + org.springframework.security spring-security-core @@ -374,6 +378,11 @@ + + org.springframework.security + spring-security-config + ${spring-security.version} + org.springframework.security spring-security-core diff --git a/pom.xml b/pom.xml index 6433b6f227de..02579c03294a 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ 11 5.3.39 2.7.18 - 5.8.14 + 5.7.12 5.6.15.Final 6.2.5.Final 42.7.3 From 4a4a8bcb22796e5b22c9bbb1796e3458b48f1c07 Mon Sep 17 00:00:00 2001 From: Brian Keese Date: Tue, 15 Oct 2024 11:38:54 -0500 Subject: [PATCH 011/178] Fix full-text indexing for files over the character limit The error handler for files over the limit logged the correct message, but never actually added the full text to the index doc. --- .../indexobject/IndexFactoryImpl.java | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java index f1ae137b9163..c9a865ec85b2 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java @@ -118,20 +118,10 @@ protected void writeDocument(SolrInputDocument doc, FullTextContentStreams strea ParseContext tikaContext = new ParseContext(); // Use Apache Tika to parse the full text stream(s) + boolean extractionSucceeded = false; try (InputStream fullTextStreams = streams.getStream()) { tikaParser.parse(fullTextStreams, tikaHandler, tikaMetadata, tikaContext); - - // Write Tika metadata to "tika_meta_*" fields. - // This metadata is not very useful right now, - // but we'll keep it just in case it becomes more useful. - for (String name : tikaMetadata.names()) { - for (String value : tikaMetadata.getValues(name)) { - doc.addField("tika_meta_" + name, value); - } - } - - // Save (parsed) full text to "fulltext" field - doc.addField("fulltext", tikaHandler.toString()); + extractionSucceeded = true; } catch (SAXException saxe) { // Check if this SAXException is just a notice that this file was longer than the character limit. // Unfortunately there is not a unique, public exception type to catch here. This error is thrown @@ -141,6 +131,7 @@ protected void writeDocument(SolrInputDocument doc, FullTextContentStreams strea // log that we only indexed up to that configured limit log.info("Full text is larger than the configured limit (discovery.solr.fulltext.charLimit)." + " Only the first {} characters were indexed.", charLimit); + extractionSucceeded = true; } else { log.error("Tika parsing error. Could not index full text.", saxe); throw new IOException("Tika parsing error. Could not index full text.", saxe); @@ -148,11 +139,19 @@ protected void writeDocument(SolrInputDocument doc, FullTextContentStreams strea } catch (TikaException | IOException ex) { log.error("Tika parsing error. Could not index full text.", ex); throw new IOException("Tika parsing error. Could not index full text.", ex); - } finally { - // Add document to index - solr.add(doc); } - return; + if (extractionSucceeded) { + // Write Tika metadata to "tika_meta_*" fields. + // This metadata is not very useful right now, + // but we'll keep it just in case it becomes more useful. + for (String name : tikaMetadata.names()) { + for (String value : tikaMetadata.getValues(name)) { + doc.addField("tika_meta_" + name, value); + } + } + // Save (parsed) full text to "fulltext" field + doc.addField("fulltext", tikaHandler.toString()); + } } // Add document to index solr.add(doc); From 6018f926d7a599b081664bf862531ededb645839 Mon Sep 17 00:00:00 2001 From: Kim Shepherd Date: Tue, 15 Oct 2024 23:23:24 +0200 Subject: [PATCH 012/178] Add Eclipse JDT .factorypath to .gitignore (cherry picked from commit 9ce645e08b50cd752e48a640e340b55466f019be) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2fcb46b9932c..529351edc5c2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ tags .project .classpath .checkstyle +.factorypath ## Ignore project files created by IntelliJ IDEA *.iml From 68266cd3c19d27dbed4f181bc12017d46fc6f64c Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 6 Nov 2024 13:36:17 -0600 Subject: [PATCH 013/178] Move logging of test methods to Abstract*Test classes in dspace-api. That way they work for **both** dspace-server-webapp and dspace-api tests. (cherry picked from commit bd20c9262b1992ff15033701fa01738329864b1b) --- .../dspace/AbstractDSpaceIntegrationTest.java | 24 ++++++++++ .../java/org/dspace/AbstractDSpaceTest.java | 23 ++++++++++ .../AbstractControllerIntegrationTest.java | 4 -- .../AbstractWebClientIntegrationTest.java | 4 -- .../test/LoggingTestExecutionListener.java | 44 ------------------- 5 files changed, 47 insertions(+), 52 deletions(-) delete mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/test/LoggingTestExecutionListener.java diff --git a/dspace-api/src/test/java/org/dspace/AbstractDSpaceIntegrationTest.java b/dspace-api/src/test/java/org/dspace/AbstractDSpaceIntegrationTest.java index 5a5ce8bf6d4c..791fdbc66abc 100644 --- a/dspace-api/src/test/java/org/dspace/AbstractDSpaceIntegrationTest.java +++ b/dspace-api/src/test/java/org/dspace/AbstractDSpaceIntegrationTest.java @@ -21,8 +21,12 @@ import org.dspace.discovery.SearchUtils; import org.dspace.servicemanager.DSpaceKernelImpl; import org.dspace.servicemanager.DSpaceKernelInit; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; /** * Abstract Test class copied from DSpace API @@ -46,6 +50,12 @@ public class AbstractDSpaceIntegrationTest { */ protected static DSpaceKernelImpl kernelImpl; + /** + * Obtain the TestName from JUnit, so that we can print it out in the test logs (see below) + */ + @Rule + public TestName testName = new TestName(); + /** * Default constructor */ @@ -90,6 +100,20 @@ public static void initTestEnvironment() { } } + @Before + public void printTestMethodBefore() { + // Log the test method being executed. Put lines around it to make it stand out. + log.info("---"); + log.info("Starting execution of test method: {}()", testName.getMethodName()); + log.info("---"); + } + + @After + public void printTestMethodAfter() { + // Log the test method just completed. + log.info("Finished execution of test method: {}()", testName.getMethodName()); + } + /** * This method will be run after all tests finish as per @AfterClass. It * will clean resources initialized by the @BeforeClass methods. diff --git a/dspace-api/src/test/java/org/dspace/AbstractDSpaceTest.java b/dspace-api/src/test/java/org/dspace/AbstractDSpaceTest.java index 36477556d3de..136af83f076f 100644 --- a/dspace-api/src/test/java/org/dspace/AbstractDSpaceTest.java +++ b/dspace-api/src/test/java/org/dspace/AbstractDSpaceTest.java @@ -18,9 +18,13 @@ import org.apache.logging.log4j.Logger; import org.dspace.servicemanager.DSpaceKernelImpl; import org.dspace.servicemanager.DSpaceKernelInit; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; +import org.junit.Rule; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; @@ -62,6 +66,12 @@ protected AbstractDSpaceTest() { } */ protected static DSpaceKernelImpl kernelImpl; + /** + * Obtain the TestName from JUnit, so that we can print it out in the test logs (see below) + */ + @Rule + public TestName testName = new TestName(); + /** * This method will be run before the first test as per @BeforeClass. It will * initialize shared resources required for all tests of this class. @@ -94,6 +104,19 @@ public static void initKernel() { } } + @Before + public void printTestMethodBefore() { + // Log the test method being executed. Put lines around it to make it stand out. + log.info("---"); + log.info("Starting execution of test method: {}()", testName.getMethodName()); + log.info("---"); + } + + @After + public void printTestMethodAfter() { + // Log the test method just completed. + log.info("Finished execution of test method: {}()", testName.getMethodName()); + } /** * This method will be run after all tests finish as per @AfterClass. It diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractControllerIntegrationTest.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractControllerIntegrationTest.java index 00339ba2e482..28c5560edf31 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractControllerIntegrationTest.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractControllerIntegrationTest.java @@ -39,7 +39,6 @@ import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.web.WebAppConfiguration; @@ -75,9 +74,6 @@ @WebAppConfiguration // Load our src/test/resources/application-test.properties to override some settings in default application.properties @TestPropertySource(locations = "classpath:application-test.properties") -// Enable our custom Logging listener to log when each test method starts/stops -@TestExecutionListeners(listeners = {LoggingTestExecutionListener.class}, - mergeMode = TestExecutionListeners.MergeMode.MERGE_WITH_DEFAULTS) public class AbstractControllerIntegrationTest extends AbstractIntegrationTestWithDatabase { protected static final String AUTHORIZATION_HEADER = "Authorization"; diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractWebClientIntegrationTest.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractWebClientIntegrationTest.java index 75b0143e3e65..df6afcb16e32 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractWebClientIntegrationTest.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractWebClientIntegrationTest.java @@ -22,7 +22,6 @@ import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; @@ -52,9 +51,6 @@ @ContextConfiguration(initializers = { DSpaceKernelInitializer.class, DSpaceConfigurationInitializer.class }) // Load our src/test/resources/application-test.properties to override some settings in default application.properties @TestPropertySource(locations = "classpath:application-test.properties") -// Enable our custom Logging listener to log when each test method starts/stops -@TestExecutionListeners(listeners = {LoggingTestExecutionListener.class}, - mergeMode = TestExecutionListeners.MergeMode.MERGE_WITH_DEFAULTS) public class AbstractWebClientIntegrationTest extends AbstractIntegrationTestWithDatabase { // (Random) port chosen for test web server @LocalServerPort diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/LoggingTestExecutionListener.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/LoggingTestExecutionListener.java deleted file mode 100644 index 2a04bab2041c..000000000000 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/LoggingTestExecutionListener.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.rest.test; - -import org.apache.logging.log4j.Logger; -import org.springframework.test.context.TestContext; -import org.springframework.test.context.support.AbstractTestExecutionListener; - -/** - * Custom DSpace TestExecutionListener which logs messages whenever a specific Test Case (i.e. test method) has - * started or ended execution. This makes Test environment logs easier to read/understand as you know which method has - * caused errors, etc. - */ -public class LoggingTestExecutionListener extends AbstractTestExecutionListener { - - private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(LoggingTestExecutionListener.class); - - /** - * Before each test method is run - * @param testContext - */ - @Override - public void beforeTestMethod(TestContext testContext) { - // Log the test method being executed. Put lines around it to make it stand out. - log.info("---"); - log.info("Starting execution of test method: {}()", testContext.getTestMethod().getName()); - log.info("---"); - } - - /** - * After each test method is run - * @param testContext - */ - @Override - public void afterTestMethod(TestContext testContext) { - // Log the test method just completed. - log.info("Finished execution of test method: {}()", testContext.getTestMethod().getName()); - } -} From 831db33018cdcfa132dc0b1da0f427aca0d03585 Mon Sep 17 00:00:00 2001 From: Jens Vannerum Date: Mon, 14 Oct 2024 17:03:15 +0200 Subject: [PATCH 014/178] 118774: status of doi should be set to TO_BE_DELETED when related item is removed permanently (cherry picked from commit 352f4c21523237cefa5bf67c99f7d7b26a51d72b) --- .../src/main/java/org/dspace/content/ItemServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java index b2cc3e939d87..ad4156328aeb 100644 --- a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java @@ -66,6 +66,7 @@ import org.dspace.harvest.HarvestedItem; import org.dspace.harvest.service.HarvestedItemService; import org.dspace.identifier.DOI; +import org.dspace.identifier.DOIIdentifierProvider; import org.dspace.identifier.IdentifierException; import org.dspace.identifier.service.DOIService; import org.dspace.identifier.service.IdentifierService; @@ -848,6 +849,7 @@ protected void rawDelete(Context context, Item item) throws AuthorizeException, DOI doi = doiService.findDOIByDSpaceObject(context, item); if (doi != null) { doi.setDSpaceObject(null); + doi.setStatus(DOIIdentifierProvider.TO_BE_DELETED); } // remove version attached to the item From fde825265db6b44ca0bd852ec6df56ca3f3eacfe Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 27 Sep 2024 12:52:03 +0200 Subject: [PATCH 015/178] fix: performance of claiming workflow task (cherry picked from commit 27dd5a2ec58970256964f15f0879834c436aa542) --- .../storedcomponents/PoolTaskServiceImpl.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/xmlworkflow/storedcomponents/PoolTaskServiceImpl.java b/dspace-api/src/main/java/org/dspace/xmlworkflow/storedcomponents/PoolTaskServiceImpl.java index fb673725e181..d3c8f6334d8f 100644 --- a/dspace-api/src/main/java/org/dspace/xmlworkflow/storedcomponents/PoolTaskServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/xmlworkflow/storedcomponents/PoolTaskServiceImpl.java @@ -13,6 +13,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.Set; import org.apache.commons.collections4.CollectionUtils; @@ -100,12 +101,17 @@ public PoolTask findByWorkflowIdAndEPerson(Context context, XmlWorkflowItem work //If the user does not have a claimedtask yet, see whether one of the groups of the user has pooltasks //for this workflow item Set groups = groupService.allMemberGroupsSet(context, ePerson); - for (Group group : groups) { - poolTask = poolTaskDAO.findByWorkflowItemAndGroup(context, group, workflowItem); - if (poolTask != null) { - return poolTask; - } + List generalTasks = poolTaskDAO.findByWorkflowItem(context, workflowItem); + Optional firstClaimedTask = groups.stream() + .flatMap(group -> generalTasks.stream() + .filter(f -> f.getGroup().getID().equals(group.getID())) + .findFirst() + .stream()) + .findFirst(); + + if (firstClaimedTask.isPresent()) { + return firstClaimedTask.get(); } } } From c66dde2aa85540dfbfa36f18f581599a6457889e Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Mon, 11 Nov 2024 10:26:29 -0600 Subject: [PATCH 016/178] Add a job to test Docker deployment with the built images --- .github/workflows/docker.yml | 43 +++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a9ff8760e763..12d49689b319 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -147,4 +147,45 @@ jobs: tags_flavor: suffix=-loadsql secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} \ No newline at end of file + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + ######################################################################## + # Test Deployment via Docker to ensure images are working properly + ######################################################################## + docker-deploy: + # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' + if: github.repository == 'dspace/dspace' + runs-on: ubuntu-latest + # Must run after all major images are built + needs: [dspace-test, dspace-cli, dspace-postgres-pgcrypto, dspace-solr] + steps: + - name: Checkout codebase + uses: actions/checkout@v4 + # Start backend using our compose script in the codebase. + - name: Start backend in Docker + env: + # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080 + dspace__P__server__P__url: http://127.0.0.1:8080/server + run: | + docker compose -f docker-compose.yml up -d + sleep 10 + docker container ls + # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml + - name: Load test data into Backend + run: | + docker compose -f docker-compose-cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en + docker compose -f docker-compose-cli.yml -f dspace/src/main/docker-compose/cli.ingest.yml run --rm dspace-cli + # Verify backend started successfully. + # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml) + # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs) + - name: Verify backend is responding properly + run: | + result=$(wget -O- -q http://127.0.0.1:8080/server/api) + echo "$result" + echo "$result" | grep -oE "\"DSpace Started with Docker Compose\"," + result=$(wget -O- -q http://127.0.0.1:8080/server/api/core/collections) + echo "$result" + echo "$result" | grep -oE "\"Dog in Yard\"," + - name: Shutdown Docker containers + run: | + docker compose -f docker-compose.yml down From 3ce85a2df3c548bf7842227cf86d8aa8bd2191b0 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Mon, 11 Nov 2024 14:10:44 -0600 Subject: [PATCH 017/178] Add a check that the Handle Server can be started & works properly --- .github/workflows/docker.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 12d49689b319..660c9eb817a6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -186,6 +186,25 @@ jobs: result=$(wget -O- -q http://127.0.0.1:8080/server/api/core/collections) echo "$result" echo "$result" | grep -oE "\"Dog in Yard\"," + # Verify Handle Server can be stared and is working properly + # 1. First generate the "[dspace]/handle-server" folder with the sitebndl.zip + # 2. Start the Handle Server (and wait 20 seconds to let it start up) + # 3. Verify logs do NOT include "Exception" in the text (as that means an error occurred) + # 4. Check that Handle Proxy HTML page is responding on default port (8000) + - name: Verify Handle Server is working properly + run: | + docker exec -i dspace /dspace/bin/make-handle-config + docker exec -i dspace /dspace/bin/start-handle-server + sleep 20 + echo "Checking for errors in handle-server.log..." + result=$(docker exec -i dspace cat /dspace/log/handle-server.log) + echo "$result" + echo "$result" | grep -vqz "Exception" + echo "Checking to see if Handle Proxy webpage is available..." + result=$(wget -O- -q http://127.0.0.1:8000/) + echo "$result" + echo "$result" | grep -oE "Handle Proxy" + # Shutdown our containers - name: Shutdown Docker containers run: | docker compose -f docker-compose.yml down From c1f168245b9756b6109bf6b53bc87317a66aa55c Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 12 Nov 2024 13:50:57 -0600 Subject: [PATCH 018/178] Ensure Docker images built from PRs are stored as artifacts. This allows us to use those new images when testing deployment (in docker-deploy) --- .github/workflows/docker.yml | 34 +++++++++--- .github/workflows/reusable-docker-build.yml | 58 +++++++++++++++++++-- 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 660c9eb817a6..eed8de5a8722 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -157,27 +157,47 @@ jobs: if: github.repository == 'dspace/dspace' runs-on: ubuntu-latest # Must run after all major images are built - needs: [dspace-test, dspace-cli, dspace-postgres-pgcrypto, dspace-solr] + needs: [dspace, dspace-test, dspace-cli, dspace-postgres-pgcrypto, dspace-solr] steps: + # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase uses: actions/checkout@v4 - # Start backend using our compose script in the codebase. + # For PRs, download Docker image artifacts (built by reusable-docker-build.yml for all PRs) + - name: Download Docker image artifacts (for PRs) + if: github.event_name == 'pull_request' + uses: actions/download-artifact@v4 + with: + # Download all Docker images (TAR files) into the /tmp/docker directory + pattern: docker-image-* + path: /tmp/docker + merge-multiple: true + # For PRs, load each of the images into Docker by calling "docker image load" for each. + # This ensures we are using the images built from this PR & not the prior versions on DockerHub + - name: Load all downloaded Docker images (for PRs) + if: github.event_name == 'pull_request' + run: | + find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \; + docker image ls -a + # Start backend using our compose script in the codebase. - name: Start backend in Docker env: # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080 dspace__P__server__P__url: http://127.0.0.1:8080/server + # Force using "pr-testing" version of Docker images. The "pr-testing" tag is a temporary tag that we assign to + # all PR-built docker images in reusabe-docker-build.yml + DSPACE_VER: pr-testing run: | docker compose -f docker-compose.yml up -d sleep 10 docker container ls - # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml + # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml - name: Load test data into Backend run: | docker compose -f docker-compose-cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en docker compose -f docker-compose-cli.yml -f dspace/src/main/docker-compose/cli.ingest.yml run --rm dspace-cli - # Verify backend started successfully. - # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml) - # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs) + # Verify backend started successfully. + # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml) + # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs) - name: Verify backend is responding properly run: | result=$(wget -O- -q http://127.0.0.1:8080/server/api) @@ -204,7 +224,7 @@ jobs: result=$(wget -O- -q http://127.0.0.1:8000/) echo "$result" echo "$result" | grep -oE "Handle Proxy" - # Shutdown our containers + # Shutdown our containers - name: Shutdown Docker containers run: | docker compose -f docker-compose.yml down diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index 12aa0cfe2864..83dfd74b9a1a 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -113,6 +113,12 @@ jobs: - name: Set up QEMU emulation to build for multiple architectures uses: docker/setup-qemu-action@v3 + #------------------------------------------------------------ + # Build & deploy steps for new commits to a branch (non-PRs) + # + # These steps build the images, push to DockerHub, and + # (if necessary) redeploy demo/sandbox sites. + #------------------------------------------------------------ # https://github.com/docker/login-action - name: Login to DockerHub # Only login if not a PR, as PRs only trigger a Docker build and not a push @@ -125,6 +131,7 @@ jobs: # https://github.com/docker/metadata-action # Get Metadata for docker_build_deps step below - name: Sync metadata (tags, labels) from GitHub to Docker for image + if: ${{ ! matrix.isPr }} id: meta_build uses: docker/metadata-action@v5 with: @@ -133,7 +140,9 @@ jobs: flavor: ${{ env.TAGS_FLAVOR }} # https://github.com/docker/build-push-action - - name: Build and push image + - name: Build and push image to DockerHub + # Only build & push if not a PR + if: ${{ ! matrix.isPr }} id: docker_build uses: docker/build-push-action@v5 with: @@ -142,9 +151,7 @@ jobs: context: ${{ inputs.dockerfile_context }} file: ${{ inputs.dockerfile_path }} platforms: ${{ matrix.arch }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ ! matrix.isPr }} + push: true # Use tags / labels provided by 'docker/metadata-action' above tags: ${{ steps.meta_build.outputs.tags }} labels: ${{ steps.meta_build.outputs.labels }} @@ -189,11 +196,54 @@ jobs: run: | curl -X POST $REDEPLOY_DEMO_URL + #------------------------------------------------------------- + # Build steps for PRs only + # + # These steps build the images and store as a build artifact. + # These artifacts can then be used by later jobs to run the + # brand-new images for automated testing. + #-------------------------------------------------------------- + # Get Metadata for docker_build_deps step below + - name: Create metadata (tags, labels) for local Docker image + if: matrix.isPr + id: meta_build_pr + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + # Hardcode to use custom "pr-testing" tag because that will allow us to spin up this PR + # for testing in docker.yml + tags: pr-testing + flavor: ${{ env.TAGS_FLAVOR }} + # Build local image and stores in a TAR file in /tmp directory + - name: Build and push image to local image + if: matrix.isPr + uses: docker/build-push-action@v5 + with: + build-contexts: | + ${{ inputs.dockerfile_additional_contexts }} + context: ${{ inputs.dockerfile_context }} + file: ${{ inputs.dockerfile_path }} + platforms: ${{ matrix.arch }} + tags: ${{ steps.meta_build_pr.outputs.tags }} + labels: ${{ steps.meta_build_pr.outputs.labels }} + # Export image to a local TAR file + outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar + # Upload the local docker image (in TAR file) to a build Artifact + - name: Upload local image to artifact + if: matrix.isPr + uses: actions/upload-artifact@v4 + with: + name: docker-image-${{ inputs.build_id }} + path: /tmp/${{ inputs.build_id }}.tar + if-no-files-found: error + retention-days: 1 + # Merge Docker digests (from various architectures) into a manifest. # This runs after all Docker builds complete above, and it tells hub.docker.com # that these builds should be all included in the manifest for this tag. # (e.g. AMD64 and ARM64 should be listed as options under the same tagged Docker image) docker-build_manifest: + # Only run if this is NOT a PR if: ${{ github.event_name != 'pull_request' }} runs-on: ubuntu-latest needs: From 9853aa5bb45ceef6ba6e01a08515f0cdea68f92c Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Mon, 11 Nov 2024 16:45:59 -0600 Subject: [PATCH 019/178] Fix error in running Handle Server in GitHub Actions. Must exclude "spring-jcl" from dependencies as it conflicts with "commons-logging" (used by more of our dependencies) --- dspace-api/pom.xml | 7 +++++++ dspace-iiif/pom.xml | 5 +++++ dspace-oai/pom.xml | 5 +++++ dspace-rdf/pom.xml | 5 +++++ dspace-server-webapp/pom.xml | 5 +++++ dspace-services/pom.xml | 7 +++++++ dspace-sword/pom.xml | 5 +++++ dspace-swordv2/pom.xml | 5 +++++ pom.xml | 7 +++++++ 9 files changed, 51 insertions(+) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index bdf035179bee..c3ef2c86044d 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -396,6 +396,13 @@ org.springframework spring-orm + + + + org.springframework + spring-jcl + + diff --git a/dspace-iiif/pom.xml b/dspace-iiif/pom.xml index 64dd5106f0af..49ee02b56500 100644 --- a/dspace-iiif/pom.xml +++ b/dspace-iiif/pom.xml @@ -38,6 +38,11 @@ org.springframework.boot spring-boot-starter-logging + + + org.springframework + spring-jcl + diff --git a/dspace-oai/pom.xml b/dspace-oai/pom.xml index 28b610e996c1..bfa2c8a27381 100644 --- a/dspace-oai/pom.xml +++ b/dspace-oai/pom.xml @@ -80,6 +80,11 @@ org.springframework.boot spring-boot-starter-logging + + + org.springframework + spring-jcl + diff --git a/dspace-rdf/pom.xml b/dspace-rdf/pom.xml index e656a920a1e6..075f2d7038e8 100644 --- a/dspace-rdf/pom.xml +++ b/dspace-rdf/pom.xml @@ -67,6 +67,11 @@ org.springframework.boot spring-boot-starter-logging + + + org.springframework + spring-jcl + diff --git a/dspace-server-webapp/pom.xml b/dspace-server-webapp/pom.xml index 5a4e1b32123d..35fa473fc170 100644 --- a/dspace-server-webapp/pom.xml +++ b/dspace-server-webapp/pom.xml @@ -405,6 +405,11 @@ org.springframework.boot spring-boot-starter-logging + + + org.springframework + spring-jcl + diff --git a/dspace-services/pom.xml b/dspace-services/pom.xml index 51c6dea62413..7c85ff7f0f0f 100644 --- a/dspace-services/pom.xml +++ b/dspace-services/pom.xml @@ -85,6 +85,13 @@ spring-context-support ${spring.version} compile + + + + org.springframework + spring-jcl + + org.apache.commons diff --git a/dspace-sword/pom.xml b/dspace-sword/pom.xml index 1403c779e720..01aa68dfbfe6 100644 --- a/dspace-sword/pom.xml +++ b/dspace-sword/pom.xml @@ -60,6 +60,11 @@ org.springframework.boot spring-boot-starter-logging + + + org.springframework + spring-jcl + diff --git a/dspace-swordv2/pom.xml b/dspace-swordv2/pom.xml index 470388883126..f575898c122a 100644 --- a/dspace-swordv2/pom.xml +++ b/dspace-swordv2/pom.xml @@ -80,6 +80,11 @@ org.springframework.boot spring-boot-starter-logging + + + org.springframework + spring-jcl + diff --git a/pom.xml b/pom.xml index 02579c03294a..b0d9d1819d3f 100644 --- a/pom.xml +++ b/pom.xml @@ -1171,6 +1171,13 @@ org.springframework spring-orm ${spring.version} + + + + org.springframework + spring-jcl + + org.swordapp From faca14ad4059de0cb66c6d6ca0baf4161ac3aadd Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 13 Nov 2024 09:17:30 -0600 Subject: [PATCH 020/178] Ensure "host" command is installed in images, so "bin/make-handle-config" will work. --- Dockerfile | 5 +++++ Dockerfile.test | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9d89710fe11c..ffce09239d1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,6 +55,11 @@ FROM tomcat:9-jdk${JDK_VERSION} ENV DSPACE_INSTALL=/dspace # Copy the /dspace directory from 'ant_build' container to /dspace in this container COPY --from=ant_build /dspace $DSPACE_INSTALL +# Need host command for "[dspace]/bin/make-handle-config" +RUN apt-get update \ + && apt-get install -y --no-install-recommends host \ + && apt-get purge -y --auto-remove \ + && rm -rf /var/lib/apt/lists/* # Expose Tomcat port and AJP port EXPOSE 8080 8009 # Give java extra memory (2GB) diff --git a/Dockerfile.test b/Dockerfile.test index 031394ad256c..d88699ca52ab 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -54,6 +54,11 @@ ENV DSPACE_INSTALL=/dspace ENV TOMCAT_INSTALL=/usr/local/tomcat # Copy the /dspace directory from 'ant_build' containger to /dspace in this container COPY --from=ant_build /dspace $DSPACE_INSTALL +# Need host command for "[dspace]/bin/make-handle-config" +RUN apt-get update \ + && apt-get install -y --no-install-recommends host \ + && apt-get purge -y --auto-remove \ + && rm -rf /var/lib/apt/lists/* # Enable the AJP connector in Tomcat's server.xml # NOTE: secretRequired="false" should only be used when AJP is NOT accessible from an external network. But, secretRequired="true" isn't supported by mod_proxy_ajp until Apache 2.5 RUN sed -i '/Service name="Catalina".*/a \\n ' $TOMCAT_INSTALL/conf/server.xml From a0ed4a40eabab6296a577b3d7c8aa5a5fe024888 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 13 Nov 2024 10:37:53 -0600 Subject: [PATCH 021/178] Bug fixes. Ensure all steps of docker-deploy use the same environment variables. Ensure Handle Server HTTP port is open --- .github/workflows/docker.yml | 13 +++++++------ Dockerfile | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eed8de5a8722..5197ed3bfe39 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -158,6 +158,12 @@ jobs: runs-on: ubuntu-latest # Must run after all major images are built needs: [dspace, dspace-test, dspace-cli, dspace-postgres-pgcrypto, dspace-solr] + env: + # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080 + dspace__P__server__P__url: http://127.0.0.1:8080/server + # Force using "pr-testing" version of all Docker images. The "pr-testing" tag is a temporary tag that we + # assign to all PR-built docker images in reusabe-docker-build.yml + DSPACE_VER: pr-testing steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase @@ -180,12 +186,6 @@ jobs: docker image ls -a # Start backend using our compose script in the codebase. - name: Start backend in Docker - env: - # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080 - dspace__P__server__P__url: http://127.0.0.1:8080/server - # Force using "pr-testing" version of Docker images. The "pr-testing" tag is a temporary tag that we assign to - # all PR-built docker images in reusabe-docker-build.yml - DSPACE_VER: pr-testing run: | docker compose -f docker-compose.yml up -d sleep 10 @@ -214,6 +214,7 @@ jobs: - name: Verify Handle Server is working properly run: | docker exec -i dspace /dspace/bin/make-handle-config + echo "Starting Handle Server..." docker exec -i dspace /dspace/bin/start-handle-server sleep 20 echo "Checking for errors in handle-server.log..." diff --git a/Dockerfile b/Dockerfile index ffce09239d1b..adc5d6125f0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,8 +60,8 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends host \ && apt-get purge -y --auto-remove \ && rm -rf /var/lib/apt/lists/* -# Expose Tomcat port and AJP port -EXPOSE 8080 8009 +# Expose Tomcat port (8080) and AJP port (8009) and Handle Server HTTP port (8000) +EXPOSE 8080 8009 8000 # Give java extra memory (2GB) ENV JAVA_OPTS=-Xmx2000m From d6d78298b31252155db7d00d7ae8eee8f2c0ac93 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 13 Nov 2024 11:45:03 -0600 Subject: [PATCH 022/178] Add check for Handle Server error.log --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5197ed3bfe39..1e839e89fceb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -217,6 +217,10 @@ jobs: echo "Starting Handle Server..." docker exec -i dspace /dspace/bin/start-handle-server sleep 20 + echo "Checking for errors in error.log" + result=$(docker exec -i dspace sh -c "cat /dspace/handle-server/logs/error.log* || echo ''") + echo "$result" + echo "$result" | grep -vqz "Exception" echo "Checking for errors in handle-server.log..." result=$(docker exec -i dspace cat /dspace/log/handle-server.log) echo "$result" From 5bb65c6b5647f74161244868c0864e9adb12d12a Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 13 Nov 2024 14:27:28 -0600 Subject: [PATCH 023/178] Fix error in Handle Server startup caused by having multiple versions of BouncyCastle in our classpath. Exclude the old version brought in by cnri-servlet-container --- dspace-api/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index c3ef2c86044d..c42895ab4c49 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -427,6 +427,16 @@ org.bouncycastle bcprov-jdk15on + + + org.bouncycastle + bcpkix-jdk15on + + + org.bouncycastle + bcprov-jdk15on + From aa71e4840be6fa9c6368e061d3e169e08a89c66a Mon Sep 17 00:00:00 2001 From: Agustina Martinez Date: Mon, 5 Aug 2024 11:06:06 +0200 Subject: [PATCH 024/178] Fix 9734: Check configured workflow.reviewer.file-edit to show item edit functionality in workflow UI (cherry picked from commit e8ec0c1b1d20dd5e812f38593a24718fff1d8c6e) --- .../processingaction/ScoreReviewAction.java | 15 ++++++++++++++- .../processingaction/SingleUserReviewAction.java | 8 ++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/xmlworkflow/state/actions/processingaction/ScoreReviewAction.java b/dspace-api/src/main/java/org/dspace/xmlworkflow/state/actions/processingaction/ScoreReviewAction.java index 43a3decacc7e..419bb1236402 100644 --- a/dspace-api/src/main/java/org/dspace/xmlworkflow/state/actions/processingaction/ScoreReviewAction.java +++ b/dspace-api/src/main/java/org/dspace/xmlworkflow/state/actions/processingaction/ScoreReviewAction.java @@ -8,6 +8,7 @@ package org.dspace.xmlworkflow.state.actions.processingaction; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -20,6 +21,8 @@ import org.dspace.authorize.AuthorizeException; import org.dspace.content.MetadataFieldName; import org.dspace.core.Context; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; import org.dspace.xmlworkflow.service.WorkflowRequirementsService; import org.dspace.xmlworkflow.state.Step; import org.dspace.xmlworkflow.state.actions.ActionAdvancedInfo; @@ -34,6 +37,9 @@ public class ScoreReviewAction extends ProcessingAction { private static final Logger log = LogManager.getLogger(ScoreReviewAction.class); + private final ConfigurationService configurationService + = DSpaceServicesFactory.getInstance().getConfigurationService(); + // Option(s) public static final String SUBMIT_SCORE = "submit_score"; @@ -114,7 +120,14 @@ private boolean checkRequestValid(int score, String review) { @Override public List getOptions() { - return List.of(SUBMIT_SCORE, RETURN_TO_POOL); + List options = new ArrayList<>(); + options.add(SUBMIT_SCORE); + if (configurationService.getBooleanProperty("workflow.reviewer.file-edit", false)) { + options.add(SUBMIT_EDIT_METADATA); + } + options.add(RETURN_TO_POOL); + + return options; } @Override diff --git a/dspace-api/src/main/java/org/dspace/xmlworkflow/state/actions/processingaction/SingleUserReviewAction.java b/dspace-api/src/main/java/org/dspace/xmlworkflow/state/actions/processingaction/SingleUserReviewAction.java index b3fe896ace24..0b8f2d13648c 100644 --- a/dspace-api/src/main/java/org/dspace/xmlworkflow/state/actions/processingaction/SingleUserReviewAction.java +++ b/dspace-api/src/main/java/org/dspace/xmlworkflow/state/actions/processingaction/SingleUserReviewAction.java @@ -21,6 +21,8 @@ import org.dspace.content.factory.ContentServiceFactory; import org.dspace.core.Context; import org.dspace.eperson.EPerson; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; import org.dspace.workflow.WorkflowException; import org.dspace.xmlworkflow.factory.XmlWorkflowServiceFactory; import org.dspace.xmlworkflow.state.Step; @@ -40,6 +42,9 @@ public class SingleUserReviewAction extends ProcessingAction { private static final Logger log = LogManager.getLogger(SingleUserReviewAction.class); + private final ConfigurationService configurationService + = DSpaceServicesFactory.getInstance().getConfigurationService(); + public static final int OUTCOME_REJECT = 1; protected static final String SUBMIT_DECLINE_TASK = "submit_decline_task"; @@ -95,6 +100,9 @@ public ActionResult processAccept(Context c, XmlWorkflowItem wfi) throws SQLExce public List getOptions() { List options = new ArrayList<>(); options.add(SUBMIT_APPROVE); + if (configurationService.getBooleanProperty("workflow.reviewer.file-edit", false)) { + options.add(SUBMIT_EDIT_METADATA); + } options.add(SUBMIT_REJECT); options.add(SUBMIT_DECLINE_TASK); return options; From 7fa31e21315db54a5f99e4d34dc91b29353a92b8 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 15 Nov 2024 08:59:19 -0600 Subject: [PATCH 025/178] Ensure we use "pr-testing" images for PRs, but use "latest" images for other builds (e.g. after PR is merged to a branch). (cherry picked from commit aa537c44902f458a684bcbb67b5ff702ded61363) --- .github/workflows/docker.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1e839e89fceb..a0aee14bea38 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -161,9 +161,10 @@ jobs: env: # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080 dspace__P__server__P__url: http://127.0.0.1:8080/server - # Force using "pr-testing" version of all Docker images. The "pr-testing" tag is a temporary tag that we - # assign to all PR-built docker images in reusabe-docker-build.yml - DSPACE_VER: pr-testing + # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, for branch commits, use the + # "latest" tag. NOTE: the "pr-testing" tag is a temporary tag that we assign to all PR-built docker images in + # reusabe-docker-build.yml + DSPACE_VER: ${{ github.event_name == 'pull_request' && 'pr-testing' || 'latest' }} steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase From f0c92ac96b6f6f22445020b9e9690886bb83830c Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 15 Nov 2024 16:06:10 -0600 Subject: [PATCH 026/178] Ensure only main branch uses "latest". Other branches should use the tag corresponding to the branch name (cherry picked from commit e0b7241acb167496ebc8c80c73ae69b9f6611a1c) --- .github/workflows/docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a0aee14bea38..143b69a2f9fe 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -161,10 +161,10 @@ jobs: env: # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080 dspace__P__server__P__url: http://127.0.0.1:8080/server - # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, for branch commits, use the - # "latest" tag. NOTE: the "pr-testing" tag is a temporary tag that we assign to all PR-built docker images in - # reusabe-docker-build.yml - DSPACE_VER: ${{ github.event_name == 'pull_request' && 'pr-testing' || 'latest' }} + # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, if on main branch, use the + # "latest" tag. Otherwise, use the branch name. NOTE: the "pr-testing" tag is a temporary tag that we assign to + # all PR-built docker images in reusabe-docker-build.yml + DSPACE_VER: ${{ (github.event_name == 'pull_request' && 'pr-testing') || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase From d952fea6a2f6b5dc8a54b4e6ba3deecca2d7a4fe Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 15 Nov 2024 11:30:14 -0600 Subject: [PATCH 027/178] Remove parboiled-java and minor cleanup of unused OAI dependencies --- dspace-oai/pom.xml | 23 +------------------ .../xoai/app/CCElementItemCompilePlugin.java | 2 +- .../main/java/org/dspace/xoai/app/XOAI.java | 2 +- .../tests/integration/xoai/PipelineTest.java | 9 ++++---- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/dspace-oai/pom.xml b/dspace-oai/pom.xml index bfa2c8a27381..2f8dae45b236 100644 --- a/dspace-oai/pom.xml +++ b/dspace-oai/pom.xml @@ -99,22 +99,9 @@ org.springframework.boot spring-boot-starter-web - - - org.parboiled - parboiled-java - - - - org.parboiled - parboiled-java - 1.3.1 - - org.dspace @@ -137,15 +124,7 @@ org.apache.logging.log4j log4j-core - - org.apache.logging.log4j - log4j-web - - - org.apache.logging.log4j - log4j-slf4j-impl - runtime - + org.apache.logging.log4j log4j-1.2-api diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/CCElementItemCompilePlugin.java b/dspace-oai/src/main/java/org/dspace/xoai/app/CCElementItemCompilePlugin.java index 225d56a4c982..370543029d8b 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/CCElementItemCompilePlugin.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/CCElementItemCompilePlugin.java @@ -11,7 +11,7 @@ import com.lyncode.xoai.dataprovider.xml.xoai.Element; import com.lyncode.xoai.dataprovider.xml.xoai.Metadata; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.dspace.content.Item; import org.dspace.core.Context; import org.dspace.license.factory.LicenseServiceFactory; diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index 4f842b8e944c..c6aaaa34b539 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -9,7 +9,7 @@ import static com.lyncode.xoai.dataprovider.core.Granularity.Second; import static java.util.Objects.nonNull; -import static org.apache.commons.lang.StringUtils.EMPTY; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.solr.common.params.CursorMarkParams.CURSOR_MARK_PARAM; import static org.apache.solr.common.params.CursorMarkParams.CURSOR_MARK_START; import static org.dspace.xoai.util.ItemUtils.retrieveMetadata; diff --git a/dspace-oai/src/test/java/org/dspace/xoai/tests/integration/xoai/PipelineTest.java b/dspace-oai/src/test/java/org/dspace/xoai/tests/integration/xoai/PipelineTest.java index 0f48824159c2..0f7ffde0bd00 100644 --- a/dspace-oai/src/test/java/org/dspace/xoai/tests/integration/xoai/PipelineTest.java +++ b/dspace-oai/src/test/java/org/dspace/xoai/tests/integration/xoai/PipelineTest.java @@ -13,13 +13,14 @@ import static org.hamcrest.MatcherAssert.assertThat; import java.io.InputStream; +import java.nio.charset.Charset; import javax.xml.transform.TransformerFactory; import javax.xml.transform.stream.StreamSource; import com.lyncode.xoai.util.XSLPipeline; +import org.apache.commons.io.IOUtils; import org.dspace.xoai.tests.support.XmlMatcherBuilder; import org.junit.Test; -import org.parboiled.common.FileUtils; public class PipelineTest { private static TransformerFactory factory = TransformerFactory.newInstance(); @@ -28,9 +29,9 @@ public class PipelineTest { public void pipelineTest() throws Exception { InputStream input = PipelineTest.class.getClassLoader().getResourceAsStream("item.xml"); InputStream xslt = PipelineTest.class.getClassLoader().getResourceAsStream("oai_dc.xsl"); - String output = FileUtils.readAllText(new XSLPipeline(input, true) - .apply(factory.newTemplates(new StreamSource(xslt))) - .getTransformed()); + String output = IOUtils.toString(new XSLPipeline(input, true) + .apply(factory.newTemplates(new StreamSource(xslt))) + .getTransformed(), Charset.defaultCharset()); assertThat(output, oai_dc().withXPath("/oai_dc:dc/dc:title", equalTo("Teste"))); From f3d15e5c04227ceb5425e671189ec4eb5ea42656 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 15 Nov 2024 12:02:20 -0600 Subject: [PATCH 028/178] Log4j cleanup. Remove last traces of log4j v1 (and remove log4j1 bridge to avoid them coming back). Create log4j2 settings for Handle Plugin. --- .../app/bulkedit/MetadataExportSearchIT.java | 5 +- .../dspace/builder/OrcidHistoryBuilder.java | 5 +- dspace-oai/pom.xml | 5 -- dspace-rdf/pom.xml | 4 -- dspace-rest/pom.xml | 4 -- .../rest/converter/SearchEventConverter.java | 5 +- .../bitstream/BitstreamLinksetProcessor.java | 5 +- .../BitstreamParentItemProcessor.java | 5 +- .../bitstream/BitstreamTypeProcessor.java | 5 +- .../processor/item/ItemAuthorProcessor.java | 5 +- .../item/ItemContentBitstreamsProcessor.java | 5 +- .../item/ItemDescribedbyProcessor.java | 5 +- .../processor/item/ItemLicenseProcessor.java | 5 +- .../processor/item/ItemLinksetProcessor.java | 5 +- .../processor/item/ItemTypeProcessor.java | 5 +- .../service/impl/LinksetServiceImpl.java | 5 +- dspace-sword/pom.xml | 4 -- dspace-swordv2/pom.xml | 4 -- dspace/bin/start-handle-server | 2 +- dspace/bin/start-handle-server.bat | 2 +- dspace/config/log4j-handle-plugin.properties | 34 ------------- dspace/config/log4j2-handle-plugin.xml | 48 +++++++++++++++++++ pom.xml | 25 ---------- 23 files changed, 89 insertions(+), 108 deletions(-) delete mode 100644 dspace/config/log4j-handle-plugin.properties create mode 100644 dspace/config/log4j2-handle-plugin.xml diff --git a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataExportSearchIT.java b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataExportSearchIT.java index 3a972692efeb..63a87a48f554 100644 --- a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataExportSearchIT.java +++ b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataExportSearchIT.java @@ -23,7 +23,8 @@ import com.google.common.io.Files; import com.opencsv.CSVReader; import com.opencsv.exceptions.CsvException; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; @@ -51,7 +52,7 @@ public class MetadataExportSearchIT extends AbstractIntegrationTestWithDatabase private Item[] itemsSubject2 = new Item[numberItemsSubject2]; private String filename; private Collection collection; - private Logger logger = Logger.getLogger(MetadataExportSearchIT.class); + private Logger logger = LogManager.getLogger(MetadataExportSearchIT.class); private ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); private SearchService searchService; diff --git a/dspace-api/src/test/java/org/dspace/builder/OrcidHistoryBuilder.java b/dspace-api/src/test/java/org/dspace/builder/OrcidHistoryBuilder.java index 199f412f8506..d811d03f5358 100644 --- a/dspace-api/src/test/java/org/dspace/builder/OrcidHistoryBuilder.java +++ b/dspace-api/src/test/java/org/dspace/builder/OrcidHistoryBuilder.java @@ -11,7 +11,8 @@ import java.sql.SQLException; import java.util.Date; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.core.Context; import org.dspace.orcid.OrcidHistory; @@ -24,7 +25,7 @@ */ public class OrcidHistoryBuilder extends AbstractBuilder { - private static final Logger log = Logger.getLogger(OrcidHistoryBuilder.class); + private static final Logger log = LogManager.getLogger(OrcidHistoryBuilder.class); private OrcidHistory orcidHistory; diff --git a/dspace-oai/pom.xml b/dspace-oai/pom.xml index 2f8dae45b236..f7acdae99292 100644 --- a/dspace-oai/pom.xml +++ b/dspace-oai/pom.xml @@ -124,11 +124,6 @@ org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api - diff --git a/dspace-rdf/pom.xml b/dspace-rdf/pom.xml index 075f2d7038e8..64d88b63c780 100644 --- a/dspace-rdf/pom.xml +++ b/dspace-rdf/pom.xml @@ -89,10 +89,6 @@ org.apache.logging.log4j log4j-core - - org.apache.logging.log4j - log4j-web - org.apache.commons diff --git a/dspace-rest/pom.xml b/dspace-rest/pom.xml index d1f3b95aafb0..3f4bdaf1d0d8 100644 --- a/dspace-rest/pom.xml +++ b/dspace-rest/pom.xml @@ -185,10 +185,6 @@ org.apache.logging.log4j log4j-core - - org.apache.logging.log4j - log4j-web - org.dspace dspace-services diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SearchEventConverter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SearchEventConverter.java index 126d37ba1ace..978ae2ca9230 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SearchEventConverter.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SearchEventConverter.java @@ -13,7 +13,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.model.PageRest; import org.dspace.app.rest.model.SearchEventRest; import org.dspace.app.rest.model.SearchResultsRest; @@ -31,7 +32,7 @@ @Component public class SearchEventConverter { /* Log4j logger */ - private static final Logger log = Logger.getLogger(SearchEventConverter.class); + private static final Logger log = LogManager.getLogger(SearchEventConverter.class); @Autowired private ScopeResolver scopeResolver; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamLinksetProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamLinksetProcessor.java index c65191cb0749..c9ee193b3536 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamLinksetProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamLinksetProcessor.java @@ -10,7 +10,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Bitstream; @@ -25,7 +26,7 @@ */ public class BitstreamLinksetProcessor extends BitstreamSignpostingProcessor { - private static final Logger log = Logger.getLogger(BitstreamLinksetProcessor.class); + private static final Logger log = LogManager.getLogger(BitstreamLinksetProcessor.class); private final BitstreamService bitstreamService; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamParentItemProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamParentItemProcessor.java index 815d7817d4cf..beb318a73481 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamParentItemProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamParentItemProcessor.java @@ -10,7 +10,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Bitstream; @@ -28,7 +29,7 @@ */ public class BitstreamParentItemProcessor extends BitstreamSignpostingProcessor { - private static final Logger log = Logger.getLogger(BitstreamParentItemProcessor.class); + private static final Logger log = LogManager.getLogger(BitstreamParentItemProcessor.class); private final BitstreamService bitstreamService; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamTypeProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamTypeProcessor.java index 005a8009836d..1dabf398da9b 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamTypeProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/bitstream/BitstreamTypeProcessor.java @@ -11,7 +11,8 @@ import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Bitstream; @@ -28,7 +29,7 @@ */ public class BitstreamTypeProcessor extends BitstreamSignpostingProcessor { - private static final Logger log = Logger.getLogger(BitstreamTypeProcessor.class); + private static final Logger log = LogManager.getLogger(BitstreamTypeProcessor.class); @Autowired private SimpleMapConverter mapConverterDSpaceToSchemaOrgUri; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemAuthorProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemAuthorProcessor.java index 1bb215c46864..b96a41b00b2b 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemAuthorProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemAuthorProcessor.java @@ -16,7 +16,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Item; @@ -37,7 +38,7 @@ public class ItemAuthorProcessor extends ItemSignpostingProcessor { /** * log4j category */ - private static final Logger log = Logger.getLogger(ItemAuthorProcessor.class); + private static final Logger log = LogManager.getLogger(ItemAuthorProcessor.class); private final ItemService itemService; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemContentBitstreamsProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemContentBitstreamsProcessor.java index 61bf371adbdf..40c3d96cf680 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemContentBitstreamsProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemContentBitstreamsProcessor.java @@ -11,7 +11,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Bitstream; @@ -33,7 +34,7 @@ public class ItemContentBitstreamsProcessor extends ItemSignpostingProcessor { /** * log4j category */ - private static final Logger log = Logger.getLogger(ItemContentBitstreamsProcessor.class); + private static final Logger log = LogManager.getLogger(ItemContentBitstreamsProcessor.class); public ItemContentBitstreamsProcessor(FrontendUrlService frontendUrlService) { super(frontendUrlService); diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemDescribedbyProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemDescribedbyProcessor.java index a16770c4d103..62374d7ee830 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemDescribedbyProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemDescribedbyProcessor.java @@ -10,7 +10,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Item; @@ -23,7 +24,7 @@ */ public class ItemDescribedbyProcessor extends ItemSignpostingProcessor { - private static final Logger log = Logger.getLogger(ItemDescribedbyProcessor.class); + private static final Logger log = LogManager.getLogger(ItemDescribedbyProcessor.class); private final ConfigurationService configurationService; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemLicenseProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemLicenseProcessor.java index 1a26fa7695b1..a88e9eba37d2 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemLicenseProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemLicenseProcessor.java @@ -11,7 +11,8 @@ import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Item; @@ -25,7 +26,7 @@ */ public class ItemLicenseProcessor extends ItemSignpostingProcessor { - private static final Logger log = Logger.getLogger(ItemLicenseProcessor.class); + private static final Logger log = LogManager.getLogger(ItemLicenseProcessor.class); private final CreativeCommonsService creativeCommonsService = LicenseServiceFactory.getInstance().getCreativeCommonsService(); diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemLinksetProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemLinksetProcessor.java index 9008a28e29a6..1c765047a62d 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemLinksetProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemLinksetProcessor.java @@ -10,7 +10,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Item; @@ -23,7 +24,7 @@ */ public class ItemLinksetProcessor extends ItemSignpostingProcessor { - private static final Logger log = Logger.getLogger(ItemLinksetProcessor.class); + private static final Logger log = LogManager.getLogger(ItemLinksetProcessor.class); private final ConfigurationService configurationService; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemTypeProcessor.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemTypeProcessor.java index ddd2da12d59a..6945c33619f7 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemTypeProcessor.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/processor/item/ItemTypeProcessor.java @@ -11,7 +11,8 @@ import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.model.LinksetRelationType; import org.dspace.content.Item; @@ -27,7 +28,7 @@ */ public class ItemTypeProcessor extends ItemSignpostingProcessor { - private static final Logger log = Logger.getLogger(ItemTypeProcessor.class); + private static final Logger log = LogManager.getLogger(ItemTypeProcessor.class); private static final String ABOUT_PAGE_URI = "https://schema.org/AboutPage"; @Autowired diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/service/impl/LinksetServiceImpl.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/service/impl/LinksetServiceImpl.java index 399b7bd1e6b0..de5556173557 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/service/impl/LinksetServiceImpl.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/signposting/service/impl/LinksetServiceImpl.java @@ -13,7 +13,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.security.BitstreamMetadataReadPermissionEvaluatorPlugin; import org.dspace.app.rest.signposting.model.LinksetNode; import org.dspace.app.rest.signposting.processor.bitstream.BitstreamSignpostingProcessor; @@ -37,7 +38,7 @@ @Service public class LinksetServiceImpl implements LinksetService { - private static final Logger log = Logger.getLogger(LinksetServiceImpl.class); + private static final Logger log = LogManager.getLogger(LinksetServiceImpl.class); @Autowired protected ItemService itemService; diff --git a/dspace-sword/pom.xml b/dspace-sword/pom.xml index 01aa68dfbfe6..b72bee24db57 100644 --- a/dspace-sword/pom.xml +++ b/dspace-sword/pom.xml @@ -101,10 +101,6 @@ org.apache.logging.log4j log4j-core - - org.apache.logging.log4j - log4j-web - xom diff --git a/dspace-swordv2/pom.xml b/dspace-swordv2/pom.xml index f575898c122a..be9430b4d123 100644 --- a/dspace-swordv2/pom.xml +++ b/dspace-swordv2/pom.xml @@ -97,10 +97,6 @@ org.apache.logging.log4j log4j-core - - org.apache.logging.log4j - log4j-web - + + + + + ${log4j:configParentLocation}/../log + + + + + + + + + yyyy-MM-dd + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index b0d9d1819d3f..9b4dcef37dcf 100644 --- a/pom.xml +++ b/pom.xml @@ -1577,36 +1577,11 @@ log4j-api ${log4j.version} - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - org.apache.logging.log4j log4j-core ${log4j.version} - - org.apache.logging.log4j - log4j-web - ${log4j.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - - org.apache.logging.log4j - log4j-jul - ${log4j.version} - - - org.slf4j - jul-to-slf4j - ${slf4j.version} - org.apache.pdfbox From 9f8240987b71f1f37d4f791854b85bcbe870b032 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 20 Nov 2024 12:45:05 -0600 Subject: [PATCH 029/178] Remove unused dependencies from several modules --- dspace-services/pom.xml | 5 ----- dspace-sword/pom.xml | 18 ------------------ dspace/modules/server/pom.xml | 5 ----- pom.xml | 14 +------------- 4 files changed, 1 insertion(+), 41 deletions(-) diff --git a/dspace-services/pom.xml b/dspace-services/pom.xml index 7c85ff7f0f0f..6504ae2e4b2a 100644 --- a/dspace-services/pom.xml +++ b/dspace-services/pom.xml @@ -131,10 +131,6 @@ 6.1.26 test - - org.apache.commons - commons-collections4 - org.apache.commons commons-configuration2 @@ -159,6 +155,5 @@ spring-boot-starter-log4j2 ${spring-boot.version} - diff --git a/dspace-sword/pom.xml b/dspace-sword/pom.xml index b72bee24db57..12325533c5c5 100644 --- a/dspace-sword/pom.xml +++ b/dspace-sword/pom.xml @@ -25,15 +25,6 @@ - - - - org.dspace dspace-api @@ -68,11 +59,6 @@ - - jaxen - jaxen - - commons-fileupload @@ -107,10 +93,6 @@ xom 1.3.9 - - commons-io - commons-io - diff --git a/dspace/modules/server/pom.xml b/dspace/modules/server/pom.xml index 5134ffb94733..cdbec2f60c8b 100644 --- a/dspace/modules/server/pom.xml +++ b/dspace/modules/server/pom.xml @@ -262,11 +262,6 @@ just adding new jar in the classloader dspace-server-webapp war - - org.apache.solr - solr-solrj - ${solr.client.version} - diff --git a/pom.xml b/pom.xml index 9b4dcef37dcf..c0127724c1d5 100644 --- a/pom.xml +++ b/pom.xml @@ -1179,11 +1179,7 @@ - - org.swordapp - sword-common - 1.1 - + spring-core @@ -1484,14 +1480,6 @@ commons-collections4 4.4 - - - commons-collections - commons-collections - 3.2.2 - org.apache.commons commons-configuration2 From f44dba60cba8cacaa785e9bc5329b569f77a0c1e Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 22 Nov 2024 11:04:11 -0600 Subject: [PATCH 030/178] Enable all optional modules/controllers to test their deployment in Spring Boot (cherry picked from commit 98768d6f4fcc5181501d02d77ef5056641ac8cd3) --- .github/workflows/docker.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 143b69a2f9fe..7013f1e71a09 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -161,6 +161,14 @@ jobs: env: # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080 dspace__P__server__P__url: http://127.0.0.1:8080/server + # Enable all optional modules / controllers for this test deployment. + # This helps check for errors in deploying these modules via Spring Boot + iiif__P__enabled: true + oai__P__enabled: true + rdf__P__enabled: true + signposting__P__enabled: true + sword-server__P__enabled: true + swordv2-server__P__enabled: true # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, if on main branch, use the # "latest" tag. Otherwise, use the branch name. NOTE: the "pr-testing" tag is a temporary tag that we assign to # all PR-built docker images in reusabe-docker-build.yml From 66a9782eeefc4e9211a4efdd1a10b1303640415e Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 22 Nov 2024 13:50:16 -0600 Subject: [PATCH 031/178] Fix syntax error in #10040. Env variables cannot have dashes or periods --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7013f1e71a09..daa940f215a1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -167,8 +167,8 @@ jobs: oai__P__enabled: true rdf__P__enabled: true signposting__P__enabled: true - sword-server__P__enabled: true - swordv2-server__P__enabled: true + sword__D__server__P__enabled: true + swordv2__D__server__P__enabled: true # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, if on main branch, use the # "latest" tag. Otherwise, use the branch name. NOTE: the "pr-testing" tag is a temporary tag that we assign to # all PR-built docker images in reusabe-docker-build.yml From cf99694a84714221ed506fa9afd54f262c11c1ce Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 22 Nov 2024 14:58:08 -0600 Subject: [PATCH 032/178] Fix startup errors for SWORDv2. Requires the log4jv1->v2 bridge to be installed. --- dspace-swordv2/pom.xml | 6 ++++++ pom.xml | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/dspace-swordv2/pom.xml b/dspace-swordv2/pom.xml index be9430b4d123..7cae05672470 100644 --- a/dspace-swordv2/pom.xml +++ b/dspace-swordv2/pom.xml @@ -97,6 +97,12 @@ org.apache.logging.log4j log4j-core + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + org.apache.pdfbox From 54a1c75cbc652bbb60e39157450043087c2d822b Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 28 Nov 2024 13:06:23 +0100 Subject: [PATCH 033/178] 109807: ArXiv mapping fix - author/name to dc.contributor.author https://info.arxiv.org/help/api/basics.html#using --- dspace/config/spring/api/arxiv-integration.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace/config/spring/api/arxiv-integration.xml b/dspace/config/spring/api/arxiv-integration.xml index e963e73a2055..3f7f408dff40 100644 --- a/dspace/config/spring/api/arxiv-integration.xml +++ b/dspace/config/spring/api/arxiv-integration.xml @@ -94,7 +94,7 @@ - + From 38a71cc664fffc5e6bfd7809cc95960a552750eb Mon Sep 17 00:00:00 2001 From: Alexandre Vryghem Date: Wed, 30 Oct 2024 13:37:53 +0100 Subject: [PATCH 034/178] 119960: Fixed NPE when retrieving a DSpace object with the api/dso/find endpoint without the required permissions (cherry picked from commit dd8b1d91cb74c5afadc1e545b2192b5ebb8848b1) --- .../app/rest/UUIDLookupRestController.java | 10 ++++- .../app/rest/UUIDLookupRestControllerIT.java | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/UUIDLookupRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/UUIDLookupRestController.java index 40c0a79b97be..21631a6737a9 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/UUIDLookupRestController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/UUIDLookupRestController.java @@ -24,9 +24,11 @@ import org.dspace.app.rest.utils.ContextUtil; import org.dspace.app.rest.utils.DSpaceObjectUtils; import org.dspace.app.rest.utils.Utils; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.service.AuthorizeService; import org.dspace.content.DSpaceObject; +import org.dspace.core.Constants; import org.dspace.core.Context; -import org.dspace.discovery.SearchServiceException; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.hateoas.Link; @@ -65,6 +67,9 @@ public class UUIDLookupRestController implements InitializingBean { @Autowired private DiscoverableEndpointsService discoverableEndpointsService; + @Autowired + private AuthorizeService authorizeService; + @Autowired private ConverterService converter; @@ -85,13 +90,14 @@ public void afterPropertiesSet() throws Exception { public void getDSObyIdentifier(HttpServletRequest request, HttpServletResponse response, @RequestParam(PARAM) UUID uuid) - throws IOException, SQLException, SearchServiceException { + throws IOException, SQLException, AuthorizeException { Context context = null; try { context = ContextUtil.obtainContext(request); DSpaceObject dso = dspaceObjectUtil.findDSpaceObject(context, uuid); if (dso != null) { + authorizeService.authorizeAction(context, dso, Constants.READ); DSpaceObjectRest dsor = converter.toRest(dso, utils.obtainProjection()); URI link = linkTo(dsor.getController(), dsor.getCategory(), dsor.getTypePlural()).slash(dsor.getId()) .toUri(); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/UUIDLookupRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/UUIDLookupRestControllerIT.java index 8a6debce3ec7..3b0821645861 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/UUIDLookupRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/UUIDLookupRestControllerIT.java @@ -17,6 +17,8 @@ import org.apache.commons.codec.CharEncoding; import org.apache.commons.io.IOUtils; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.authorize.ResourcePolicy; +import org.dspace.authorize.service.ResourcePolicyService; import org.dspace.builder.BitstreamBuilder; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; @@ -31,6 +33,7 @@ import org.dspace.eperson.Group; import org.junit.Ignore; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; /** * Integration test for the UUIDLookup endpoint @@ -39,6 +42,9 @@ */ public class UUIDLookupRestControllerIT extends AbstractControllerIntegrationTest { + @Autowired + ResourcePolicyService resourcePolicyService; + @Test /** * Test the proper redirection of a site's uuid @@ -307,4 +313,35 @@ public void testMissingIdentifierParameter() throws Exception { .andExpect(status().isUnprocessableEntity()); } + @Test + public void testUnauthorized() throws Exception { + context.turnOffAuthorisationSystem(); + Community community = CommunityBuilder.createCommunity(context) + .build(); + for (ResourcePolicy rp : resourcePolicyService.find(context, community)) { + resourcePolicyService.delete(context, rp); + } + context.restoreAuthSystemState(); + + getClient().perform(get("/api/dso/find") + .param("uuid", community.getID().toString())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void testForbidden() throws Exception { + context.turnOffAuthorisationSystem(); + Community community = CommunityBuilder.createCommunity(context) + .build(); + for (ResourcePolicy rp : resourcePolicyService.find(context, community)) { + resourcePolicyService.delete(context, rp); + } + context.restoreAuthSystemState(); + + String authToken = getAuthToken(eperson.getEmail(), password); + getClient(authToken).perform(get("/api/dso/find") + .param("uuid", community.getID().toString())) + .andExpect(status().isForbidden()); + } + } From 2136dbf690389f881fc73373fda6be87b63249ab Mon Sep 17 00:00:00 2001 From: Nathan Buckingham Date: Tue, 4 Jun 2024 16:02:53 -0400 Subject: [PATCH 035/178] 110719: Port fix to checkLinks that works on redirects (cherry picked from commit 3dab2a7cea812420422af160d86371a59b48374f) --- .../ctask/general/BasicLinkChecker.java | 14 +++++-- .../impl/TestDSpaceRunnableHandler.java | 39 +++++++++++++++++ .../org/dspace/curate/CurationScriptIT.java | 42 +++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/ctask/general/BasicLinkChecker.java b/dspace-api/src/main/java/org/dspace/ctask/general/BasicLinkChecker.java index fbc6eebdb5b8..388036bd6744 100644 --- a/dspace-api/src/main/java/org/dspace/ctask/general/BasicLinkChecker.java +++ b/dspace-api/src/main/java/org/dspace/ctask/general/BasicLinkChecker.java @@ -132,10 +132,18 @@ protected int getResponseStatus(String url) { try { URL theURL = new URL(url); HttpURLConnection connection = (HttpURLConnection) theURL.openConnection(); - int code = connection.getResponseCode(); - connection.disconnect(); + connection.setInstanceFollowRedirects(true); + int statusCode = connection.getResponseCode(); + if ((statusCode == HttpURLConnection.HTTP_MOVED_TEMP || statusCode == HttpURLConnection.HTTP_MOVED_PERM || + statusCode == HttpURLConnection.HTTP_SEE_OTHER)) { + connection.disconnect(); + String newUrl = connection.getHeaderField("Location"); + if (newUrl != null) { + return getResponseStatus(newUrl); + } - return code; + } + return statusCode; } catch (IOException ioe) { // Must be a bad URL diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/scripts/handler/impl/TestDSpaceRunnableHandler.java b/dspace-server-webapp/src/test/java/org/dspace/app/scripts/handler/impl/TestDSpaceRunnableHandler.java index 1b5b3fa7ac1a..aced81cbdfdb 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/scripts/handler/impl/TestDSpaceRunnableHandler.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/scripts/handler/impl/TestDSpaceRunnableHandler.java @@ -7,6 +7,9 @@ */ package org.dspace.app.scripts.handler.impl; +import java.util.ArrayList; +import java.util.List; + import org.dspace.scripts.handler.impl.CommandLineDSpaceRunnableHandler; /** @@ -17,6 +20,12 @@ public class TestDSpaceRunnableHandler extends CommandLineDSpaceRunnableHandler private Exception exception = null; + private final List infoMessages = new ArrayList<>(); + + private final List errorMessages = new ArrayList<>(); + + private final List warningMessages = new ArrayList<>(); + /** * We're overriding this method so that we can stop the script from doing the System.exit() if * an exception within the script is thrown @@ -33,4 +42,34 @@ public void handleException(String message, Exception e) { public Exception getException() { return exception; } + + @Override + public void logInfo(String message) { + super.logInfo(message); + infoMessages.add(message); + } + + @Override + public void logWarning(String message) { + super.logWarning(message); + warningMessages.add(message); + } + + @Override + public void logError(String message) { + super.logError(message); + errorMessages.add(message); + } + + public List getInfoMessages() { + return infoMessages; + } + + public List getErrorMessages() { + return errorMessages; + } + + public List getWarningMessages() { + return warningMessages; + } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java index 3e40a8559482..6bb9acb92482 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java @@ -9,6 +9,7 @@ import static com.jayway.jsonpath.JsonPath.read; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -21,6 +22,7 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.databind.ObjectMapper; +import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.rest.converter.DSpaceRunnableParameterConverter; import org.dspace.app.rest.matcher.ProcessMatcher; import org.dspace.app.rest.model.ParameterValueRest; @@ -28,6 +30,7 @@ import org.dspace.app.rest.model.ScriptRest; import org.dspace.app.rest.projection.Projection; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; import org.dspace.builder.EPersonBuilder; @@ -41,7 +44,9 @@ import org.dspace.content.factory.ContentServiceFactory; import org.dspace.eperson.EPerson; import org.dspace.scripts.DSpaceCommandLineParameter; +import org.dspace.scripts.DSpaceRunnable; import org.dspace.scripts.configuration.ScriptConfiguration; +import org.dspace.scripts.factory.ScriptServiceFactory; import org.dspace.scripts.service.ScriptService; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -640,4 +645,41 @@ public void securityCurateTest() throws Exception { ProcessBuilder.deleteProcess(idItemRef.get()); } } + + @Test + public void testURLRedirectCurateTest() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + + Item publicItem1 = ItemBuilder.createItem(context, col1) + .withTitle("Public item 1") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald").withAuthor("Doe, John") + .withMetadata("dc", "identifier", "uri", "https://tinyurl.com/2rxpte5s") + .withSubject("ExtraEntry") + .build(); + + String[] args = new String[] {"curate", "-t", "checklinks", "-i", publicItem1.getHandle()}; + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + + ScriptService scriptService = ScriptServiceFactory.getInstance().getScriptService(); + ScriptConfiguration scriptConfiguration = scriptService.getScriptConfiguration(args[0]); + + DSpaceRunnable script = null; + if (scriptConfiguration != null) { + script = scriptService.createDSpaceRunnableForScriptConfiguration(scriptConfiguration); + } + if (script != null) { + script.initialize(args, testDSpaceRunnableHandler, admin); + script.run(); + } + + assertTrue(testDSpaceRunnableHandler.getInfoMessages().contains("200 - OK")); + } } From be905a088750911928b9515f36cc2a24fc2b518f Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 13 Jun 2024 20:48:31 +0200 Subject: [PATCH 036/178] 110719: IT checking redirect links accepted by checklinks curate task (cherry picked from commit e826660cb0bd926de390e04e466d54ceedb67a1e) --- .../org/dspace/curate/CurationScriptIT.java | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java index 6bb9acb92482..8c0744a09cce 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java @@ -9,6 +9,7 @@ import static com.jayway.jsonpath.JsonPath.read; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -22,7 +23,7 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.databind.ObjectMapper; -import org.dspace.app.launcher.ScriptLauncher; +import org.apache.commons.lang3.StringUtils; import org.dspace.app.rest.converter.DSpaceRunnableParameterConverter; import org.dspace.app.rest.matcher.ProcessMatcher; import org.dspace.app.rest.model.ParameterValueRest; @@ -661,12 +662,19 @@ public void testURLRedirectCurateTest() throws Exception { .withTitle("Public item 1") .withIssueDate("2017-10-17") .withAuthor("Smith, Donald").withAuthor("Doe, John") - .withMetadata("dc", "identifier", "uri", "https://tinyurl.com/2rxpte5s") + // Value not starting with http or https + .withMetadata("dc", "identifier", "uri", "demo.dspace.org/home") + // MetadataValueLinkChecker uri field with regular link + .withMetadata("dc", "description", null, "https://google.com") + // MetadataValueLinkChecker uri field with redirect link + .withMetadata("dc", "description", "uri", "https://demo7.dspace.org/handle/123456789/1") + // MetadataValueLinkChecker uri field with non resolving link + .withMetadata("dc", "description", "uri", "https://www.atmire.com/broken-link") .withSubject("ExtraEntry") .build(); String[] args = new String[] {"curate", "-t", "checklinks", "-i", publicItem1.getHandle()}; - TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); ScriptService scriptService = ScriptServiceFactory.getInstance().getScriptService(); ScriptConfiguration scriptConfiguration = scriptService.getScriptConfiguration(args[0]); @@ -676,10 +684,27 @@ public void testURLRedirectCurateTest() throws Exception { script = scriptService.createDSpaceRunnableForScriptConfiguration(scriptConfiguration); } if (script != null) { - script.initialize(args, testDSpaceRunnableHandler, admin); + script.initialize(args, handler, admin); script.run(); } - assertTrue(testDSpaceRunnableHandler.getInfoMessages().contains("200 - OK")); + // field that should be ignored + assertFalse(checkIfInfoTextLoggedByHandler(handler, "demo.dspace.org/home")); + // redirect links in field that should not be ignored (https) => expect OK + assertTrue(checkIfInfoTextLoggedByHandler(handler, "https://demo7.dspace.org/handle/123456789/1 = 200 - OK")); + // regular link in field that should not be ignored (http) => expect OK + assertTrue(checkIfInfoTextLoggedByHandler(handler, "https://google.com = 200 - OK")); + // nonexistent link in field that should not be ignored => expect 404 + assertTrue(checkIfInfoTextLoggedByHandler(handler, "https://www.atmire.com/broken-link = 404 - FAILED")); } + + boolean checkIfInfoTextLoggedByHandler(TestDSpaceRunnableHandler handler, String text) { + for (String message: handler.getInfoMessages()) { + if (StringUtils.containsIgnoreCase(message, text)) { + return true; + } + } + return false; + } + } From 3ff38ec4b44acbae43292b0b26623623aac64200 Mon Sep 17 00:00:00 2001 From: Nathan Buckingham Date: Tue, 12 Nov 2024 15:59:17 -0500 Subject: [PATCH 037/178] 115778: Adjust redirect check to only follow a limited amount defined in curate.cfg (cherry picked from commit ef381aa1516df0d45227cd6c752b32658a69509b) --- .../dspace/ctask/general/BasicLinkChecker.java | 16 ++++++++++++---- dspace/config/modules/curate.cfg | 3 +++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/ctask/general/BasicLinkChecker.java b/dspace-api/src/main/java/org/dspace/ctask/general/BasicLinkChecker.java index 388036bd6744..a302159ea9a4 100644 --- a/dspace-api/src/main/java/org/dspace/ctask/general/BasicLinkChecker.java +++ b/dspace-api/src/main/java/org/dspace/ctask/general/BasicLinkChecker.java @@ -19,6 +19,8 @@ import org.dspace.content.MetadataValue; import org.dspace.curate.AbstractCurationTask; import org.dspace.curate.Curator; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; /** * A basic link checker that is designed to be extended. By default this link checker @@ -42,6 +44,9 @@ public class BasicLinkChecker extends AbstractCurationTask { // The log4j logger for this class private static Logger log = org.apache.logging.log4j.LogManager.getLogger(BasicLinkChecker.class); + protected static final ConfigurationService configurationService + = DSpaceServicesFactory.getInstance().getConfigurationService(); + /** * Perform the link checking. @@ -110,7 +115,8 @@ protected List getURLs(Item item) { */ protected boolean checkURL(String url, StringBuilder results) { // Link check the URL - int httpStatus = getResponseStatus(url); + int redirects = 0; + int httpStatus = getResponseStatus(url, redirects); if ((httpStatus >= 200) && (httpStatus < 300)) { results.append(" - " + url + " = " + httpStatus + " - OK\n"); @@ -128,18 +134,20 @@ protected boolean checkURL(String url, StringBuilder results) { * @param url The url to open * @return The HTTP response code (e.g. 200 / 301 / 404 / 500) */ - protected int getResponseStatus(String url) { + protected int getResponseStatus(String url, int redirects) { try { URL theURL = new URL(url); HttpURLConnection connection = (HttpURLConnection) theURL.openConnection(); connection.setInstanceFollowRedirects(true); int statusCode = connection.getResponseCode(); + int maxRedirect = configurationService.getIntProperty("curate.checklinks.max-redirect", 0); if ((statusCode == HttpURLConnection.HTTP_MOVED_TEMP || statusCode == HttpURLConnection.HTTP_MOVED_PERM || statusCode == HttpURLConnection.HTTP_SEE_OTHER)) { connection.disconnect(); String newUrl = connection.getHeaderField("Location"); - if (newUrl != null) { - return getResponseStatus(newUrl); + if (newUrl != null && (maxRedirect >= redirects || maxRedirect == -1)) { + redirects++; + return getResponseStatus(newUrl, redirects); } } diff --git a/dspace/config/modules/curate.cfg b/dspace/config/modules/curate.cfg index 1d7b87960df1..6e75738de543 100644 --- a/dspace/config/modules/curate.cfg +++ b/dspace/config/modules/curate.cfg @@ -26,3 +26,6 @@ curate.taskqueue.dir = ${dspace.dir}/ctqueues # (optional) directory location of scripted (non-java) tasks # curate.script.dir = ${dspace.dir}/ctscripts + +# Maximum amount of redirects set to 0 for none and -1 for unlimited +curate.checklinks.max-redirect = 0 From 25fb8111f1aa18cd45b331d169dce32789a97068 Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Tue, 23 Jul 2024 18:28:43 +0200 Subject: [PATCH 038/178] 116609: Improve running process observability - keep temp process log files in [dspace]/log/processes/ instead of temp dir - reformat file names of process logs - ensure that running and scheduled processes are cleaned up during startup (cherry picked from commit d80f49e0238981c22703abbb1423aa2896eca694) --- .../dspace/scripts/ProcessServiceImpl.java | 56 +++++++++++++++---- .../app/rest/ProcessRestRepositoryIT.java | 8 +-- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java index f242ec7acdad..f48fd02a920a 100644 --- a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java @@ -45,14 +45,15 @@ import org.dspace.core.LogHelper; import org.dspace.eperson.EPerson; import org.dspace.eperson.Group; -import org.dspace.eperson.service.EPersonService; import org.dspace.scripts.service.ProcessService; +import org.dspace.services.ConfigurationService; +import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; /** * The implementation for the {@link ProcessService} class */ -public class ProcessServiceImpl implements ProcessService { +public class ProcessServiceImpl implements ProcessService, InitializingBean { private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(ProcessService.class); @@ -72,7 +73,26 @@ public class ProcessServiceImpl implements ProcessService { private MetadataFieldService metadataFieldService; @Autowired - private EPersonService ePersonService; + private ConfigurationService configurationService; + + @Override + public void afterPropertiesSet() throws Exception { + Context context = new Context(); + + // Processes that were running or scheduled when tomcat crashed, should be cleaned up during startup. + List processesToBeFailed = findByStatusAndCreationTimeOlderThan( + context, List.of(ProcessStatus.RUNNING, ProcessStatus.SCHEDULED), new Date()); + for (Process process : processesToBeFailed) { + context.setCurrentUser(process.getEPerson()); + // Fail the process. + log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", process.getID()); + fail(context, process); + // But still attach its log to the process. + createLogBitstream(context, process); + } + + context.complete(); + } @Override public Process create(Context context, EPerson ePerson, String scriptName, @@ -293,8 +313,8 @@ public int countSearch(Context context, ProcessQueryParameterContainer processQu @Override public void appendLog(int processId, String scriptName, String output, ProcessLogLevel processLogLevel) throws IOException { - File tmpDir = FileUtils.getTempDirectory(); - File tempFile = new File(tmpDir, scriptName + processId + ".log"); + File logsDir = getLogsDirectory(); + File tempFile = new File(logsDir, processId + "-" + scriptName + ".log"); FileWriter out = new FileWriter(tempFile, true); try { try (BufferedWriter writer = new BufferedWriter(out)) { @@ -309,12 +329,15 @@ public void appendLog(int processId, String scriptName, String output, ProcessLo @Override public void createLogBitstream(Context context, Process process) throws IOException, SQLException, AuthorizeException { - File tmpDir = FileUtils.getTempDirectory(); - File tempFile = new File(tmpDir, process.getName() + process.getID() + ".log"); - FileInputStream inputStream = FileUtils.openInputStream(tempFile); - appendFile(context, process, inputStream, Process.OUTPUT_TYPE, process.getName() + process.getID() + ".log"); - inputStream.close(); - tempFile.delete(); + File logsDir = getLogsDirectory(); + File tempFile = new File(logsDir, process.getID() + "-" + process.getName() + ".log"); + if (tempFile.exists()) { + FileInputStream inputStream = FileUtils.openInputStream(tempFile); + appendFile(context, process, inputStream, Process.OUTPUT_TYPE, + process.getID() + "-" + process.getName() + ".log"); + inputStream.close(); + tempFile.delete(); + } } @Override @@ -343,4 +366,15 @@ private String formatLogLine(int processId, String scriptName, String output, Pr return sb.toString(); } + private File getLogsDirectory() { + String pathStr = configurationService.getProperty("dspace.dir") + + File.separator + "log" + File.separator + "processes"; + File logsDir = new File(pathStr); + if (!logsDir.exists()) { + if (!logsDir.mkdirs()) { + throw new RuntimeException("Couldn't create [dspace.dir]/log/processes/ directory."); + } + } + return logsDir; + } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java index 670d8e2f35b0..6c018df6d070 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java @@ -856,10 +856,10 @@ public void getProcessOutput() throws Exception { getClient(token).perform(get("/api/system/processes/" + process1.getID() + "/output")) .andExpect(status().isOk()) .andExpect(jsonPath("$.name", - is(process1.getName() + process1.getID() + ".log"))) + is(process1.getID() + "-" + process1.getName() + ".log"))) .andExpect(jsonPath("$.type", is("bitstream"))) .andExpect(jsonPath("$.metadata['dc.title'][0].value", - is(process1.getName() + process1.getID() + ".log"))) + is(process1.getID() + "-" + process1.getName() + ".log"))) .andExpect(jsonPath("$.metadata['dspace.process.filetype'][0].value", is("script_output"))); @@ -869,10 +869,10 @@ public void getProcessOutput() throws Exception { .perform(get("/api/system/processes/" + process1.getID() + "/output")) .andExpect(status().isOk()) .andExpect(jsonPath("$.name", - is(process1.getName() + process1.getID() + ".log"))) + is(process1.getID() + "-" + process1.getName() + ".log"))) .andExpect(jsonPath("$.type", is("bitstream"))) .andExpect(jsonPath("$.metadata['dc.title'][0].value", - is(process1.getName() + process1.getID() + ".log"))) + is(process1.getID() + "-" + process1.getName() + ".log"))) .andExpect(jsonPath("$.metadata['dspace.process.filetype'][0].value", is("script_output"))); From 1ba7ca3bdda705d26d7dbccf33aa19933e810bde Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Thu, 25 Jul 2024 09:33:50 +0200 Subject: [PATCH 039/178] 116609: Add tomcat shutdown line to process log (cherry picked from commit 156ad471b575dfc1c2b643ec4b2f677d6f89a314) --- .../src/main/java/org/dspace/scripts/ProcessServiceImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java index f48fd02a920a..c34705a64d5d 100644 --- a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java @@ -88,6 +88,9 @@ public void afterPropertiesSet() throws Exception { log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", process.getID()); fail(context, process); // But still attach its log to the process. + appendLog(process.getID(), process.getName(), + "Process did not complete before tomcat shutdown.", + ProcessLogLevel.ERROR); createLogBitstream(context, process); } From 3f836ae7a58a635163ca6a3a0a62cf9c514700ad Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Thu, 25 Jul 2024 14:13:16 +0200 Subject: [PATCH 040/178] 116687: Never handle exception with null message (cherry picked from commit bdf7069cb752782a4f8df62885c9ae6116be9dde) --- .../rest/scripts/handler/impl/RestDSpaceRunnableHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java index 596ab4429093..ee67baa8ab38 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java @@ -130,7 +130,7 @@ public void handleCompletion() { @Override public void handleException(Exception e) { - handleException(null, e); + handleException(e.getMessage(), e); } @Override From 8bc100375af69d02ea4ead19561f80effbf6362e Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Mon, 29 Jul 2024 13:48:32 +0200 Subject: [PATCH 041/178] 116609: Add try catch to init method in ProcessServiceImpl (cherry picked from commit 070fe689d70d8dbfcde63899d377f2524f38d6db) --- .../dspace/scripts/ProcessServiceImpl.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java index c34705a64d5d..ab5147221cfc 100644 --- a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java @@ -77,24 +77,29 @@ public class ProcessServiceImpl implements ProcessService, InitializingBean { @Override public void afterPropertiesSet() throws Exception { - Context context = new Context(); - - // Processes that were running or scheduled when tomcat crashed, should be cleaned up during startup. - List processesToBeFailed = findByStatusAndCreationTimeOlderThan( - context, List.of(ProcessStatus.RUNNING, ProcessStatus.SCHEDULED), new Date()); - for (Process process : processesToBeFailed) { - context.setCurrentUser(process.getEPerson()); - // Fail the process. - log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", process.getID()); - fail(context, process); - // But still attach its log to the process. - appendLog(process.getID(), process.getName(), - "Process did not complete before tomcat shutdown.", - ProcessLogLevel.ERROR); - createLogBitstream(context, process); - } + try { + Context context = new Context(); + + // Processes that were running or scheduled when tomcat crashed, should be cleaned up during startup. + List processesToBeFailed = findByStatusAndCreationTimeOlderThan( + context, List.of(ProcessStatus.RUNNING, ProcessStatus.SCHEDULED), new Date()); + for (Process process : processesToBeFailed) { + context.setCurrentUser(process.getEPerson()); + // Fail the process. + log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", + process.getID()); + fail(context, process); + // But still attach its log to the process. + appendLog(process.getID(), process.getName(), + "Process did not complete before tomcat shutdown.", + ProcessLogLevel.ERROR); + createLogBitstream(context, process); + } - context.complete(); + context.complete(); + } catch (Exception e) { + log.error("Unable to clean up Processes: ", e); + } } @Override From f8bf278f06af860fe47c92947f15f9be7a68326a Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 3 Dec 2024 12:07:18 -0600 Subject: [PATCH 042/178] Ensure login occurs *before* setup-buildx, as some buildx commands appear to be unauthenticated. --- .github/workflows/reusable-docker-build.yml | 36 ++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index 83dfd74b9a1a..d5964454a538 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -105,29 +105,29 @@ jobs: - name: Checkout codebase uses: actions/checkout@v4 - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 + # https://github.com/docker/login-action + - name: Login to DockerHub + # Only login if not a PR, as PRs only trigger a Docker build and not a push + if: ${{ ! matrix.isPr }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} # https://github.com/docker/setup-qemu-action - name: Set up QEMU emulation to build for multiple architectures uses: docker/setup-qemu-action@v3 + # https://github.com/docker/setup-buildx-action + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + #------------------------------------------------------------ # Build & deploy steps for new commits to a branch (non-PRs) # # These steps build the images, push to DockerHub, and # (if necessary) redeploy demo/sandbox sites. #------------------------------------------------------------ - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: ${{ ! matrix.isPr }} - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - # https://github.com/docker/metadata-action # Get Metadata for docker_build_deps step below - name: Sync metadata (tags, labels) from GitHub to Docker for image @@ -257,6 +257,12 @@ jobs: pattern: digests-${{ inputs.build_id }}-* merge-multiple: true + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -268,12 +274,6 @@ jobs: tags: ${{ env.IMAGE_TAGS }} flavor: ${{ env.TAGS_FLAVOR }} - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - name: Create manifest list from digests and push working-directory: /tmp/digests run: | From 8d34f06396328c888ff3695230a92ceb9d181eab Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Thu, 5 Dec 2024 10:39:34 -0600 Subject: [PATCH 043/178] Update Actions to always deploy the locally created image. non-PRs do not need to redownload images from DockerHub. --- .github/workflows/docker.yml | 24 ++++++++++----------- .github/workflows/reusable-docker-build.yml | 21 ++++++++++++++---- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index daa940f215a1..080ffc1c080b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -149,9 +149,9 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} - ######################################################################## - # Test Deployment via Docker to ensure images are working properly - ######################################################################## + ################################################################################# + # Test Deployment via Docker to ensure newly built images are working properly + ################################################################################# docker-deploy: # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' if: github.repository == 'dspace/dspace' @@ -171,25 +171,23 @@ jobs: swordv2__D__server__P__enabled: true # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, if on main branch, use the # "latest" tag. Otherwise, use the branch name. NOTE: the "pr-testing" tag is a temporary tag that we assign to - # all PR-built docker images in reusabe-docker-build.yml + # all PR-built docker images in reusable-docker-build.yml DSPACE_VER: ${{ (github.event_name == 'pull_request' && 'pr-testing') || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase uses: actions/checkout@v4 - # For PRs, download Docker image artifacts (built by reusable-docker-build.yml for all PRs) - - name: Download Docker image artifacts (for PRs) - if: github.event_name == 'pull_request' + # Download Docker image artifacts (which were just built by reusable-docker-build.yml) + - name: Download Docker image artifacts uses: actions/download-artifact@v4 with: - # Download all Docker images (TAR files) into the /tmp/docker directory - pattern: docker-image-* + # Download all amd64 Docker images (TAR files) into the /tmp/docker directory + pattern: docker-image-*-linux-amd64 path: /tmp/docker merge-multiple: true - # For PRs, load each of the images into Docker by calling "docker image load" for each. - # This ensures we are using the images built from this PR & not the prior versions on DockerHub - - name: Load all downloaded Docker images (for PRs) - if: github.event_name == 'pull_request' + # Load each of the images into Docker by calling "docker image load" for each. + # This ensures we are using the images just built & not any prior versions on DockerHub + - name: Load all downloaded Docker images run: | find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \; docker image ls -a diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index d5964454a538..e6f24bb64f40 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -130,7 +130,7 @@ jobs: #------------------------------------------------------------ # https://github.com/docker/metadata-action # Get Metadata for docker_build_deps step below - - name: Sync metadata (tags, labels) from GitHub to Docker for image + - name: Extract metadata (tags, labels) from GitHub for Docker image if: ${{ ! matrix.isPr }} id: meta_build uses: docker/metadata-action@v5 @@ -151,10 +151,23 @@ jobs: context: ${{ inputs.dockerfile_context }} file: ${{ inputs.dockerfile_path }} platforms: ${{ matrix.arch }} - push: true # Use tags / labels provided by 'docker/metadata-action' above tags: ${{ steps.meta_build.outputs.tags }} labels: ${{ steps.meta_build.outputs.labels }} + # Export image to both Docker registry & to a local TAR file + outputs: | + registry + type=docker,dest=/tmp/${{ inputs.build_id }}.tar + + # Upload the local docker image (in TAR file) to a build Artifact + - name: Upload local image to artifact + if: ${{ ! matrix.isPr }} + uses: actions/upload-artifact@v4 + with: + name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} + path: /tmp/${{ inputs.build_id }}.tar + if-no-files-found: error + retention-days: 1 # Export the digest of Docker build locally (for non PRs only) - name: Export Docker build digest @@ -204,7 +217,7 @@ jobs: # brand-new images for automated testing. #-------------------------------------------------------------- # Get Metadata for docker_build_deps step below - - name: Create metadata (tags, labels) for local Docker image + - name: Extract metadata (tags, labels) for local Docker image if: matrix.isPr id: meta_build_pr uses: docker/metadata-action@v5 @@ -233,7 +246,7 @@ jobs: if: matrix.isPr uses: actions/upload-artifact@v4 with: - name: docker-image-${{ inputs.build_id }} + name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} path: /tmp/${{ inputs.build_id }}.tar if-no-files-found: error retention-days: 1 From 1f5defe643bdcf27cca1ea51a00d3a78b937124b Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Thu, 5 Dec 2024 11:24:31 -0600 Subject: [PATCH 044/178] Ensure non-PRs run image build twice. First for a DockerHub image, then to export a local tarball. Since these are different image types they cannot be combined --- .github/workflows/reusable-docker-build.yml | 41 ++++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index e6f24bb64f40..b8f8d0f91718 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -151,23 +151,10 @@ jobs: context: ${{ inputs.dockerfile_context }} file: ${{ inputs.dockerfile_path }} platforms: ${{ matrix.arch }} + push: true # Use tags / labels provided by 'docker/metadata-action' above tags: ${{ steps.meta_build.outputs.tags }} labels: ${{ steps.meta_build.outputs.labels }} - # Export image to both Docker registry & to a local TAR file - outputs: | - registry - type=docker,dest=/tmp/${{ inputs.build_id }}.tar - - # Upload the local docker image (in TAR file) to a build Artifact - - name: Upload local image to artifact - if: ${{ ! matrix.isPr }} - uses: actions/upload-artifact@v4 - with: - name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} - path: /tmp/${{ inputs.build_id }}.tar - if-no-files-found: error - retention-days: 1 # Export the digest of Docker build locally (for non PRs only) - name: Export Docker build digest @@ -187,6 +174,32 @@ jobs: if-no-files-found: error retention-days: 1 + # Build local image (again) and store in a TAR file in /tmp directory + # NOTE: This cannot be combined with push to DockerHub registry above as it's a different type of output. + - name: Build and push image to local image + if: ${{ ! matrix.isPr }} + uses: docker/build-push-action@v5 + with: + build-contexts: | + ${{ inputs.dockerfile_additional_contexts }} + context: ${{ inputs.dockerfile_context }} + file: ${{ inputs.dockerfile_path }} + platforms: ${{ matrix.arch }} + tags: ${{ steps.meta_build.outputs.tags }} + labels: ${{ steps.meta_build.outputs.labels }} + # Export image to a local TAR file + outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar + + # Upload the local docker image (in TAR file) to a build Artifact + - name: Upload local image to artifact + if: ${{ ! matrix.isPr }} + uses: actions/upload-artifact@v4 + with: + name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} + path: /tmp/${{ inputs.build_id }}.tar + if-no-files-found: error + retention-days: 1 + # If this build is NOT a PR and passed in a REDEPLOY_SANDBOX_URL secret, # Then redeploy https://sandbox.dspace.org if this build is for our deployment architecture and 'main' branch. - name: Redeploy sandbox.dspace.org (based on main branch) From 0fc0f181fd8b49c1b6e278b4bf8aa5b7f8711332 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 6 Dec 2024 11:26:59 -0600 Subject: [PATCH 045/178] Enable caching of Docker builds using GitHub Actions cache --- .github/workflows/reusable-docker-build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index b8f8d0f91718..92dd08623c05 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -155,6 +155,10 @@ jobs: # Use tags / labels provided by 'docker/metadata-action' above tags: ${{ steps.meta_build.outputs.tags }} labels: ${{ steps.meta_build.outputs.labels }} + # Use GitHub cache to load cached Docker images and cache the results of this build + # This decreases the number of images we need to fetch from DockerHub + cache-from: type=gha + cache-to: type=gha,mode=max # Export the digest of Docker build locally (for non PRs only) - name: Export Docker build digest @@ -240,6 +244,7 @@ jobs: # for testing in docker.yml tags: pr-testing flavor: ${{ env.TAGS_FLAVOR }} + # Build local image and stores in a TAR file in /tmp directory - name: Build and push image to local image if: matrix.isPr @@ -252,8 +257,13 @@ jobs: platforms: ${{ matrix.arch }} tags: ${{ steps.meta_build_pr.outputs.tags }} labels: ${{ steps.meta_build_pr.outputs.labels }} + # Use GitHub cache to load cached Docker images and cache the results of this build + # This decreases the number of images we need to fetch from DockerHub + cache-from: type=gha + cache-to: type=gha,mode=max # Export image to a local TAR file outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar + # Upload the local docker image (in TAR file) to a build Artifact - name: Upload local image to artifact if: matrix.isPr From 46c169fb7947acfc559ef07f67d0672880da7d0e Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 6 Dec 2024 12:51:07 -0600 Subject: [PATCH 046/178] Refactor to use the same local TAR image build for PRs and non-PRs. These TAR images act as a cache for our current build. --- .github/workflows/docker.yml | 4 -- .github/workflows/reusable-docker-build.yml | 76 ++++++--------------- 2 files changed, 20 insertions(+), 60 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 080ffc1c080b..6195aadb4eb3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -169,10 +169,6 @@ jobs: signposting__P__enabled: true sword__D__server__P__enabled: true swordv2__D__server__P__enabled: true - # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, if on main branch, use the - # "latest" tag. Otherwise, use the branch name. NOTE: the "pr-testing" tag is a temporary tag that we assign to - # all PR-built docker images in reusable-docker-build.yml - DSPACE_VER: ${{ (github.event_name == 'pull_request' && 'pr-testing') || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index 92dd08623c05..3c497410a274 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -122,16 +122,9 @@ jobs: - name: Setup Docker Buildx uses: docker/setup-buildx-action@v3 - #------------------------------------------------------------ - # Build & deploy steps for new commits to a branch (non-PRs) - # - # These steps build the images, push to DockerHub, and - # (if necessary) redeploy demo/sandbox sites. - #------------------------------------------------------------ # https://github.com/docker/metadata-action - # Get Metadata for docker_build_deps step below + # Extract metadata used for Docker images in all build steps below - name: Extract metadata (tags, labels) from GitHub for Docker image - if: ${{ ! matrix.isPr }} id: meta_build uses: docker/metadata-action@v5 with: @@ -139,6 +132,12 @@ jobs: tags: ${{ env.IMAGE_TAGS }} flavor: ${{ env.TAGS_FLAVOR }} + #------------------------------------------------------------ + # Build & deploy steps for new commits to a branch (non-PRs) + # + # These steps build the images, push to DockerHub, and + # (if necessary) redeploy demo/sandbox sites. + #------------------------------------------------------------ # https://github.com/docker/build-push-action - name: Build and push image to DockerHub # Only build & push if not a PR @@ -178,32 +177,6 @@ jobs: if-no-files-found: error retention-days: 1 - # Build local image (again) and store in a TAR file in /tmp directory - # NOTE: This cannot be combined with push to DockerHub registry above as it's a different type of output. - - name: Build and push image to local image - if: ${{ ! matrix.isPr }} - uses: docker/build-push-action@v5 - with: - build-contexts: | - ${{ inputs.dockerfile_additional_contexts }} - context: ${{ inputs.dockerfile_context }} - file: ${{ inputs.dockerfile_path }} - platforms: ${{ matrix.arch }} - tags: ${{ steps.meta_build.outputs.tags }} - labels: ${{ steps.meta_build.outputs.labels }} - # Export image to a local TAR file - outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar - - # Upload the local docker image (in TAR file) to a build Artifact - - name: Upload local image to artifact - if: ${{ ! matrix.isPr }} - uses: actions/upload-artifact@v4 - with: - name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} - path: /tmp/${{ inputs.build_id }}.tar - if-no-files-found: error - retention-days: 1 - # If this build is NOT a PR and passed in a REDEPLOY_SANDBOX_URL secret, # Then redeploy https://sandbox.dspace.org if this build is for our deployment architecture and 'main' branch. - name: Redeploy sandbox.dspace.org (based on main branch) @@ -227,27 +200,19 @@ jobs: curl -X POST $REDEPLOY_DEMO_URL #------------------------------------------------------------- - # Build steps for PRs only + # Shared Build steps. + # These are used for PRs as well as new commits to a branch (non-PRs) # - # These steps build the images and store as a build artifact. - # These artifacts can then be used by later jobs to run the - # brand-new images for automated testing. + # These steps build the images and cache/store as a build artifact. + # These artifacts can then be used by later jobs to install the + # brand-new images for automated testing. For non-PRs, this cache is + # also used to avoid pulling the images we just built from DockerHub. #-------------------------------------------------------------- - # Get Metadata for docker_build_deps step below - - name: Extract metadata (tags, labels) for local Docker image - if: matrix.isPr - id: meta_build_pr - uses: docker/metadata-action@v5 - with: - images: ${{ env.IMAGE_NAME }} - # Hardcode to use custom "pr-testing" tag because that will allow us to spin up this PR - # for testing in docker.yml - tags: pr-testing - flavor: ${{ env.TAGS_FLAVOR }} - # Build local image and stores in a TAR file in /tmp directory - - name: Build and push image to local image - if: matrix.isPr + # Build local image (again) and store in a TAR file in /tmp directory + # NOTE: This build is run for both PRs and non-PRs as it's used to "cache" our built images as artifacts. + # NOTE #2: This cannot be combined with push to DockerHub registry above as it's a different type of output. + - name: Build and push image to local TAR file uses: docker/build-push-action@v5 with: build-contexts: | @@ -255,8 +220,8 @@ jobs: context: ${{ inputs.dockerfile_context }} file: ${{ inputs.dockerfile_path }} platforms: ${{ matrix.arch }} - tags: ${{ steps.meta_build_pr.outputs.tags }} - labels: ${{ steps.meta_build_pr.outputs.labels }} + tags: ${{ steps.meta_build.outputs.tags }} + labels: ${{ steps.meta_build.outputs.labels }} # Use GitHub cache to load cached Docker images and cache the results of this build # This decreases the number of images we need to fetch from DockerHub cache-from: type=gha @@ -265,8 +230,7 @@ jobs: outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar # Upload the local docker image (in TAR file) to a build Artifact - - name: Upload local image to artifact - if: matrix.isPr + - name: Upload local image TAR to artifact uses: actions/upload-artifact@v4 with: name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} From 020fec96d8206572b2517dcd9dc67fa920c10b23 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 6 Dec 2024 14:12:28 -0600 Subject: [PATCH 047/178] Ensure PRs are tagging their images with same tag as the base branch the PR was created against --- .github/workflows/reusable-docker-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index 3c497410a274..723c1c77256f 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -54,10 +54,13 @@ env: # For a new commit on default branch (main), use the literal tag 'latest' on Docker image. # For a new commit on other branches, use the branch name as the tag for Docker image. # For a new tag, copy that tag name as the tag for Docker image. + # For a pull request, use the name of the base branch that the PR was created against or "latest" (for main). + # e.g. PR against 'main' will use "latest". a PR against 'dspace-7_x' will use 'dspace-7_x'. IMAGE_TAGS: | type=raw,value=latest,enable=${{ github.ref_name == github.event.repository.default_branch }} type=ref,event=branch,enable=${{ github.ref_name != github.event.repository.default_branch }} type=ref,event=tag + type=raw,value=${{ (github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || github.event.pull_request.base.ref }},enable=${{ github.event_name == 'pull_request' }} # Define default tag "flavor" for docker/metadata-action per # https://github.com/docker/metadata-action#flavor-input # We manage the 'latest' tag ourselves to the 'main' branch (see settings above) From 3c0e7158a26b13ae01de500298e94d65e3601c97 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 6 Dec 2024 15:01:08 -0600 Subject: [PATCH 048/178] Ensure each image has a separate cache. This allows later builds of that same image to inherit that cache. --- .github/workflows/reusable-docker-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index 723c1c77256f..d9f4c5342b40 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -159,8 +159,8 @@ jobs: labels: ${{ steps.meta_build.outputs.labels }} # Use GitHub cache to load cached Docker images and cache the results of this build # This decreases the number of images we need to fetch from DockerHub - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ inputs.build_id }} + cache-to: type=gha,scope=${{ inputs.build_id }},mode=max # Export the digest of Docker build locally (for non PRs only) - name: Export Docker build digest @@ -227,8 +227,8 @@ jobs: labels: ${{ steps.meta_build.outputs.labels }} # Use GitHub cache to load cached Docker images and cache the results of this build # This decreases the number of images we need to fetch from DockerHub - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ inputs.build_id }} + cache-to: type=gha,scope=${{ inputs.build_id }},mode=max # Export image to a local TAR file outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar From 78855cdbbab98038f61e2bd94922e4fa3db9ad18 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Mon, 9 Dec 2024 10:18:32 -0600 Subject: [PATCH 049/178] Ensure we use the main Docker image, and not the "-test" image. --- .github/workflows/docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6195aadb4eb3..45d6fd6b0277 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -169,6 +169,9 @@ jobs: signposting__P__enabled: true sword__D__server__P__enabled: true swordv2__D__server__P__enabled: true + # If this is a PR, used the base branch name. If on main branch, use the "latest" tag. Otherwise, use branch name. + # NOTE: DSPACE_VER is used because our docker compose scripts default to using the "-test" image. + DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase From b8c88ef284f95e2a8cb8873f00dcc227fcf58e74 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Mon, 9 Dec 2024 12:06:14 -0600 Subject: [PATCH 050/178] Ensure PRs against main also use "latest" tag --- .github/workflows/docker.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 45d6fd6b0277..ebbd8cc50984 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -169,9 +169,12 @@ jobs: signposting__P__enabled: true sword__D__server__P__enabled: true swordv2__D__server__P__enabled: true - # If this is a PR, used the base branch name. If on main branch, use the "latest" tag. Otherwise, use branch name. + # If this is a PR against main (default branch), use "latest". + # Else if this is a PR against a different branch, used the base branch name. + # Else if this is a commit on main (default branch), use the "latest" tag. + # Else, just use the branch name. # NOTE: DSPACE_VER is used because our docker compose scripts default to using the "-test" image. - DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} + DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase From c017a662e0642d32ae59017451ea8994de48ab44 Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Wed, 11 Dec 2024 21:26:42 +0100 Subject: [PATCH 051/178] 121971: #9669 - Remove unsupported OpenAIRE date types --- .../oai/metadataFormats/oai_openaire.xsl | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl b/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl index 3a1d75eb56c6..b6abd9daa056 100644 --- a/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl +++ b/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl @@ -1039,6 +1039,8 @@ + + @@ -1050,24 +1052,6 @@ Available - - Collected - - - Copyrighted - - - Created - - - Submitted - - - Updated - - - Valid - From 04953b94d919bf5e0135aa3311b70767cba20160 Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Wed, 11 Dec 2024 21:39:08 +0100 Subject: [PATCH 052/178] 121971: #9715 - Only dc.date.issued should have date types Accepted and Issued --- dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl b/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl index b6abd9daa056..f92367646d1f 100644 --- a/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl +++ b/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl @@ -731,7 +731,7 @@ From 31c79500ceec1c8e36228195b802a289ac9c30c9 Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Wed, 11 Dec 2024 21:49:37 +0100 Subject: [PATCH 053/178] 121971: #9716 - Only dc.date.embargo should have date type Available --- dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl b/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl index f92367646d1f..7b141e48b3a0 100644 --- a/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl +++ b/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl @@ -1049,7 +1049,9 @@ - + + + Available From 52e5b35c0653afc17ebd47ce1272549ddf205469 Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Thu, 12 Dec 2024 09:16:21 +0100 Subject: [PATCH 054/178] 121971: #9664 - Make cclicense step required in openaire submission form --- dspace/config/item-submission.xml | 4 ++++ dspace/config/submission-forms.xml | 23 ----------------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/dspace/config/item-submission.xml b/dspace/config/item-submission.xml index a6cd49bdf1e8..866426afd61d 100644 --- a/dspace/config/item-submission.xml +++ b/dspace/config/item-submission.xml @@ -379,6 +379,10 @@ + + + + diff --git a/dspace/config/submission-forms.xml b/dspace/config/submission-forms.xml index 39a4778356c0..13c8fc208f6e 100644 --- a/dspace/config/submission-forms.xml +++ b/dspace/config/submission-forms.xml @@ -1133,29 +1133,6 @@ - - - dc - rights - false - - dropdown - Select the access type of the resource. - - - - - - oaire - license - condition - false - - dropdown - Select the license of the resource. - - - dc From e5401685944b6135795462748c791ed7ba7bab7d Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Thu, 12 Dec 2024 09:18:40 +0100 Subject: [PATCH 055/178] 121971: #9867 - Remove objectType attribute from openaire crosswalk --- .../crosswalks/oai/metadataFormats/oai_openaire.xsl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl b/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl index 7b141e48b3a0..6c6c369071a6 100644 --- a/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl +++ b/dspace/config/crosswalks/oai/metadataFormats/oai_openaire.xsl @@ -888,13 +888,13 @@ - + + fulltext - + other - + --> From a6d2c4897b46062cca59add6946eb7a86c648585 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 4 Dec 2024 13:44:44 -0600 Subject: [PATCH 056/178] Add Docker registry to all scripts. Allow it to be configurable for DSpace images (only). Other minor Dockerfile cleanup --- Dockerfile | 17 +++++++++++------ Dockerfile.cli | 17 +++++++++++------ Dockerfile.dependencies | 2 +- Dockerfile.test | 17 +++++++++++------ docker-compose-cli.yml | 2 +- docker-compose.yml | 6 +++--- .../dspace-postgres-pgcrypto-curl/Dockerfile | 2 +- .../docker/dspace-postgres-pgcrypto/Dockerfile | 2 +- .../main/docker/dspace-shibboleth/Dockerfile | 2 +- dspace/src/main/docker/dspace-solr/Dockerfile | 2 +- 10 files changed, 42 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index adc5d6125f0a..ff35f2a2e3a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,14 @@ # This Dockerfile uses JDK11 by default, but has also been tested with JDK17. # To build with JDK17, use "--build-arg JDK_VERSION=17" ARG JDK_VERSION=11 +# The Docker version tag to build from +ARG DSPACE_VERSION=dspace-7_x +# The Docker registry to use for DSpace images. Defaults to "docker.io" +# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument +ARG DOCKER_REGISTRY=docker.io # Step 1 - Run Maven Build -FROM dspace/dspace-dependencies:dspace-7_x AS build +FROM ${DOCKER_REGISTRY}/dspace/dspace-dependencies:${DSPACE_VERSION} AS build ARG TARGET_DIR=dspace-installer WORKDIR /app # The dspace-installer directory will be written to /install @@ -28,15 +33,15 @@ RUN mvn --no-transfer-progress package ${MAVEN_FLAGS} && \ mvn clean # Step 2 - Run Ant Deploy -FROM eclipse-temurin:${JDK_VERSION} AS ant_build +FROM docker.io/eclipse-temurin:${JDK_VERSION} AS ant_build ARG TARGET_DIR=dspace-installer # COPY the /install directory from 'build' container to /dspace-src in this container COPY --from=build /install /dspace-src WORKDIR /dspace-src # Create the initial install deployment using ANT -ENV ANT_VERSION 1.10.13 -ENV ANT_HOME /tmp/ant-$ANT_VERSION -ENV PATH $ANT_HOME/bin:$PATH +ENV ANT_VERSION=1.10.13 +ENV ANT_HOME=/tmp/ant-$ANT_VERSION +ENV PATH=$ANT_HOME/bin:$PATH # Need wget to install ant RUN apt-get update \ && apt-get install -y --no-install-recommends wget \ @@ -50,7 +55,7 @@ RUN ant init_installation update_configs update_code update_webapps # Step 3 - Run tomcat # Create a new tomcat image that does not retain the the build directory contents -FROM tomcat:9-jdk${JDK_VERSION} +FROM docker.io/tomcat:9-jdk${JDK_VERSION} # NOTE: DSPACE_INSTALL must align with the "dspace.dir" default configuration. ENV DSPACE_INSTALL=/dspace # Copy the /dspace directory from 'ant_build' container to /dspace in this container diff --git a/Dockerfile.cli b/Dockerfile.cli index 8a69b32e2dc6..be03e8922b2c 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -6,9 +6,14 @@ # This Dockerfile uses JDK11 by default, but has also been tested with JDK17. # To build with JDK17, use "--build-arg JDK_VERSION=17" ARG JDK_VERSION=11 +# The Docker version tag to build from +ARG DSPACE_VERSION=dspace-7_x +# The Docker registry to use for DSpace images. Defaults to "docker.io" +# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument +ARG DOCKER_REGISTRY=docker.io # Step 1 - Run Maven Build -FROM dspace/dspace-dependencies:dspace-7_x AS build +FROM ${DOCKER_REGISTRY}/dspace/dspace-dependencies:${DSPACE_VERSION} AS build ARG TARGET_DIR=dspace-installer WORKDIR /app # The dspace-installer directory will be written to /install @@ -24,15 +29,15 @@ RUN mvn --no-transfer-progress package && \ mvn clean # Step 2 - Run Ant Deploy -FROM eclipse-temurin:${JDK_VERSION} AS ant_build +FROM docker.io/eclipse-temurin:${JDK_VERSION} AS ant_build ARG TARGET_DIR=dspace-installer # COPY the /install directory from 'build' container to /dspace-src in this container COPY --from=build /install /dspace-src WORKDIR /dspace-src # Create the initial install deployment using ANT -ENV ANT_VERSION 1.10.13 -ENV ANT_HOME /tmp/ant-$ANT_VERSION -ENV PATH $ANT_HOME/bin:$PATH +ENV ANT_VERSION=1.10.13 +ENV ANT_HOME=/tmp/ant-$ANT_VERSION +ENV PATH=$ANT_HOME/bin:$PATH # Need wget to install ant RUN apt-get update \ && apt-get install -y --no-install-recommends wget \ @@ -45,7 +50,7 @@ RUN mkdir $ANT_HOME && \ RUN ant init_installation update_configs update_code # Step 3 - Run jdk -FROM eclipse-temurin:${JDK_VERSION} +FROM docker.io/eclipse-temurin:${JDK_VERSION} # NOTE: DSPACE_INSTALL must align with the "dspace.dir" default configuration. ENV DSPACE_INSTALL=/dspace # Copy the /dspace directory from 'ant_build' container to /dspace in this container diff --git a/Dockerfile.dependencies b/Dockerfile.dependencies index 123206ea5887..794dfa9a66a8 100644 --- a/Dockerfile.dependencies +++ b/Dockerfile.dependencies @@ -7,7 +7,7 @@ ARG JDK_VERSION=11 # Step 1 - Run Maven Build -FROM maven:3-eclipse-temurin-${JDK_VERSION} AS build +FROM docker.io/maven:3-eclipse-temurin-${JDK_VERSION} AS build ARG TARGET_DIR=dspace-installer WORKDIR /app # Create the 'dspace' user account & home directory diff --git a/Dockerfile.test b/Dockerfile.test index d88699ca52ab..08b6b3018b80 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -8,9 +8,14 @@ # This Dockerfile uses JDK11 by default, but has also been tested with JDK17. # To build with JDK17, use "--build-arg JDK_VERSION=17" ARG JDK_VERSION=11 +# The Docker version tag to build from +ARG DSPACE_VERSION=dspace-7_x +# The Docker registry to use for DSpace images. Defaults to "docker.io" +# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument +ARG DOCKER_REGISTRY=docker.io # Step 1 - Run Maven Build -FROM dspace/dspace-dependencies:dspace-7_x AS build +FROM ${DOCKER_REGISTRY}/dspace/dspace-dependencies:${DSPACE_VERSION} AS build ARG TARGET_DIR=dspace-installer WORKDIR /app # The dspace-installer directory will be written to /install @@ -27,15 +32,15 @@ RUN mvn --no-transfer-progress package -Pdspace-rest && \ mvn clean # Step 2 - Run Ant Deploy -FROM eclipse-temurin:${JDK_VERSION} AS ant_build +FROM docker.io/eclipse-temurin:${JDK_VERSION} AS ant_build ARG TARGET_DIR=dspace-installer # COPY the /install directory from 'build' container to /dspace-src in this container COPY --from=build /install /dspace-src WORKDIR /dspace-src # Create the initial install deployment using ANT -ENV ANT_VERSION 1.10.12 -ENV ANT_HOME /tmp/ant-$ANT_VERSION -ENV PATH $ANT_HOME/bin:$PATH +ENV ANT_VERSION=1.10.12 +ENV ANT_HOME=/tmp/ant-$ANT_VERSION +ENV PATH=$ANT_HOME/bin:$PATH # Need wget to install ant RUN apt-get update \ && apt-get install -y --no-install-recommends wget \ @@ -49,7 +54,7 @@ RUN ant init_installation update_configs update_code update_webapps # Step 3 - Run tomcat # Create a new tomcat image that does not retain the the build directory contents -FROM tomcat:9-jdk${JDK_VERSION} +FROM docker.io/tomcat:9-jdk${JDK_VERSION} ENV DSPACE_INSTALL=/dspace ENV TOMCAT_INSTALL=/usr/local/tomcat # Copy the /dspace directory from 'ant_build' containger to /dspace in this container diff --git a/docker-compose-cli.yml b/docker-compose-cli.yml index d6a194617e02..ce7676bdc6c3 100644 --- a/docker-compose-cli.yml +++ b/docker-compose-cli.yml @@ -6,7 +6,7 @@ networks: external: true services: dspace-cli: - image: "${DOCKER_OWNER:-dspace}/dspace-cli:${DSPACE_VER:-dspace-7_x}" + image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-cli:${DSPACE_VER:-dspace-7_x}" container_name: dspace-cli build: context: . diff --git a/docker-compose.yml b/docker-compose.yml index 23fce37db245..a16c3677503c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: # from the host machine. This IP range MUST correspond to the 'dspacenet' subnet defined above. proxies__P__trusted__P__ipranges: '172.23.0' LOGGING_CONFIG: /dspace/config/log4j2-container.xml - image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}" + image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}" build: context: . dockerfile: Dockerfile.test @@ -66,7 +66,7 @@ services: dspacedb: container_name: dspacedb # Uses a custom Postgres image with pgcrypto installed - image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7_x}" + image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7_x}" build: # Must build out of subdirectory to have access to install script for pgcrypto context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ @@ -86,7 +86,7 @@ services: # DSpace Solr container dspacesolr: container_name: dspacesolr - image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7_x}" + image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7_x}" build: context: ./dspace/src/main/docker/dspace-solr/ # Provide path to Solr configs necessary to build Docker image diff --git a/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile b/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile index aabf87df3eda..791242f5bdc7 100644 --- a/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile +++ b/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile @@ -13,7 +13,7 @@ ARG POSTGRES_VERSION=15 ARG POSTGRES_PASSWORD=dspace -FROM postgres:${POSTGRES_VERSION} +FROM docker.io/postgres:${POSTGRES_VERSION} ENV POSTGRES_DB dspace ENV POSTGRES_USER dspace diff --git a/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile b/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile index 2298cd4e76ea..116d2dbcda31 100644 --- a/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile +++ b/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile @@ -13,7 +13,7 @@ ARG POSTGRES_VERSION=15 ARG POSTGRES_PASSWORD=dspace -FROM postgres:${POSTGRES_VERSION} +FROM docker.io/postgres:${POSTGRES_VERSION} ENV POSTGRES_DB dspace ENV POSTGRES_USER dspace diff --git a/dspace/src/main/docker/dspace-shibboleth/Dockerfile b/dspace/src/main/docker/dspace-shibboleth/Dockerfile index 79f2921bd7d6..25fb510e3d39 100644 --- a/dspace/src/main/docker/dspace-shibboleth/Dockerfile +++ b/dspace/src/main/docker/dspace-shibboleth/Dockerfile @@ -10,7 +10,7 @@ # Build from Ubuntu as it has easy Apache tooling (e.g. a2enmod script is debian only). # Apache & mod_shib are required for DSpace to act as an SP # See also https://wiki.lyrasis.org/display/DSDOC7x/Authentication+Plugins#AuthenticationPlugins-ShibbolethAuthentication -FROM ubuntu:20.04 +FROM docker.io/ubuntu:20.04 # Apache ENVs (default values) ENV APACHE_RUN_USER www-data diff --git a/dspace/src/main/docker/dspace-solr/Dockerfile b/dspace/src/main/docker/dspace-solr/Dockerfile index eb8e93493fa8..0c011d43105f 100644 --- a/dspace/src/main/docker/dspace-solr/Dockerfile +++ b/dspace/src/main/docker/dspace-solr/Dockerfile @@ -12,7 +12,7 @@ ARG SOLR_VERSION=8.11 -FROM solr:${SOLR_VERSION}-slim +FROM docker.io/solr:${SOLR_VERSION}-slim ENV AUTHORITY_CONFIGSET_PATH=/opt/solr/server/solr/configsets/authority/conf \ OAI_CONFIGSET_PATH=/opt/solr/server/solr/configsets/oai/conf \ From 2ee328ff18aa973dd2ed82abd062d8a923cbddf5 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 4 Dec 2024 13:45:33 -0600 Subject: [PATCH 057/178] Minor Dockerfile cleanup. Use new syntax for ENV variables --- .../docker/dspace-postgres-pgcrypto-curl/Dockerfile | 6 +++--- .../main/docker/dspace-postgres-pgcrypto/Dockerfile | 6 +++--- dspace/src/main/docker/dspace-shibboleth/Dockerfile | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile b/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile index 791242f5bdc7..f314fdb79e80 100644 --- a/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile +++ b/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile @@ -15,9 +15,9 @@ ARG POSTGRES_PASSWORD=dspace FROM docker.io/postgres:${POSTGRES_VERSION} -ENV POSTGRES_DB dspace -ENV POSTGRES_USER dspace -ENV POSTGRES_PASSWORD ${POSTGRES_PASSWORD} +ENV POSTGRES_DB=dspace +ENV POSTGRES_USER=dspace +ENV POSTGRES_PASSWORD=${POSTGRES_PASSWORD} # Install curl which is necessary to load SQL file RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* diff --git a/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile b/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile index 116d2dbcda31..4f639f361024 100644 --- a/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile +++ b/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile @@ -15,9 +15,9 @@ ARG POSTGRES_PASSWORD=dspace FROM docker.io/postgres:${POSTGRES_VERSION} -ENV POSTGRES_DB dspace -ENV POSTGRES_USER dspace -ENV POSTGRES_PASSWORD ${POSTGRES_PASSWORD} +ENV POSTGRES_DB=dspace +ENV POSTGRES_USER=dspace +ENV POSTGRES_PASSWORD=${POSTGRES_PASSWORD} # Copy over script which will initialize database and install pgcrypto extension COPY install-pgcrypto.sh /docker-entrypoint-initdb.d/ diff --git a/dspace/src/main/docker/dspace-shibboleth/Dockerfile b/dspace/src/main/docker/dspace-shibboleth/Dockerfile index 25fb510e3d39..15a436d7b6f7 100644 --- a/dspace/src/main/docker/dspace-shibboleth/Dockerfile +++ b/dspace/src/main/docker/dspace-shibboleth/Dockerfile @@ -13,12 +13,12 @@ FROM docker.io/ubuntu:20.04 # Apache ENVs (default values) -ENV APACHE_RUN_USER www-data -ENV APACHE_RUN_GROUP www-data -ENV APACHE_LOCK_DIR /var/lock/apache2 -ENV APACHE_LOG_DIR /var/log/apache2 -ENV APACHE_PID_FILE /var/run/apache2/apache2.pid -ENV APACHE_SERVER_NAME localhost +ENV APACHE_RUN_USER=www-data +ENV APACHE_RUN_GROUP=www-data +ENV APACHE_LOCK_DIR=/var/lock/apache2 +ENV APACHE_LOG_DIR=/var/log/apache2 +ENV APACHE_PID_FILE=/var/run/apache2/apache2.pid +ENV APACHE_SERVER_NAME=localhost # Ensure Apache2, mod_shib & shibboleth daemon are installed. # Also install ssl-cert to provide a local SSL cert for use with Apache From 2a6c60e1392cf643b4ae4c8512ce92c7aba4b2eb Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Thu, 12 Dec 2024 16:50:02 -0600 Subject: [PATCH 058/178] Refactor Docker build process to use ghcr.io for build, and then copy to docker.io once build completes --- .github/workflows/docker.yml | 1 + .github/workflows/reusable-docker-build.yml | 189 ++++++++++++++------ 2 files changed, 131 insertions(+), 59 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ebbd8cc50984..fceea6403d2a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,6 +15,7 @@ on: permissions: contents: read # to fetch code (actions/checkout) + packages: write # to write images to GitHub Container Registry (GHCR) jobs: #################################################### diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index d9f4c5342b40..36546af3a5da 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -74,7 +74,11 @@ env: # Current DSpace maintenance branch (and architecture) which is deployed to demo.dspace.org / sandbox.dspace.org # (NOTE: No deployment branch specified for sandbox.dspace.org as it uses the default_branch) DEPLOY_DEMO_BRANCH: 'dspace-8_x' + DEPLOY_SANDBOX_BRANCH: 'main' DEPLOY_ARCH: 'linux/amd64' + # Registry used during building of Docker images. (All images are later copied to docker.io registry) + # We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub. + DOCKER_BUILD_REGISTRY: ghcr.io jobs: docker-build: @@ -99,6 +103,7 @@ jobs: # This step converts the slashes in the "arch" matrix values above into dashes & saves to env.ARCH_NAME # E.g. "linux/amd64" becomes "linux-amd64" # This is necessary because all upload artifacts CANNOT have special chars (like slashes) + # NOTE: The regex-like syntax below is Bash Parameter Substitution - name: Prepare run: | platform=${{ matrix.arch }} @@ -109,13 +114,14 @@ jobs: uses: actions/checkout@v4 # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push + - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }} + # Only login if not a PR, as PRs only trigger a Docker build and not a push if: ${{ ! matrix.isPr }} uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + registry: ${{ env.DOCKER_BUILD_REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} # https://github.com/docker/setup-qemu-action - name: Set up QEMU emulation to build for multiple architectures @@ -131,19 +137,20 @@ jobs: id: meta_build uses: docker/metadata-action@v5 with: - images: ${{ env.IMAGE_NAME }} + images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }} tags: ${{ env.IMAGE_TAGS }} flavor: ${{ env.TAGS_FLAVOR }} - #------------------------------------------------------------ - # Build & deploy steps for new commits to a branch (non-PRs) + #-------------------------------------------------------------------- + # First, for all branch commits (non-PRs) we build the image & upload + # to GitHub Container Registry (GHCR). After uploading the image + # to GHCR, we store the image digest in an artifact, so we can + # create a merged manifest later (see 'docker-build_manifest' job). # - # These steps build the images, push to DockerHub, and - # (if necessary) redeploy demo/sandbox sites. - #------------------------------------------------------------ + # NOTE: We use GHCR in order to avoid aggressive rate limits at DockerHub. + #-------------------------------------------------------------------- # https://github.com/docker/build-push-action - - name: Build and push image to DockerHub - # Only build & push if not a PR + - name: Build and push image to ${{ env.DOCKER_BUILD_REGISTRY }} if: ${{ ! matrix.isPr }} id: docker_build uses: docker/build-push-action@v5 @@ -152,6 +159,9 @@ jobs: ${{ inputs.dockerfile_additional_contexts }} context: ${{ inputs.dockerfile_context }} file: ${{ inputs.dockerfile_path }} + # Tell DSpace's Docker files to use the build registry instead of DockerHub + build-args: + DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }} platforms: ${{ matrix.arch }} push: true # Use tags / labels provided by 'docker/metadata-action' above @@ -162,7 +172,7 @@ jobs: cache-from: type=gha,scope=${{ inputs.build_id }} cache-to: type=gha,scope=${{ inputs.build_id }},mode=max - # Export the digest of Docker build locally (for non PRs only) + # Export the digest of Docker build locally - name: Export Docker build digest if: ${{ ! matrix.isPr }} run: | @@ -170,7 +180,8 @@ jobs: digest="${{ steps.docker_build.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" - # Upload digest to an artifact, so that it can be used in manifest below + # Upload digest to an artifact, so that it can be used in combined manifest below + # (The purpose of the combined manifest is to list both amd64 and arm64 builds under same tag) - name: Upload Docker build digest to artifact if: ${{ ! matrix.isPr }} uses: actions/upload-artifact@v4 @@ -180,48 +191,31 @@ jobs: if-no-files-found: error retention-days: 1 - # If this build is NOT a PR and passed in a REDEPLOY_SANDBOX_URL secret, - # Then redeploy https://sandbox.dspace.org if this build is for our deployment architecture and 'main' branch. - - name: Redeploy sandbox.dspace.org (based on main branch) - if: | - !matrix.isPR && - env.REDEPLOY_SANDBOX_URL != '' && - matrix.arch == env.DEPLOY_ARCH && - github.ref_name == github.event.repository.default_branch - run: | - curl -X POST $REDEPLOY_SANDBOX_URL - - # If this build is NOT a PR and passed in a REDEPLOY_DEMO_URL secret, - # Then redeploy https://demo.dspace.org if this build is for our deployment architecture and demo branch. - - name: Redeploy demo.dspace.org (based on maintenance branch) - if: | - !matrix.isPR && - env.REDEPLOY_DEMO_URL != '' && - matrix.arch == env.DEPLOY_ARCH && - github.ref_name == env.DEPLOY_DEMO_BRANCH - run: | - curl -X POST $REDEPLOY_DEMO_URL - - #------------------------------------------------------------- - # Shared Build steps. - # These are used for PRs as well as new commits to a branch (non-PRs) + #------------------------------------------------------------------------------ + # Second, we build the image again in order to store it in a local TAR file. + # This TAR of the image is cached/saved as an artifact, so that it can be used + # by later jobs to install the brand-new images for automated testing. + # This TAR build is performed BOTH for PRs and for branch commits (non-PRs). # - # These steps build the images and cache/store as a build artifact. - # These artifacts can then be used by later jobs to install the - # brand-new images for automated testing. For non-PRs, this cache is - # also used to avoid pulling the images we just built from DockerHub. - #-------------------------------------------------------------- - + # (This approach has the advantage of avoiding having to download the newly built + # image from DockerHub or GHCR during automated testing.) + # + # See the 'docker-deploy' job in docker.yml as an example of where this TAR is used. + #------------------------------------------------------------------------------- # Build local image (again) and store in a TAR file in /tmp directory - # NOTE: This build is run for both PRs and non-PRs as it's used to "cache" our built images as artifacts. - # NOTE #2: This cannot be combined with push to DockerHub registry above as it's a different type of output. + # This step is only done for AMD64, as that's the only image we use in our automated testing (at this time). + # NOTE: This step cannot be combined with the build above as it's a different type of output. - name: Build and push image to local TAR file + if: ${{ matrix.arch == 'linux/amd64'}} uses: docker/build-push-action@v5 with: build-contexts: | ${{ inputs.dockerfile_additional_contexts }} context: ${{ inputs.dockerfile_context }} file: ${{ inputs.dockerfile_path }} + # Tell DSpace's Docker files to use the build registry instead of DockerHub + build-args: + DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }} platforms: ${{ matrix.arch }} tags: ${{ steps.meta_build.outputs.tags }} labels: ${{ steps.meta_build.outputs.labels }} @@ -233,7 +227,9 @@ jobs: outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar # Upload the local docker image (in TAR file) to a build Artifact + # This step is only done for AMD64, as that's the only image we use in our automated testing (at this time). - name: Upload local image TAR to artifact + if: ${{ matrix.arch == 'linux/amd64'}} uses: actions/upload-artifact@v4 with: name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} @@ -241,10 +237,12 @@ jobs: if-no-files-found: error retention-days: 1 - # Merge Docker digests (from various architectures) into a manifest. - # This runs after all Docker builds complete above, and it tells hub.docker.com - # that these builds should be all included in the manifest for this tag. - # (e.g. AMD64 and ARM64 should be listed as options under the same tagged Docker image) + ########################################################################################## + # Merge Docker digests (from various architectures) into a single manifest. + # This runs after all Docker builds complete above. The purpose is to include all builds + # under a single manifest for this tag. + # (e.g. both linux/amd64 and linux/arm64 should be listed under the same tagged Docker image) + ########################################################################################## docker-build_manifest: # Only run if this is NOT a PR if: ${{ github.event_name != 'pull_request' }} @@ -260,11 +258,12 @@ jobs: pattern: digests-${{ inputs.build_id }}-* merge-multiple: true - - name: Login to Docker Hub + - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }} uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + registry: ${{ env.DOCKER_BUILD_REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -273,16 +272,88 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.IMAGE_NAME }} + images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }} tags: ${{ env.IMAGE_TAGS }} flavor: ${{ env.TAGS_FLAVOR }} - - name: Create manifest list from digests and push + - name: Create manifest list from digests and push to ${{ env.DOCKER_BUILD_REGISTRY }} working-directory: /tmp/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *) + $(printf '${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect manifest in ${{ env.DOCKER_BUILD_REGISTRY }} + run: | + docker buildx imagetools inspect ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} + + ########################################################################################## + # Copy images / manifest to DockerHub. + # This MUST run after *both* images (AMD64 and ARM64) are built and uploaded to GitHub + # Container Registry (GHCR). Attempting to run this in parallel to GHCR builds can result + # in a race condition...i.e. the copy to DockerHub may fail if GHCR image has been updated + # at the moment when the copy occurs. + ########################################################################################## + docker-copy_to_dockerhub: + # Only run if this is NOT a PR + if: ${{ github.event_name != 'pull_request' }} + runs-on: ubuntu-latest + needs: + - docker-build_manifest + + steps: + # 'regctl' is used to more easily copy the image to DockerHub and obtain the digest from DockerHub + # See https://github.com/regclient/regclient/blob/main/docs/regctl.md + - name: Install regctl for Docker registry tools + uses: regclient/actions/regctl-installer@main + with: + release: 'v0.8.0' - - name: Inspect image + # This recreates Docker tags for DockerHub + - name: Add Docker metadata for image + id: meta_dockerhub + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: ${{ env.IMAGE_TAGS }} + flavor: ${{ env.TAGS_FLAVOR }} + + # Login to source registry first, as this is where we are copying *from* + - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.DOCKER_BUILD_REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Login to DockerHub, since this is where we are copying *to* + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + # Copy the image from source to DockerHub + - name: Copy image from ${{ env.DOCKER_BUILD_REGISTRY }} to docker.io + run: | + regctl image copy ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }} docker.io/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }} + + #-------------------------------------------------------------------- + # Finally, check whether demo.dspace.org or sandbox.dspace.org need + # to be redeployed based on these new DockerHub images. + #-------------------------------------------------------------------- + # If this build is for the branch that Sandbox uses and passed in a REDEPLOY_SANDBOX_URL secret, + # Then redeploy https://sandbox.dspace.org + - name: Redeploy sandbox.dspace.org (based on main branch) + if: | + env.REDEPLOY_SANDBOX_URL != '' && + github.ref_name == env.DEPLOY_SANDBOX_BRANCH + run: | + curl -X POST $REDEPLOY_SANDBOX_URL + # If this build is for the branch that Demo uses and passed in a REDEPLOY_DEMO_URL secret, + # Then redeploy https://demo.dspace.org + - name: Redeploy demo.dspace.org (based on maintenance branch) + if: | + env.REDEPLOY_DEMO_URL != '' && + github.ref_name == env.DEPLOY_DEMO_BRANCH run: | - docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} + curl -X POST $REDEPLOY_DEMO_URL \ No newline at end of file From 296c9a12f489d779c15bf07fd8c38fac6bc6d233 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Mon, 16 Dec 2024 12:05:41 -0600 Subject: [PATCH 059/178] PRs must also login to ghcr.io in order to read private images for the build process --- .github/workflows/reusable-docker-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index 36546af3a5da..3b74f250b539 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -114,9 +114,9 @@ jobs: uses: actions/checkout@v4 # https://github.com/docker/login-action + # NOTE: This login occurs for BOTH non-PRs or PRs. PRs *must* also login to access private images from GHCR + # during the build process - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }} - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: ${{ ! matrix.isPr }} uses: docker/login-action@v3 with: registry: ${{ env.DOCKER_BUILD_REGISTRY }} From a27f1ed1757b193cf6f831a0fd27585b655bacc2 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Mon, 16 Dec 2024 12:28:16 -0600 Subject: [PATCH 060/178] Ensure "docker-deploy" job also uses ghcr.io by default. --- .github/workflows/docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fceea6403d2a..815aec5cf6b4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -176,6 +176,9 @@ jobs: # Else, just use the branch name. # NOTE: DSPACE_VER is used because our docker compose scripts default to using the "-test" image. DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} + # Docker Registry to use for Docker compose scripts below. + # We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub. + DOCKER_REGISTRY: ghcr.io steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase From 64cb3bda0034d8e3051c31290a3eeea8a560d939 Mon Sep 17 00:00:00 2001 From: autavares-dev Date: Wed, 7 Aug 2024 16:34:38 -0300 Subject: [PATCH 061/178] Changes Group2GroupCache computation --- .../org/dspace/eperson/Group2GroupCache.java | 3 +- .../org/dspace/eperson/GroupServiceImpl.java | 72 +++++++++--------- .../eperson/dao/Group2GroupCacheDAO.java | 74 +++++++++++++++++-- .../dao/impl/Group2GroupCacheDAOImpl.java | 34 +++++++++ 4 files changed, 143 insertions(+), 40 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/eperson/Group2GroupCache.java b/dspace-api/src/main/java/org/dspace/eperson/Group2GroupCache.java index 09bdf34d4cad..58781cd402fa 100644 --- a/dspace-api/src/main/java/org/dspace/eperson/Group2GroupCache.java +++ b/dspace-api/src/main/java/org/dspace/eperson/Group2GroupCache.java @@ -14,6 +14,7 @@ import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; import javax.persistence.Table; +import javax.persistence.UniqueConstraint; import org.hibernate.proxy.HibernateProxyHelper; @@ -23,7 +24,7 @@ * @author kevinvandevelde at atmire.com */ @Entity -@Table(name = "group2groupcache") +@Table(name = "group2groupcache", uniqueConstraints = { @UniqueConstraint(columnNames = {"parent_id", "child_id"}) }) public class Group2GroupCache implements Serializable { @Id diff --git a/dspace-api/src/main/java/org/dspace/eperson/GroupServiceImpl.java b/dspace-api/src/main/java/org/dspace/eperson/GroupServiceImpl.java index 3fb20e2f1e6f..b52f17a5c692 100644 --- a/dspace-api/src/main/java/org/dspace/eperson/GroupServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/eperson/GroupServiceImpl.java @@ -20,6 +20,7 @@ import java.util.UUID; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.SetUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; @@ -673,15 +674,14 @@ protected boolean isEPersonInGroup(Context context, Group group, EPerson ePerson /** - * Regenerate the group cache AKA the group2groupcache table in the database - - * meant to be called when a group is added or removed from another group + * Returns a set with pairs of parent and child group UUIDs, representing the new cache table rows. * - * @param context The relevant DSpace Context. - * @param flushQueries flushQueries Flush all pending queries + * @param context The relevant DSpace Context. + * @param flushQueries flushQueries Flush all pending queries + * @return Pairs of parent and child group UUID of the new cache. * @throws SQLException An exception that provides information on a database access error or other errors. */ - protected void rethinkGroupCache(Context context, boolean flushQueries) throws SQLException { - + private Set> computeNewCache(Context context, boolean flushQueries) throws SQLException { Map> parents = new HashMap<>(); List> group2groupResults = groupDAO.getGroup2GroupResults(context, flushQueries); @@ -689,19 +689,8 @@ protected void rethinkGroupCache(Context context, boolean flushQueries) throws S UUID parent = group2groupResult.getLeft(); UUID child = group2groupResult.getRight(); - // if parent doesn't have an entry, create one - if (!parents.containsKey(parent)) { - Set children = new HashSet<>(); - - // add child id to the list - children.add(child); - parents.put(parent, children); - } else { - // parent has an entry, now add the child to the parent's record - // of children - Set children = parents.get(parent); - children.add(child); - } + parents.putIfAbsent(parent, new HashSet<>()); + parents.get(parent).add(child); } // now parents is a hash of all of the IDs of groups that are parents @@ -714,27 +703,42 @@ protected void rethinkGroupCache(Context context, boolean flushQueries) throws S parent.getValue().addAll(myChildren); } - // empty out group2groupcache table - group2GroupCacheDAO.deleteAll(context); - - // write out new one + // write out new cache IN MEMORY ONLY and returns it + Set> newCache = new HashSet<>(); for (Map.Entry> parent : parents.entrySet()) { UUID key = parent.getKey(); - for (UUID child : parent.getValue()) { + newCache.add(Pair.of(key, child)); + } + } + return newCache; + } - Group parentGroup = find(context, key); - Group childGroup = find(context, child); + /** + * Regenerate the group cache AKA the group2groupcache table in the database - + * meant to be called when a group is added or removed from another group + * + * @param context The relevant DSpace Context. + * @param flushQueries flushQueries Flush all pending queries + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + protected void rethinkGroupCache(Context context, boolean flushQueries) throws SQLException { + // current cache in the database + var oldCache = group2GroupCacheDAO.getCache(context); - if (parentGroup != null && childGroup != null && group2GroupCacheDAO - .find(context, parentGroup, childGroup) == null) { - Group2GroupCache group2GroupCache = group2GroupCacheDAO.create(context, new Group2GroupCache()); - group2GroupCache.setParent(parentGroup); - group2GroupCache.setChild(childGroup); - group2GroupCacheDAO.save(context, group2GroupCache); - } - } + // correct cache, computed from the Group table + var newCache = computeNewCache(context, flushQueries); + + var toDelete = SetUtils.difference(oldCache, newCache); + var toCreate = SetUtils.difference(newCache, oldCache); + + for (var pair : toDelete ) { + group2GroupCacheDAO.deleteFromCache(context, pair.getLeft(), pair.getRight()); + } + + for (var pair : toCreate ) { + group2GroupCacheDAO.addToCache(context, pair.getLeft(), pair.getRight()); } } diff --git a/dspace-api/src/main/java/org/dspace/eperson/dao/Group2GroupCacheDAO.java b/dspace-api/src/main/java/org/dspace/eperson/dao/Group2GroupCacheDAO.java index 7db569a59e2b..d41d52c7e618 100644 --- a/dspace-api/src/main/java/org/dspace/eperson/dao/Group2GroupCacheDAO.java +++ b/dspace-api/src/main/java/org/dspace/eperson/dao/Group2GroupCacheDAO.java @@ -9,7 +9,10 @@ import java.sql.SQLException; import java.util.List; +import java.util.Set; +import java.util.UUID; +import org.apache.commons.lang3.tuple.Pair; import org.dspace.core.Context; import org.dspace.core.GenericDAO; import org.dspace.eperson.Group; @@ -25,13 +28,74 @@ */ public interface Group2GroupCacheDAO extends GenericDAO { - public List findByParent(Context context, Group group) throws SQLException; + /** + * Returns the current cache table as a set of UUID pairs. + * @param context The relevant DSpace Context. + * @return Set of UUID pairs, where the first element is the parent UUID and the second one is the child UUID. + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + Set> getCache(Context context) throws SQLException; - public List findByChildren(Context context, Iterable groups) throws SQLException; + /** + * Returns all cache entities that are children of a given parent Group entity. + * @param context The relevant DSpace Context. + * @param group Parent group to perform the search. + * @return List of cached groups that are children of the parent group. + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + List findByParent(Context context, Group group) throws SQLException; - public Group2GroupCache findByParentAndChild(Context context, Group parent, Group child) throws SQLException; + /** + * Returns all cache entities that are parents of at least one group from a children groups list. + * @param context The relevant DSpace Context. + * @param groups Children groups to perform the search. + * @return List of cached groups that are parents of at least one group from the children groups list. + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + List findByChildren(Context context, Iterable groups) throws SQLException; - public Group2GroupCache find(Context context, Group parent, Group child) throws SQLException; + /** + * Returns the cache entity given specific parent and child groups. + * @param context The relevant DSpace Context. + * @param parent Parent group. + * @param child Child gruoup. + * @return Cached group. + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + Group2GroupCache findByParentAndChild(Context context, Group parent, Group child) throws SQLException; - public void deleteAll(Context context) throws SQLException; + /** + * Returns the cache entity given specific parent and child groups. + * @param context The relevant DSpace Context. + * @param parent Parent group. + * @param child Child gruoup. + * @return Cached group. + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + Group2GroupCache find(Context context, Group parent, Group child) throws SQLException; + + /** + * Completely deletes the current cache table. + * @param context The relevant DSpace Context. + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + void deleteAll(Context context) throws SQLException; + + /** + * Deletes a specific cache row given parent and child groups UUIDs. + * @param context The relevant DSpace Context. + * @param parent Parent group UUID. + * @param child Child group UUID. + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + void deleteFromCache(Context context, UUID parent, UUID child) throws SQLException; + + /** + * Adds a single row to the cache table given parent and child groups UUIDs. + * @param context The relevant DSpace Context. + * @param parent Parent group UUID. + * @param child Child group UUID. + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + void addToCache(Context context, UUID parent, UUID child) throws SQLException; } diff --git a/dspace-api/src/main/java/org/dspace/eperson/dao/impl/Group2GroupCacheDAOImpl.java b/dspace-api/src/main/java/org/dspace/eperson/dao/impl/Group2GroupCacheDAOImpl.java index 83fb48aaf03d..42ca2bb5d48c 100644 --- a/dspace-api/src/main/java/org/dspace/eperson/dao/impl/Group2GroupCacheDAOImpl.java +++ b/dspace-api/src/main/java/org/dspace/eperson/dao/impl/Group2GroupCacheDAOImpl.java @@ -8,14 +8,18 @@ package org.dspace.eperson.dao.impl; import java.sql.SQLException; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Set; +import java.util.UUID; import javax.persistence.Query; import javax.persistence.criteria.CriteriaBuilder; import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; +import org.apache.commons.lang3.tuple.Pair; import org.dspace.core.AbstractHibernateDAO; import org.dspace.core.Context; import org.dspace.eperson.Group; @@ -35,6 +39,16 @@ protected Group2GroupCacheDAOImpl() { super(); } + @Override + public Set> getCache(Context context) throws SQLException { + Query query = createQuery( + context, + "SELECT new org.apache.commons.lang3.tuple.ImmutablePair(g.parent.id, g.child.id) FROM Group2GroupCache g" + ); + List> results = query.getResultList(); + return new HashSet>(results); + } + @Override public List findByParent(Context context, Group group) throws SQLException { CriteriaBuilder criteriaBuilder = getCriteriaBuilder(context); @@ -90,4 +104,24 @@ public Group2GroupCache find(Context context, Group parent, Group child) throws public void deleteAll(Context context) throws SQLException { createQuery(context, "delete from Group2GroupCache").executeUpdate(); } + + @Override + public void deleteFromCache(Context context, UUID parent, UUID child) throws SQLException { + Query query = getHibernateSession(context).createNativeQuery( + "delete from group2groupcache g WHERE g.parent_id = :parent AND g.child_id = :child" + ); + query.setParameter("parent", parent); + query.setParameter("child", child); + query.executeUpdate(); + } + + @Override + public void addToCache(Context context, UUID parent, UUID child) throws SQLException { + Query query = getHibernateSession(context).createNativeQuery( + "insert into group2groupcache (parent_id, child_id) VALUES (:parent, :child)" + ); + query.setParameter("parent", parent); + query.setParameter("child", child); + query.executeUpdate(); + } } From 05bac14716e1b011ab56abc7b9d662e9d689e546 Mon Sep 17 00:00:00 2001 From: autavares-dev Date: Thu, 8 Aug 2024 15:52:03 -0300 Subject: [PATCH 062/178] Refactor 'var' variables to explicit types --- .../java/org/dspace/eperson/GroupServiceImpl.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/eperson/GroupServiceImpl.java b/dspace-api/src/main/java/org/dspace/eperson/GroupServiceImpl.java index b52f17a5c692..4cec4c9c0d93 100644 --- a/dspace-api/src/main/java/org/dspace/eperson/GroupServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/eperson/GroupServiceImpl.java @@ -725,19 +725,19 @@ private Set> computeNewCache(Context context, boolean flushQuer */ protected void rethinkGroupCache(Context context, boolean flushQueries) throws SQLException { // current cache in the database - var oldCache = group2GroupCacheDAO.getCache(context); + Set> oldCache = group2GroupCacheDAO.getCache(context); // correct cache, computed from the Group table - var newCache = computeNewCache(context, flushQueries); + Set> newCache = computeNewCache(context, flushQueries); - var toDelete = SetUtils.difference(oldCache, newCache); - var toCreate = SetUtils.difference(newCache, oldCache); + SetUtils.SetView> toDelete = SetUtils.difference(oldCache, newCache); + SetUtils.SetView> toCreate = SetUtils.difference(newCache, oldCache); - for (var pair : toDelete ) { + for (Pair pair : toDelete ) { group2GroupCacheDAO.deleteFromCache(context, pair.getLeft(), pair.getRight()); } - for (var pair : toCreate ) { + for (Pair pair : toCreate ) { group2GroupCacheDAO.addToCache(context, pair.getLeft(), pair.getRight()); } } From aa027aefae2e39558273fcdbd511c288f8b0def3 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 17 Dec 2024 13:11:43 -0600 Subject: [PATCH 063/178] Improve Apache Ant download process. Switch to using curl so that we can retry the request if it initially fails. (cherry picked from commit e236634a4c758a6c0d81e929b2f36d49e81b0835) --- Dockerfile | 10 ++++------ Dockerfile.cli | 10 ++++------ Dockerfile.test | 10 ++++------ 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff35f2a2e3a7..102f00abe1f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,14 +42,12 @@ WORKDIR /dspace-src ENV ANT_VERSION=1.10.13 ENV ANT_HOME=/tmp/ant-$ANT_VERSION ENV PATH=$ANT_HOME/bin:$PATH -# Need wget to install ant -RUN apt-get update \ - && apt-get install -y --no-install-recommends wget \ - && apt-get purge -y --auto-remove \ - && rm -rf /var/lib/apt/lists/* # Download and install 'ant' RUN mkdir $ANT_HOME && \ - wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME + curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \ + https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \ + tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \ + rm /tmp/apache-ant.tar.gz # Run necessary 'ant' deploy scripts RUN ant init_installation update_configs update_code update_webapps diff --git a/Dockerfile.cli b/Dockerfile.cli index be03e8922b2c..e208e3d92145 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -38,14 +38,12 @@ WORKDIR /dspace-src ENV ANT_VERSION=1.10.13 ENV ANT_HOME=/tmp/ant-$ANT_VERSION ENV PATH=$ANT_HOME/bin:$PATH -# Need wget to install ant -RUN apt-get update \ - && apt-get install -y --no-install-recommends wget \ - && apt-get purge -y --auto-remove \ - && rm -rf /var/lib/apt/lists/* # Download and install 'ant' RUN mkdir $ANT_HOME && \ - wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME + curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \ + https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \ + tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \ + rm /tmp/apache-ant.tar.gz # Run necessary 'ant' deploy scripts RUN ant init_installation update_configs update_code diff --git a/Dockerfile.test b/Dockerfile.test index 08b6b3018b80..cc73c655b637 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -41,14 +41,12 @@ WORKDIR /dspace-src ENV ANT_VERSION=1.10.12 ENV ANT_HOME=/tmp/ant-$ANT_VERSION ENV PATH=$ANT_HOME/bin:$PATH -# Need wget to install ant -RUN apt-get update \ - && apt-get install -y --no-install-recommends wget \ - && apt-get purge -y --auto-remove \ - && rm -rf /var/lib/apt/lists/* # Download and install 'ant' RUN mkdir $ANT_HOME && \ - wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME + curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \ + https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \ + tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \ + rm /tmp/apache-ant.tar.gz # Run necessary 'ant' deploy scripts RUN ant init_installation update_configs update_code update_webapps From 58af9fd224ba37a8f3a9c8b354c04d772d6f3ee2 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 17 Dec 2024 14:13:21 -0600 Subject: [PATCH 064/178] Significantly speed up build of dspace-dependencies by only copying over POM files (cherry picked from commit 6d7a3fcb725bbb5e42790ba5c57292477f2f3f01) --- Dockerfile.dependencies | 58 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/Dockerfile.dependencies b/Dockerfile.dependencies index 794dfa9a66a8..d3ca9d5e3c87 100644 --- a/Dockerfile.dependencies +++ b/Dockerfile.dependencies @@ -6,7 +6,7 @@ # To build with JDK17, use "--build-arg JDK_VERSION=17" ARG JDK_VERSION=11 -# Step 1 - Run Maven Build +# Step 1 - Download all Dependencies FROM docker.io/maven:3-eclipse-temurin-${JDK_VERSION} AS build ARG TARGET_DIR=dspace-installer WORKDIR /app @@ -19,16 +19,60 @@ RUN chown -Rv dspace: /app # Switch to dspace user & run below commands as that user USER dspace -# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents) -ADD --chown=dspace . /app/ +# This next part may look odd, but it speeds up the build of this image *significantly*. +# Copy ONLY the POMs to this image (from local machine). This will allow us to download all dependencies *without* +# performing any code compilation steps. + +# Parent POM +ADD --chown=dspace pom.xml /app/ +RUN mkdir -p /app/dspace + +# 'dspace' module POM. Includes 'additions' ONLY, as it's the only submodule that is required to exist. +ADD --chown=dspace dspace/pom.xml /app/dspace/ +RUN mkdir -p /app/dspace/modules/ +ADD --chown=dspace dspace/modules/pom.xml /app/dspace/modules/ +RUN mkdir -p /app/dspace/modules/additions +ADD --chown=dspace dspace/modules/additions/pom.xml /app/dspace/modules/additions/ + +# 'dspace-api' module POM +RUN mkdir -p /app/dspace-api +ADD --chown=dspace dspace-api/pom.xml /app/dspace-api/ + +# 'dspace-iiif' module POM +RUN mkdir -p /app/dspace-iiif +ADD --chown=dspace dspace-iiif/pom.xml /app/dspace-iiif/ + +# 'dspace-oai' module POM +RUN mkdir -p /app/dspace-oai +ADD --chown=dspace dspace-oai/pom.xml /app/dspace-oai/ + +# 'dspace-rdf' module POM +RUN mkdir -p /app/dspace-rdf +ADD --chown=dspace dspace-rdf/pom.xml /app/dspace-rdf/ + +# 'dspace-server-webapp' module POM +RUN mkdir -p /app/dspace-server-webapp +ADD --chown=dspace dspace-server-webapp/pom.xml /app/dspace-server-webapp/ + +# 'dspace-services' module POM +RUN mkdir -p /app/dspace-services +ADD --chown=dspace dspace-services/pom.xml /app/dspace-services/ + +# 'dspace-sword' module POM +RUN mkdir -p /app/dspace-sword +ADD --chown=dspace dspace-sword/pom.xml /app/dspace-sword/ + +# 'dspace-swordv2' module POM +RUN mkdir -p /app/dspace-swordv2 +ADD --chown=dspace dspace-swordv2/pom.xml /app/dspace-swordv2/ # Trigger the installation of all maven dependencies (hide download progress messages) # Maven flags here ensure that we skip final assembly, skip building test environment and skip all code verification checks. -# These flags speed up this installation as much as reasonably possible. -ENV MAVEN_FLAGS="-P-assembly -P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true" -RUN mvn --no-transfer-progress install ${MAVEN_FLAGS} +# These flags speed up this installation and skip tasks we cannot perform as we don't have the full source code. +ENV MAVEN_FLAGS="-P-assembly -P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxjc.skip=true -Dxml.skip=true" +RUN mvn --no-transfer-progress verify ${MAVEN_FLAGS} -# Clear the contents of the /app directory (including all maven builds), so no artifacts remain. +# Clear the contents of the /app directory (including all maven target folders), so no artifacts remain. # This ensures when dspace:dspace is built, it will use the Maven local cache (~/.m2) for dependencies USER root RUN rm -rf /app/* From e5568157c87370531e6b86cbf582e160504858fb Mon Sep 17 00:00:00 2001 From: "Mark H. Wood" Date: Tue, 17 Sep 2024 08:47:54 -0400 Subject: [PATCH 065/178] More information about failed DOI registrations. (cherry picked from commit b8f4ab0eb3e90b080eaae057fb5ee673d3477dc2) --- .../dspace/identifier/doi/DOIOrganiser.java | 3 ++- .../identifier/doi/DataCiteConnector.java | 22 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/identifier/doi/DOIOrganiser.java b/dspace-api/src/main/java/org/dspace/identifier/doi/DOIOrganiser.java index 088e2b1cbc87..12f6d7c4fca4 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/doi/DOIOrganiser.java +++ b/dspace-api/src/main/java/org/dspace/identifier/doi/DOIOrganiser.java @@ -577,7 +577,8 @@ public void update(DOI doiRow) { } } catch (IdentifierException ex) { if (!(ex instanceof DOIIdentifierException)) { - LOG.error("It wasn't possible to register the identifier online. ", ex); + LOG.error("Registering DOI {} for object {}: the registrar returned an error.", + doiRow.getDoi(), dso.getID(), ex); } DOIIdentifierException doiIdentifierException = (DOIIdentifierException) ex; diff --git a/dspace-api/src/main/java/org/dspace/identifier/doi/DataCiteConnector.java b/dspace-api/src/main/java/org/dspace/identifier/doi/DataCiteConnector.java index a15e3f7fdbfe..5bb37add2d9a 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/doi/DataCiteConnector.java +++ b/dspace-api/src/main/java/org/dspace/identifier/doi/DataCiteConnector.java @@ -464,6 +464,10 @@ public void reserveDOI(Context context, DSpaceObject dso, String doi) log.warn("While reserving the DOI {}, we got a http status code " + "{} and the message \"{}\".", doi, Integer.toString(resp.statusCode), resp.getContent()); + Format format = Format.getCompactFormat(); + format.setEncoding("UTF-8"); + XMLOutputter xout = new XMLOutputter(format); + log.info("We send the following XML:\n{}", xout.outputString(root)); throw new DOIIdentifierException("Unable to parse an answer from " + "DataCite API. Please have a look into DSpace logs.", DOIIdentifierException.BAD_ANSWER); @@ -635,6 +639,14 @@ protected DataCiteResponse sendGetRequest(String doi, String path) return sendHttpRequest(httpget, doi); } + /** + * Send a DataCite metadata document to the registrar. + * + * @param doi identify the object. + * @param metadataRoot describe the object. The root element of the document. + * @return the registrar's response. + * @throws DOIIdentifierException passed through. + */ protected DataCiteResponse sendMetadataPostRequest(String doi, Element metadataRoot) throws DOIIdentifierException { Format format = Format.getCompactFormat(); @@ -643,6 +655,14 @@ protected DataCiteResponse sendMetadataPostRequest(String doi, Element metadataR return sendMetadataPostRequest(doi, xout.outputString(new Document(metadataRoot))); } + /** + * Send a DataCite metadata document to the registrar. + * + * @param doi identify the object. + * @param metadata describe the object. + * @return the registrar's response. + * @throws DOIIdentifierException passed through. + */ protected DataCiteResponse sendMetadataPostRequest(String doi, String metadata) throws DOIIdentifierException { // post mds/metadata/ @@ -690,7 +710,7 @@ protected DataCiteResponse sendMetadataPostRequest(String doi, String metadata) * properties such as request URI and method type. * @param doi DOI string to operate on * @return response from DataCite - * @throws DOIIdentifierException if DOI error + * @throws DOIIdentifierException if registrar returns an error. */ protected DataCiteResponse sendHttpRequest(HttpUriRequest req, String doi) throws DOIIdentifierException { From fcc650e1a6fbc1311e905a349fc5ed3957dd1db7 Mon Sep 17 00:00:00 2001 From: Toni Prieto Date: Sat, 26 Oct 2024 23:33:13 +0200 Subject: [PATCH 066/178] Add limit, offset, and a new parameter to calculate the total entry count in the Solr query used for the metadata navigation index (cherry picked from commit e71de8a4d075d897f68c145233ac19ba0ec3718b) --- .../java/org/dspace/browse/BrowseEngine.java | 30 ++-------------- .../java/org/dspace/browse/SolrBrowseDAO.java | 32 ++++++++++++----- .../org/dspace/discovery/DiscoverResult.java | 11 ++++++ .../org/dspace/discovery/SolrServiceImpl.java | 36 ++++++++++++++++++- 4 files changed, 73 insertions(+), 36 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/browse/BrowseEngine.java b/dspace-api/src/main/java/org/dspace/browse/BrowseEngine.java index 351c36248209..be7a34086a46 100644 --- a/dspace-api/src/main/java/org/dspace/browse/BrowseEngine.java +++ b/dspace-api/src/main/java/org/dspace/browse/BrowseEngine.java @@ -422,9 +422,6 @@ private BrowseInfo browseByValue(BrowserScope bs) } } - // this is the total number of results in answer to the query - int total = getTotalResults(true); - // set the ordering field (there is only one option) dao.setOrderField("sort_value"); @@ -444,6 +441,9 @@ private BrowseInfo browseByValue(BrowserScope bs) dao.setOffset(offset); dao.setLimit(scope.getResultsPerPage()); + // this is the total number of results in answer to the query + int total = getTotalResults(true); + // Holder for the results List results = null; @@ -680,33 +680,9 @@ private int getTotalResults(boolean distinct) // tell the browse query whether we are distinct dao.setDistinct(distinct); - // ensure that the select is set to "*" - String[] select = {"*"}; - dao.setCountValues(select); - - // FIXME: it would be nice to have a good way of doing this in the DAO - // now reset all of the fields that we don't want to have constraining - // our count, storing them locally to reinstate later - String focusField = dao.getJumpToField(); - String focusValue = dao.getJumpToValue(); - int limit = dao.getLimit(); - int offset = dao.getOffset(); - - dao.setJumpToField(null); - dao.setJumpToValue(null); - dao.setLimit(-1); - dao.setOffset(-1); - // perform the query and get the result int count = dao.doCountQuery(); - // now put back the values we removed for this method - dao.setJumpToField(focusField); - dao.setJumpToValue(focusValue); - dao.setLimit(limit); - dao.setOffset(offset); - dao.setCountValues(null); - log.debug(LogHelper.getHeader(context, "get_total_results_return", "return=" + count)); return count; diff --git a/dspace-api/src/main/java/org/dspace/browse/SolrBrowseDAO.java b/dspace-api/src/main/java/org/dspace/browse/SolrBrowseDAO.java index f99aab852bf5..1917dec423ec 100644 --- a/dspace-api/src/main/java/org/dspace/browse/SolrBrowseDAO.java +++ b/dspace-api/src/main/java/org/dspace/browse/SolrBrowseDAO.java @@ -13,6 +13,8 @@ import java.util.Comparator; import java.util.List; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.util.ClientUtils; @@ -180,18 +182,33 @@ private DiscoverResult getSolrResponse() throws BrowseException { addDefaultFilterQueries(query); if (distinct) { DiscoverFacetField dff; + + // To get the number of distinct values we use the next "json.facet" query param + // {"entries_count": {"type":"terms","field": "_filter", "limit":0, "numBuckets":true}}" + ObjectNode jsonFacet = JsonNodeFactory.instance.objectNode(); + ObjectNode entriesCount = JsonNodeFactory.instance.objectNode(); + entriesCount.put("type", "terms"); + entriesCount.put("field", facetField + "_filter"); + entriesCount.put("limit", 0); + entriesCount.put("numBuckets", true); + jsonFacet.set("entries_count", entriesCount); + if (StringUtils.isNotBlank(startsWith)) { dff = new DiscoverFacetField(facetField, - DiscoveryConfigurationParameters.TYPE_TEXT, -1, - DiscoveryConfigurationParameters.SORT.VALUE, startsWith); + DiscoveryConfigurationParameters.TYPE_TEXT, limit, + DiscoveryConfigurationParameters.SORT.VALUE, startsWith, offset); + + // Add the prefix to the json facet query + entriesCount.put("prefix", startsWith); } else { dff = new DiscoverFacetField(facetField, - DiscoveryConfigurationParameters.TYPE_TEXT, -1, - DiscoveryConfigurationParameters.SORT.VALUE); + DiscoveryConfigurationParameters.TYPE_TEXT, limit, + DiscoveryConfigurationParameters.SORT.VALUE, offset); } query.addFacetField(dff); query.setFacetMinCount(1); query.setMaxResults(0); + query.addProperty("json.facet", jsonFacet.toString()); } else { query.setMaxResults(limit/* > 0 ? limit : 20*/); if (offset > 0) { @@ -248,8 +265,7 @@ public int doCountQuery() throws BrowseException { DiscoverResult resp = getSolrResponse(); int count = 0; if (distinct) { - List facetResults = resp.getFacetResult(facetField); - count = facetResults.size(); + count = (int) resp.getTotalEntries(); } else { // we need to cast to int to respect the BrowseDAO contract... count = (int) resp.getTotalSearchResults(); @@ -266,8 +282,8 @@ public List doValueQuery() throws BrowseException { DiscoverResult resp = getSolrResponse(); List facet = resp.getFacetResult(facetField); int count = doCountQuery(); - int start = offset > 0 ? offset : 0; - int max = limit > 0 ? limit : count; //if negative, return everything + int start = 0; + int max = facet.size(); List result = new ArrayList<>(); if (ascending) { for (int i = start; i < (start + max) && i < count; i++) { diff --git a/dspace-api/src/main/java/org/dspace/discovery/DiscoverResult.java b/dspace-api/src/main/java/org/dspace/discovery/DiscoverResult.java index 00236d2bfe32..a56804e3e7ea 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/DiscoverResult.java +++ b/dspace-api/src/main/java/org/dspace/discovery/DiscoverResult.java @@ -32,6 +32,9 @@ public class DiscoverResult { private List indexableObjects; private Map> facetResults; + // Total count of facet entries calculated for a metadata browsing query + private long totalEntries; + /** * A map that contains all the documents sougth after, the key is a string representation of the Indexable Object */ @@ -64,6 +67,14 @@ public void setTotalSearchResults(long totalSearchResults) { this.totalSearchResults = totalSearchResults; } + public long getTotalEntries() { + return totalEntries; + } + + public void setTotalEntries(long totalEntries) { + this.totalEntries = totalEntries; + } + public int getStart() { return start; } diff --git a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java index cd3797e3e34e..9339b574b578 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java @@ -1055,6 +1055,8 @@ protected DiscoverResult retrieveResult(Context context, DiscoverQuery query) } //Resolve our facet field values resolveFacetFields(context, query, result, skipLoadingResponse, solrQueryResponse); + //Add total entries count for metadata browsing + resolveEntriesCount(result, solrQueryResponse); } // If any stale entries are found in the current page of results, // we remove those stale entries and rerun the same query again. @@ -1080,7 +1082,39 @@ protected DiscoverResult retrieveResult(Context context, DiscoverQuery query) return result; } - + /** + * Stores the total count of entries for metadata index browsing. The count is calculated by the + * json.facet parameter with the following value: + * + * 
    
+     * {
+     *     "entries_count": {
+     *         "type": "terms",
+     *         "field": "facetNameField_filter",
+     *         "limit": 0,
+     *         "prefix": "prefix_value",
+     *         "numBuckets": true
+     *     }
+     * }
+     *
    + * + * This value is returned in the facets field of the Solr response. + * + * @param result DiscoverResult object where the total entries count will be stored + * @param solrQueryResponse QueryResponse object containing the solr response + */ + private void resolveEntriesCount(DiscoverResult result, QueryResponse solrQueryResponse) { + + Object facetsObj = solrQueryResponse.getResponse().get("facets"); + if (facetsObj instanceof NamedList) { + NamedList facets = (NamedList) facetsObj; + Object bucketsInfoObj = facets.get("entries_count"); + if (bucketsInfoObj instanceof NamedList) { + NamedList bucketsInfo = (NamedList) bucketsInfoObj; + result.setTotalEntries((int) bucketsInfo.get("numBuckets")); + } + } + } private void resolveFacetFields(Context context, DiscoverQuery query, DiscoverResult result, boolean skipLoadingResponse, QueryResponse solrQueryResponse) throws SQLException { From 95742fe4f38842fc0747188ec941b05364ec7797 Mon Sep 17 00:00:00 2001 From: Jukka Lipka <3710455+jlipka@users.noreply.github.com> Date: Mon, 21 Oct 2024 18:11:08 +0200 Subject: [PATCH 067/178] fix(submission): Submission scope naming fixed According to the documentation, the value for the property is 'submission' in the 'submission-forms.xml'. Without this change, an empty input field will never be marked as an error, even if the field is marked as 'required'. (cherry picked from commit 02f52c7d5c245eaae09dbd636fc8c7935d7cc242) --- dspace-api/src/main/java/org/dspace/app/util/DCInput.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/app/util/DCInput.java b/dspace-api/src/main/java/org/dspace/app/util/DCInput.java index 11f9aadd869b..a3d1ba208b56 100644 --- a/dspace-api/src/main/java/org/dspace/app/util/DCInput.java +++ b/dspace-api/src/main/java/org/dspace/app/util/DCInput.java @@ -163,7 +163,7 @@ public class DCInput { * The scope of the input sets, this restricts hidden metadata fields from * view by the end user during submission. */ - public static final String SUBMISSION_SCOPE = "submit"; + public static final String SUBMISSION_SCOPE = "submission"; /** * Class constructor for creating a DCInput object based on the contents of From 31faf809d15d968a021a64720b620faf5f47bf0e Mon Sep 17 00:00:00 2001 From: Jukka Lipka <3710455+jlipka@users.noreply.github.com> Date: Mon, 9 Dec 2024 10:50:59 +0100 Subject: [PATCH 068/178] fix(submission): Submission scope naming fixed Corrected wording in related code comment (cherry picked from commit ec2187ea6532d7db5f5ffa38e20e971a49b90871) --- dspace-api/src/main/java/org/dspace/app/util/DCInput.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/app/util/DCInput.java b/dspace-api/src/main/java/org/dspace/app/util/DCInput.java index a3d1ba208b56..0a1e77ee7292 100644 --- a/dspace-api/src/main/java/org/dspace/app/util/DCInput.java +++ b/dspace-api/src/main/java/org/dspace/app/util/DCInput.java @@ -262,7 +262,7 @@ protected void initRegex(String regex) { /** * Is this DCInput for display in the given scope? The scope should be - * either "workflow" or "submit", as per the input forms definition. If the + * either "workflow" or "submission", as per the input forms definition. If the * internal visibility is set to "null" then this will always return true. * * @param scope String identifying the scope that this input's visibility From 973beee2ce5f8dc993ef5eb91711b595c7786789 Mon Sep 17 00:00:00 2001 From: igorbaptist4 Date: Tue, 10 Sep 2024 18:44:15 -0300 Subject: [PATCH 069/178] fix: properly type in field id, adjust use of getProperty and add error handling when dbPath is null (when property usage-statistics.dbfile is commented (cherry picked from commit 412d5751f29f31bbdd872820a62b74970333f066) --- .../org/dspace/statistics/util/StatisticsImporter.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java index 95736a8bd6d9..1bd97a6f5ec9 100644 --- a/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java +++ b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java @@ -357,7 +357,7 @@ protected void load(String filename, Context context, boolean verbose) { SolrInputDocument sid = new SolrInputDocument(); sid.addField("ip", ip); sid.addField("type", dso.getType()); - sid.addField("id", dso.getID()); + sid.addField("id", dso.getID().toString()); sid.addField("time", DateFormatUtils.format(date, SolrLoggerServiceImpl.DATE_FORMAT_8601)); sid.addField("continent", continent); sid.addField("country", country); @@ -471,13 +471,13 @@ public static void main(String[] args) throws Exception { boolean verbose = line.hasOption('v'); // Find our solr server - String sserver = configurationService.getProperty("solr-statistics", "server"); + String sserver = configurationService.getProperty("solr-statistics.server"); if (verbose) { System.out.println("Writing to solr server at: " + sserver); } solr = new HttpSolrClient.Builder(sserver).build(); - String dbPath = configurationService.getProperty("usage-statistics", "dbfile"); + String dbPath = configurationService.getProperty("usage-statistics.dbfile"); try { File dbFile = new File(dbPath); geoipLookup = new DatabaseReader.Builder(dbFile).build(); @@ -492,6 +492,10 @@ public static void main(String[] args) throws Exception { "Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the DSpace " + "installation instructions for more details.", e); + } catch (NullPointerException e) { + log.error( + "The value of the property usage-statistics.dbfile is null. You may need to install the GeoLite Database file and/or uncomment the property in the config file!", + e); } From 3a223e3fefec9172538e7defa34ce982b0d72762 Mon Sep 17 00:00:00 2001 From: igorbaptist4 Date: Tue, 10 Sep 2024 20:16:47 -0300 Subject: [PATCH 070/178] fix line length checkstyle (cherry picked from commit 338f3b1d3efaa0f7f3852654121cf1f93adeae7e) --- .../java/org/dspace/statistics/util/StatisticsImporter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java index 1bd97a6f5ec9..354c803fe2ae 100644 --- a/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java +++ b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java @@ -494,7 +494,8 @@ public static void main(String[] args) throws Exception { e); } catch (NullPointerException e) { log.error( - "The value of the property usage-statistics.dbfile is null. You may need to install the GeoLite Database file and/or uncomment the property in the config file!", + "The value of the property usage-statistics.dbfile is null. You may need to install the GeoLite " + + "Database file and/or uncomment the property in the config file!", e); } From c86d082d5f6fe4a76eae018bbe6890cc0b72ff33 Mon Sep 17 00:00:00 2001 From: igorbaptist4 Date: Tue, 10 Sep 2024 13:10:14 -0300 Subject: [PATCH 071/178] fix: set default configFile (cherry picked from commit a5e8d7aa15bc56c268ba67f19657f19edaa45253) --- .../java/org/dspace/app/statistics/LogAnalyser.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java b/dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java index 2e4ed69b268e..c787261419f8 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java @@ -281,10 +281,14 @@ public class LogAnalyser { */ private static String fileTemplate = "dspace\\.log.*"; + private static final ConfigurationService configurationService = + DSpaceServicesFactory.getInstance().getConfigurationService(); + /** * the configuration file from which to configure the analyser */ - private static String configFile; + private static String configFile = configurationService.getProperty("dspace.dir") + + File.separator + "config" + File.separator + "dstat.cfg"; /** * the output file to which to write aggregation data @@ -616,8 +620,6 @@ public static String processLogs(Context context, String myLogDir, } // now do the host name and url lookup - ConfigurationService configurationService - = DSpaceServicesFactory.getInstance().getConfigurationService(); hostName = Utils.getHostName(configurationService.getProperty("dspace.ui.url")); name = configurationService.getProperty("dspace.name").trim(); url = configurationService.getProperty("dspace.ui.url").trim(); @@ -658,8 +660,6 @@ public static void setParameters(String myLogDir, String myFileTemplate, String myConfigFile, String myOutFile, Date myStartDate, Date myEndDate, boolean myLookUp) { - ConfigurationService configurationService - = DSpaceServicesFactory.getInstance().getConfigurationService(); if (myLogDir != null) { logDir = myLogDir; @@ -673,9 +673,6 @@ public static void setParameters(String myLogDir, String myFileTemplate, if (myConfigFile != null) { configFile = myConfigFile; - } else { - configFile = configurationService.getProperty("dspace.dir") - + File.separator + "config" + File.separator + "dstat.cfg"; } if (myStartDate != null) { From 1ce4e08333559bffadde8d266a1009c1c875df33 Mon Sep 17 00:00:00 2001 From: DSpace Bot <68393067+dspace-bot@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:45:16 -0600 Subject: [PATCH 072/178] [Port dspace-7_x] Fix issue with submission sections visibility (#10140) * README.md: v8 is the current release, not v7 (cherry picked from commit 2b698eff609d510c487ad2331d4e11cd28f64e9a) (cherry picked from commit 83460afb3700a2ddaabce16069b2094e4edbe8fa) * Update README.md (cherry picked from commit 671234b08f909810d798dd950f87d1818b098363) (cherry picked from commit 7a6785b1c353cbc45c4dfaad89f5252064ef1e4e) * [DURACOM-291] Expose section scope attribute (cherry picked from commit 4107f937fda1b49da2b6a81dc91026ebf4b1cd37) * README.md: v8 is the current release, not v7 (cherry picked from commit 2b698eff609d510c487ad2331d4e11cd28f64e9a) (cherry picked from commit d98499a39416d026c0b45199e90ddb9bbfc1cd9c) * Update README.md (cherry picked from commit 671234b08f909810d798dd950f87d1818b098363) (cherry picked from commit 6a707548ffa6929e8a36775063ce05e2dc02a586) --------- Co-authored-by: Christian Clauss Co-authored-by: Giuseppe Digilio --- .../dspace/app/rest/converter/SubmissionSectionConverter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SubmissionSectionConverter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SubmissionSectionConverter.java index 0391cbce7a2d..3cd263493b5d 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SubmissionSectionConverter.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SubmissionSectionConverter.java @@ -10,6 +10,7 @@ import java.sql.SQLException; import org.apache.logging.log4j.Logger; +import org.dspace.app.rest.model.ScopeEnum; import org.dspace.app.rest.model.SubmissionSectionRest; import org.dspace.app.rest.model.SubmissionVisibilityRest; import org.dspace.app.rest.model.VisibilityEnum; @@ -41,6 +42,7 @@ public SubmissionSectionRest convert(SubmissionStepConfig step, Projection proje sp.setHeader(step.getHeading()); sp.setSectionType(step.getType()); sp.setId(step.getId()); + sp.setScope(ScopeEnum.fromString(step.getScope())); sp.setVisibility(new SubmissionVisibilityRest(VisibilityEnum.fromString(step.getVisibility()), VisibilityEnum.fromString(step.getVisibilityOutside()))); return sp; From 51df5c1fe67b7660dafa85d57f688347ca7f024d Mon Sep 17 00:00:00 2001 From: Kristof De Langhe Date: Mon, 2 Dec 2024 15:35:10 +0100 Subject: [PATCH 073/178] 119664: Search event scope fix (cherry picked from commit 48956d90b7619bdb336632b454ac62ca62967a39) --- .../org/dspace/app/rest/converter/SearchEventConverter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SearchEventConverter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SearchEventConverter.java index 978ae2ca9230..f2fb12f2bda9 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SearchEventConverter.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SearchEventConverter.java @@ -67,8 +67,8 @@ public UsageSearchEvent convert(Context context, HttpServletRequest request, Sea if (searchEventRest.getScope() != null) { IndexableObject scopeObject = scopeResolver.resolveScope(context, String.valueOf(searchEventRest.getScope())); - if (scopeObject instanceof DSpaceObject) { - usageSearchEvent.setScope((DSpaceObject) scopeObject); + if (scopeObject != null && scopeObject.getIndexedObject() instanceof DSpaceObject) { + usageSearchEvent.setScope((DSpaceObject) scopeObject.getIndexedObject()); } } usageSearchEvent.setConfiguration(searchEventRest.getConfiguration()); From 8849212895d987abaca92865178c6b4d2f99452d Mon Sep 17 00:00:00 2001 From: Chris Wilper Date: Wed, 28 Apr 2021 09:29:36 -0400 Subject: [PATCH 074/178] Add Context method to uncache all entities (cherry picked from commit 8ea664adb2161b04ab8b6171bbb426d8875fe27f) --- .../main/java/org/dspace/core/Context.java | 14 +++++++- .../java/org/dspace/core/DBConnection.java | 32 ++++++++++++------- .../dspace/core/HibernateDBConnection.java | 5 +++ .../java/org/dspace/core/ContextTest.java | 25 +++++++++++++++ .../core/HibernateDBConnectionTest.java | 22 +++++++++++++ 5 files changed, 86 insertions(+), 12 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/core/Context.java b/dspace-api/src/main/java/org/dspace/core/Context.java index 02a3fee09f8a..dab6ab7fbd66 100644 --- a/dspace-api/src/main/java/org/dspace/core/Context.java +++ b/dspace-api/src/main/java/org/dspace/core/Context.java @@ -883,7 +883,19 @@ public E reloadEntity(E entity) throws SQLException } /** - * Remove an entity from the cache. This is necessary when batch processing a large number of items. + * Remove all entities from the cache and reload the current user entity. This is useful when batch processing + * a large number of entities when the calling code requires the cache to be completely cleared before continuing. + * + * @throws SQLException if a database error occurs. + */ + public void uncacheEntities() throws SQLException { + dbConnection.uncacheEntities(); + reloadContextBoundEntities(); + } + + /** + * Remove an entity from the cache. This is useful when batch processing a large number of entities + * when the calling code needs to retain some items in the cache while removing others. * * @param entity The entity to reload * @param The class of the entity. The entity must implement the {@link ReloadableEntity} interface. diff --git a/dspace-api/src/main/java/org/dspace/core/DBConnection.java b/dspace-api/src/main/java/org/dspace/core/DBConnection.java index 66e4a65dbfe1..c9c4ce0953e4 100644 --- a/dspace-api/src/main/java/org/dspace/core/DBConnection.java +++ b/dspace-api/src/main/java/org/dspace/core/DBConnection.java @@ -124,28 +124,38 @@ public interface DBConnection { public long getCacheSize() throws SQLException; /** - * Reload a DSpace object from the database. This will make sure the object + * Reload an entity from the database. This will make sure the object * is valid and stored in the cache. The returned object should be used * henceforth instead of the passed object. * - * @param type of {@link entity} - * @param entity The DSpace object to reload + * @param type of entity. + * @param entity The entity to reload. * @return the reloaded entity. - * @throws java.sql.SQLException passed through. + * @throws SQLException passed through. */ public E reloadEntity(E entity) throws SQLException; /** - * Remove a DSpace object from the session cache when batch processing a - * large number of objects. + * Remove all entities from the session cache. * - *

    Objects removed from cache are not saved in any way. Therefore, if you - * have modified an object, you should be sure to {@link commit()} changes + *

    Entities removed from cache are not saved in any way. Therefore, if you + * have modified any entities, you should be sure to {@link #commit()} changes * before calling this method. * - * @param Type of {@link entity} - * @param entity The DSpace object to decache. - * @throws java.sql.SQLException passed through. + * @throws SQLException passed through. + */ + public void uncacheEntities() throws SQLException; + + /** + * Remove an entity from the session cache. + * + *

    Entities removed from cache are not saved in any way. Therefore, if you + * have modified the entity, you should be sure to {@link #commit()} changes + * before calling this method. + * + * @param Type of entity. + * @param entity The entity to decache. + * @throws SQLException passed through. */ public void uncacheEntity(E entity) throws SQLException; diff --git a/dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java b/dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java index b371af80eede..bd00b844ba9a 100644 --- a/dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java +++ b/dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java @@ -243,6 +243,11 @@ private void configureDatabaseMode() throws SQLException { } } + @Override + public void uncacheEntities() throws SQLException { + getSession().clear(); + } + /** * Evict an entity from the hibernate cache. *

    diff --git a/dspace-api/src/test/java/org/dspace/core/ContextTest.java b/dspace-api/src/test/java/org/dspace/core/ContextTest.java index c6cd849d2110..ccc1d2f732cc 100644 --- a/dspace-api/src/test/java/org/dspace/core/ContextTest.java +++ b/dspace-api/src/test/java/org/dspace/core/ContextTest.java @@ -558,4 +558,29 @@ protected void init() { cleanupContext(instance); } + @Test + public void testUncacheEntities() throws Throwable { + // To set up the test, ensure the cache contains more than the current user entity + groupService.findByName(context, Group.ANONYMOUS); + assertTrue("Cache size should be greater than one", context.getDBConnection().getCacheSize() > 1); + + context.uncacheEntities(); + + assertThat("Cache size should be one (current user)", context.getDBConnection().getCacheSize(), equalTo(1L)); + context.reloadEntity(context.getCurrentUser()); + assertThat("Cache should only contain the current user", context.getDBConnection().getCacheSize(), equalTo(1L)); + } + + @Test + public void testUncacheEntity() throws Throwable { + // Remember the cache size after loading an entity + Group group = groupService.findByName(context, Group.ANONYMOUS); + long oldCacheSize = context.getDBConnection().getCacheSize(); + + // Uncache the entity + context.uncacheEntity(group); + + long newCacheSize = context.getDBConnection().getCacheSize(); + assertThat("Cache size should be reduced by one", newCacheSize, equalTo(oldCacheSize - 1)); + } } diff --git a/dspace-api/src/test/java/org/dspace/core/HibernateDBConnectionTest.java b/dspace-api/src/test/java/org/dspace/core/HibernateDBConnectionTest.java index 093f693d567f..302844ce62ac 100644 --- a/dspace-api/src/test/java/org/dspace/core/HibernateDBConnectionTest.java +++ b/dspace-api/src/test/java/org/dspace/core/HibernateDBConnectionTest.java @@ -205,6 +205,28 @@ public void testReloadEntityAfterCommit() throws SQLException { .contains(person)); } + /** + * Test of uncacheEntities method + */ + @Test + public void testUncacheEntities() throws SQLException { + // Get DBConnection associated with DSpace Context + HibernateDBConnection dbConnection = (HibernateDBConnection) context.getDBConnection(); + EPerson person = context.getCurrentUser(); + + assertTrue("Current user should be cached in session", dbConnection.getSession() + .contains(person)); + + dbConnection.uncacheEntities(); + assertFalse("Current user should be gone from cache", dbConnection.getSession() + .contains(person)); + + // Test ability to reload an uncached entity + person = dbConnection.reloadEntity(person); + assertTrue("Current user should be cached back in session", dbConnection.getSession() + .contains(person)); + } + /** * Test of uncacheEntity method */ From e856ae3291831feac74f01133f7e235d8513cba0 Mon Sep 17 00:00:00 2001 From: Toni Prieto Date: Sun, 27 Oct 2024 08:56:10 +0100 Subject: [PATCH 075/178] Uncache all entities during OAI indexing to reduce memory usage (cherry picked from commit 9af2e2e17cf289a2a6921d27a21d90db6ca8b9f9) --- dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index c6aaaa34b539..2d2679577802 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -334,6 +334,11 @@ private int index(Iterator iterator) throws DSpaceSolrIndexerException { server.add(list); server.commit(); list.clear(); + try { + context.uncacheEntities(); + } catch (SQLException ex) { + log.error("Error uncaching entities", ex); + } } } System.out.println("Total: " + i + " items"); From 964e4bf476b9cb49ce07c5d05242c63e9ae638d4 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Mon, 23 Dec 2024 12:26:58 +0100 Subject: [PATCH 076/178] remove usage of deprecated constructor call (cherry picked from commit 45cdb4d9d47d3151a4672e99152a0b3bedb6cd18) --- dspace-api/src/main/java/org/dspace/curate/Curation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/curate/Curation.java b/dspace-api/src/main/java/org/dspace/curate/Curation.java index 4d70286e79e0..ea47d817c07d 100644 --- a/dspace-api/src/main/java/org/dspace/curate/Curation.java +++ b/dspace-api/src/main/java/org/dspace/curate/Curation.java @@ -185,7 +185,7 @@ private Curator initCurator() throws FileNotFoundException { Curator curator = new Curator(handler); OutputStream reporterStream; if (null == this.reporter) { - reporterStream = new NullOutputStream(); + reporterStream = NullOutputStream.NULL_OUTPUT_STREAM; } else if ("-".equals(this.reporter)) { reporterStream = System.out; } else { From 214c669e8063f93eca2f878610647cc3b87346bd Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Wed, 11 Dec 2024 15:35:26 +0100 Subject: [PATCH 077/178] fix missing +1 offset (cherry picked from commit ab00de05b46fe98306e0047af7d6d98bba9d0077) --- dspace-oai/src/main/resources/static/style.xsl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dspace-oai/src/main/resources/static/style.xsl b/dspace-oai/src/main/resources/static/style.xsl index 17eb865e8f1f..67aeb975b26b 100644 --- a/dspace-oai/src/main/resources/static/style.xsl +++ b/dspace-oai/src/main/resources/static/style.xsl @@ -522,15 +522,14 @@ - + - - + - - + From 6c82e2dba40e50e72ab8459995802a5d7f526a53 Mon Sep 17 00:00:00 2001 From: Koen Pauwels Date: Tue, 7 Jan 2025 15:51:28 +0100 Subject: [PATCH 078/178] Bugfix: Enforce unique item id in workspace table (#9340) * 106798 Enforce values in item_id column of workspaceitem table to be unique, both at database level and at WorkspaceItemService level * 106798 Removed Oracle SQL migration * 106798 workspaceitem table migration: delete duplicate rows before introducing uniqueness constraint * 106798: update migration for H2 --------- Co-authored-by: Koen Pauwels Co-authored-by: wout --- .../content/WorkspaceItemServiceImpl.java | 8 +++ ...paceitem_add_item_id_unique_constraint.sql | 21 +++++++ ...paceitem_add_item_id_unique_constraint.sql | 21 +++++++ .../org/dspace/content/WorkspaceItemTest.java | 12 ++++ .../org/dspace/workflow/MockWorkflowItem.java | 62 +++++++++++++++++++ 5 files changed, 124 insertions(+) create mode 100644 dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2024.12.17__workspaceitem_add_item_id_unique_constraint.sql create mode 100644 dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2024.12.17__workspaceitem_add_item_id_unique_constraint.sql create mode 100644 dspace-api/src/test/java/org/dspace/workflow/MockWorkflowItem.java diff --git a/dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java index 1da9e6e44a6a..543f5a55efe2 100644 --- a/dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java @@ -178,6 +178,14 @@ public WorkspaceItem create(Context context, Collection collection, UUID uuid, b @Override public WorkspaceItem create(Context c, WorkflowItem workflowItem) throws SQLException, AuthorizeException { + WorkspaceItem potentialDuplicate = findByItem(c, workflowItem.getItem()); + if (potentialDuplicate != null) { + throw new IllegalArgumentException(String.format( + "A workspace item referring to item %s already exists (%d)", + workflowItem.getItem().getID(), + potentialDuplicate.getID() + )); + } WorkspaceItem workspaceItem = workspaceItemDAO.create(c, new WorkspaceItem()); workspaceItem.setItem(workflowItem.getItem()); workspaceItem.setCollection(workflowItem.getCollection()); diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2024.12.17__workspaceitem_add_item_id_unique_constraint.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2024.12.17__workspaceitem_add_item_id_unique_constraint.sql new file mode 100644 index 000000000000..38389bf2d19b --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2024.12.17__workspaceitem_add_item_id_unique_constraint.sql @@ -0,0 +1,21 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +-- In the workspaceitem table, if there are multiple rows referring to the same item ID, keep only the first of them. +DELETE FROM workspaceitem WHERE EXISTS ( + SELECT item_id + FROM workspaceitem + GROUP BY item_id + HAVING COUNT(workspace_item_id) > 1 +) AND workspaceitem.workspace_item_id NOT IN ( + SELECT MIN(workspace_item_id) AS workspace_item_id + FROM workspaceitem + GROUP BY item_id +); +-- Identify which rows have duplicates, and compute their replacements. +ALTER TABLE workspaceitem ADD CONSTRAINT unique_item_id UNIQUE(item_id); diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2024.12.17__workspaceitem_add_item_id_unique_constraint.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2024.12.17__workspaceitem_add_item_id_unique_constraint.sql new file mode 100644 index 000000000000..20eb0f9119d3 --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2024.12.17__workspaceitem_add_item_id_unique_constraint.sql @@ -0,0 +1,21 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +-- In the workspaceitem table, if there are multiple rows referring to the same item ID, keep only the first of them. +WITH dedup AS ( + SELECT item_id, MIN(workspace_item_id) AS workspace_item_id + FROM workspaceitem + GROUP BY item_id + HAVING COUNT(workspace_item_id) > 1 +) +DELETE FROM workspaceitem +USING dedup +WHERE workspaceitem.item_id = dedup.item_id AND workspaceitem.workspace_item_id <> dedup.workspace_item_id; + +-- Enforce uniqueness of item_id in workspaceitem table. +ALTER TABLE workspaceitem ADD CONSTRAINT unique_item_id UNIQUE(item_id); diff --git a/dspace-api/src/test/java/org/dspace/content/WorkspaceItemTest.java b/dspace-api/src/test/java/org/dspace/content/WorkspaceItemTest.java index d018a15f9765..15d4720c9378 100644 --- a/dspace-api/src/test/java/org/dspace/content/WorkspaceItemTest.java +++ b/dspace-api/src/test/java/org/dspace/content/WorkspaceItemTest.java @@ -12,6 +12,7 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -39,6 +40,7 @@ import org.dspace.eperson.EPerson; import org.dspace.eperson.factory.EPersonServiceFactory; import org.dspace.eperson.service.EPersonService; +import org.dspace.workflow.MockWorkflowItem; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -468,4 +470,14 @@ public void testSetPublishedBefore() { assertTrue("testSetPublishedBefore 0", wi.isPublishedBefore()); } + @Test + public void testDuplicateItemID() throws Exception { + context.turnOffAuthorisationSystem(); + Item item = wi.getItem(); + MockWorkflowItem wfItem = new MockWorkflowItem(); + wfItem.item = item; + wfItem.collection = collection; + assertThrows(IllegalArgumentException.class, () -> workspaceItemService.create(context, wfItem)); + context.restoreAuthSystemState(); + } } diff --git a/dspace-api/src/test/java/org/dspace/workflow/MockWorkflowItem.java b/dspace-api/src/test/java/org/dspace/workflow/MockWorkflowItem.java new file mode 100644 index 000000000000..d36ecf7331b8 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/workflow/MockWorkflowItem.java @@ -0,0 +1,62 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.workflow; + +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.eperson.EPerson; + +public class MockWorkflowItem implements WorkflowItem { + public Integer id; + public Item item; + public Collection collection; + public EPerson submitter; + boolean hasMultipleFiles; + boolean hasMultipleTitles; + boolean isPublishedBefore; + + public Integer getID() { + return id; + } + + public Item getItem() { + return item; + } + + public Collection getCollection() { + return collection; + } + + public EPerson getSubmitter() { + return submitter; + } + + public boolean hasMultipleFiles() { + return hasMultipleFiles; + } + + public void setMultipleFiles(boolean b) { + hasMultipleFiles = b; + } + + public boolean hasMultipleTitles() { + return hasMultipleTitles; + } + + public void setMultipleTitles(boolean b) { + hasMultipleTitles = b; + } + + public boolean isPublishedBefore() { + return isPublishedBefore; + } + + public void setPublishedBefore(boolean b) { + isPublishedBefore = b; + } +} From 22511a17b67c8034b039634e7bf98d9e9a58ae5d Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 6 Nov 2024 12:03:31 -0600 Subject: [PATCH 079/178] Refactor identifier ITs to ensure they unregister all utilized IdentifierProviders which are non-default. Cannot use "getApplicationContext().refresh()" as that seems to result in empty test database in Hibernate 6.6. (cherry picked from commit cfca2adbb1d597761b3169a5a46170de8cabfa4f) --- .../general/CreateMissingIdentifiersIT.java | 48 +++---------- .../AbstractIdentifierProviderIT.java | 68 +++++++++++++++++++ .../VersionedHandleIdentifierProviderIT.java | 49 ++----------- 3 files changed, 84 insertions(+), 81 deletions(-) create mode 100644 dspace-api/src/test/java/org/dspace/identifier/AbstractIdentifierProviderIT.java diff --git a/dspace-api/src/test/java/org/dspace/ctask/general/CreateMissingIdentifiersIT.java b/dspace-api/src/test/java/org/dspace/ctask/general/CreateMissingIdentifiersIT.java index 8038a7153325..4934404c3b3e 100644 --- a/dspace-api/src/test/java/org/dspace/ctask/general/CreateMissingIdentifiersIT.java +++ b/dspace-api/src/test/java/org/dspace/ctask/general/CreateMissingIdentifiersIT.java @@ -10,10 +10,7 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; import org.dspace.builder.ItemBuilder; @@ -21,13 +18,11 @@ import org.dspace.content.Item; import org.dspace.core.factory.CoreServiceFactory; import org.dspace.curate.Curator; -import org.dspace.identifier.IdentifierProvider; -import org.dspace.identifier.IdentifierServiceImpl; +import org.dspace.identifier.AbstractIdentifierProviderIT; +import org.dspace.identifier.VersionedHandleIdentifierProvider; import org.dspace.identifier.VersionedHandleIdentifierProviderWithCanonicalHandles; -import org.dspace.kernel.ServiceManager; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; -import org.junit.After; import org.junit.Test; /** @@ -36,30 +31,19 @@ * @author mwood */ public class CreateMissingIdentifiersIT - extends AbstractIntegrationTestWithDatabase { - private ServiceManager serviceManager; - private IdentifierServiceImpl identifierService; + extends AbstractIdentifierProviderIT { + private static final String P_TASK_DEF = "plugin.named.org.dspace.curate.CurationTask"; private static final String TASK_NAME = "test"; - @Override - public void setUp() throws Exception { - super.setUp(); - context.turnOffAuthorisationSystem(); - - serviceManager = DSpaceServicesFactory.getInstance().getServiceManager(); - identifierService = serviceManager.getServicesByType(IdentifierServiceImpl.class).get(0); - // Clean out providers to avoid any being used for creation of community and collection - identifierService.setProviders(new ArrayList<>()); - } + private ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); @Test public void testPerform() throws IOException { // Must remove any cached named plugins before creating a new one CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); - ConfigurationService configurationService = kernelImpl.getConfigurationService(); // Define a new task dynamically configurationService.setProperty(P_TASK_DEF, CreateMissingIdentifiers.class.getCanonicalName() + " = " + TASK_NAME); @@ -76,7 +60,7 @@ public void testPerform() .build(); /* - * Curate with regular test configuration -- should succeed. + * Curate with default Handle Provider */ curator.curate(context, item); int status = curator.getStatus(TASK_NAME); @@ -92,22 +76,10 @@ public void testPerform() curator.getResult(TASK_NAME)); assertEquals("Curation should fail", Curator.CURATE_ERROR, curator.getStatus(TASK_NAME)); - } - - @Override - @After - public void destroy() throws Exception { - super.destroy(); - DSpaceServicesFactory.getInstance().getServiceManager().getApplicationContext().refresh(); - } - - private void registerProvider(Class type) { - // Register our new provider - serviceManager.registerServiceClass(type.getName(), type); - IdentifierProvider identifierProvider = - (IdentifierProvider) serviceManager.getServiceByName(type.getName(), type); - // Overwrite the identifier-service's providers with the new one to ensure only this provider is used - identifierService.setProviders(List.of(identifierProvider)); + // Unregister this non-default provider + unregisterProvider(VersionedHandleIdentifierProviderWithCanonicalHandles.class); + // Re-register the default provider (for later tests which may depend on it) + registerProvider(VersionedHandleIdentifierProvider.class); } } diff --git a/dspace-api/src/test/java/org/dspace/identifier/AbstractIdentifierProviderIT.java b/dspace-api/src/test/java/org/dspace/identifier/AbstractIdentifierProviderIT.java new file mode 100644 index 000000000000..6ec9efddf5e2 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/identifier/AbstractIdentifierProviderIT.java @@ -0,0 +1,68 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.identifier; + +import java.util.ArrayList; +import java.util.List; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.kernel.ServiceManager; +import org.dspace.services.factory.DSpaceServicesFactory; + +/** + * AbstractIdentifierProviderIT which contains a few useful utility methods for IdentifierProvider Integration Tests + */ +public class AbstractIdentifierProviderIT extends AbstractIntegrationTestWithDatabase { + + protected final ServiceManager serviceManager = DSpaceServicesFactory.getInstance().getServiceManager(); + protected final IdentifierServiceImpl identifierService = + serviceManager.getServicesByType(IdentifierServiceImpl.class).get(0); + + /** + * Register a specific IdentifierProvider into the current IdentifierService (replacing any existing providers). + * This method will also ensure the IdentifierProvider service is registered in the DSpace Service Manager. + * @param type IdentifierProvider Class + */ + protected void registerProvider(Class type) { + // Register our new provider + IdentifierProvider identifierProvider = + (IdentifierProvider) DSpaceServicesFactory.getInstance().getServiceManager() + .getServiceByName(type.getName(), type); + if (identifierProvider == null) { + DSpaceServicesFactory.getInstance().getServiceManager().registerServiceClass(type.getName(), type); + identifierProvider = (IdentifierProvider) DSpaceServicesFactory.getInstance().getServiceManager() + .getServiceByName(type.getName(), type); + } + + identifierService.setProviders(List.of(identifierProvider)); + } + + /** + * Unregister a specific IdentifierProvider from the current IdentifierService (removing all existing providers). + * This method will also ensure the IdentifierProvider service is unregistered in the DSpace Service Manager, + * which ensures it does not conflict with other IdentifierProvider services. + * @param type IdentifierProvider Class + */ + protected void unregisterProvider(Class type) { + // Find the provider service + IdentifierProvider identifierProvider = + (IdentifierProvider) DSpaceServicesFactory.getInstance().getServiceManager() + .getServiceByName(type.getName(), type); + // If found, unregister it + if (identifierProvider == null) { + DSpaceServicesFactory.getInstance().getServiceManager().unregisterService(type.getName()); + } + + // Overwrite the identifier-service's providers with an empty list + identifierService.setProviders(new ArrayList<>()); + } + +} + + + diff --git a/dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java b/dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java index 57acf1f1c453..8c0f7e5b5e10 100644 --- a/dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java +++ b/dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java @@ -11,10 +11,7 @@ import static org.junit.Assert.assertTrue; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.authorize.AuthorizeException; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; @@ -22,15 +19,10 @@ import org.dspace.builder.VersionBuilder; import org.dspace.content.Collection; import org.dspace.content.Item; -import org.dspace.kernel.ServiceManager; -import org.dspace.services.factory.DSpaceServicesFactory; -import org.junit.After; import org.junit.Before; import org.junit.Test; -public class VersionedHandleIdentifierProviderIT extends AbstractIntegrationTestWithDatabase { - private ServiceManager serviceManager; - private IdentifierServiceImpl identifierService; +public class VersionedHandleIdentifierProviderIT extends AbstractIdentifierProviderIT { private String firstHandle; @@ -44,12 +36,6 @@ public class VersionedHandleIdentifierProviderIT extends AbstractIntegrationTest public void setUp() throws Exception { super.setUp(); context.turnOffAuthorisationSystem(); - - serviceManager = DSpaceServicesFactory.getInstance().getServiceManager(); - identifierService = serviceManager.getServicesByType(IdentifierServiceImpl.class).get(0); - // Clean out providers to avoid any being used for creation of community and collection - identifierService.setProviders(new ArrayList<>()); - parentCommunity = CommunityBuilder.createCommunity(context) .withName("Parent Community") .build(); @@ -58,33 +44,6 @@ public void setUp() throws Exception { .build(); } - @After - @Override - public void destroy() throws Exception { - super.destroy(); - // After this test has finished running, refresh application context and - // set the expected 'default' versioned handle provider back to ensure other tests don't fail - DSpaceServicesFactory.getInstance().getServiceManager().getApplicationContext().refresh(); - } - - private void registerProvider(Class type) { - // Register our new provider - IdentifierProvider identifierProvider = - (IdentifierProvider) DSpaceServicesFactory.getInstance().getServiceManager() - .getServiceByName(type.getName(), type); - if (identifierProvider == null) { - DSpaceServicesFactory.getInstance().getServiceManager().registerServiceClass(type.getName(), type); - identifierProvider = (IdentifierProvider) DSpaceServicesFactory.getInstance().getServiceManager() - .getServiceByName(type.getName(), type); - } - - // Overwrite the identifier-service's providers with the new one to ensure only this provider is used - identifierService = DSpaceServicesFactory.getInstance().getServiceManager() - .getServicesByType(IdentifierServiceImpl.class).get(0); - identifierService.setProviders(new ArrayList<>()); - identifierService.setProviders(List.of(identifierProvider)); - } - private void createVersions() throws SQLException, AuthorizeException { itemV1 = ItemBuilder.createItem(context, collection) .withTitle("First version") @@ -96,7 +55,6 @@ private void createVersions() throws SQLException, AuthorizeException { @Test public void testDefaultVersionedHandleProvider() throws Exception { - registerProvider(VersionedHandleIdentifierProvider.class); createVersions(); // Confirm the original item only has its original handle @@ -125,6 +83,11 @@ public void testCanonicalVersionedHandleProvider() throws Exception { assertEquals(firstHandle, itemV3.getHandle()); assertEquals(2, itemV3.getHandles().size()); containsHandle(itemV3, firstHandle + ".3"); + + // Unregister this non-default provider + unregisterProvider(VersionedHandleIdentifierProviderWithCanonicalHandles.class); + // Re-register the default provider (for later tests) + registerProvider(VersionedHandleIdentifierProvider.class); } private void containsHandle(Item item, String handle) { From 41207d5dee23c78d9f8a8ef93be89ed331607b13 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Fri, 8 Nov 2024 10:17:31 -0600 Subject: [PATCH 080/178] Update CreateMissingIdentifiers to better identify when CanonicalHandles provider is enabled. Update CreateMissingIdentifiersIT to verify that we are accurately resetting to our default IdentifierProvider (cherry picked from commit 2385c13f2d48048306a1280d88da5d8c022cc24f) --- .../general/CreateMissingIdentifiers.java | 28 +++++++++---------- .../identifier/IdentifierServiceImpl.java | 5 ++++ .../identifier/service/IdentifierService.java | 6 ++++ .../general/CreateMissingIdentifiersIT.java | 17 +++++------ 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/ctask/general/CreateMissingIdentifiers.java b/dspace-api/src/main/java/org/dspace/ctask/general/CreateMissingIdentifiers.java index 9639461426ef..0734d60946bc 100644 --- a/dspace-api/src/main/java/org/dspace/ctask/general/CreateMissingIdentifiers.java +++ b/dspace-api/src/main/java/org/dspace/ctask/general/CreateMissingIdentifiers.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.sql.SQLException; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -25,7 +26,6 @@ import org.dspace.identifier.VersionedHandleIdentifierProviderWithCanonicalHandles; import org.dspace.identifier.factory.IdentifierServiceFactory; import org.dspace.identifier.service.IdentifierService; -import org.dspace.services.factory.DSpaceServicesFactory; /** * Ensure that an object has all of the identifiers that it should, minting them @@ -45,20 +45,6 @@ public int perform(DSpaceObject dso) return Curator.CURATE_SKIP; } - // XXX Temporary escape when an incompatible provider is configured. - // XXX Remove this when the provider is fixed. - boolean compatible = DSpaceServicesFactory - .getInstance() - .getServiceManager() - .getServiceByName( - VersionedHandleIdentifierProviderWithCanonicalHandles.class.getCanonicalName(), - IdentifierProvider.class) == null; - if (!compatible) { - setResult("This task is not compatible with VersionedHandleIdentifierProviderWithCanonicalHandles"); - return Curator.CURATE_ERROR; - } - // XXX End of escape - String typeText = Constants.typeText[dso.getType()]; // Get a Context @@ -75,6 +61,18 @@ public int perform(DSpaceObject dso) .getInstance() .getIdentifierService(); + // XXX Temporary escape when an incompatible provider is configured. + // XXX Remove this when the provider is fixed. + List providerList = identifierService.getProviders(); + boolean compatible = + providerList.stream().noneMatch(p -> p instanceof VersionedHandleIdentifierProviderWithCanonicalHandles); + + if (!compatible) { + setResult("This task is not compatible with VersionedHandleIdentifierProviderWithCanonicalHandles"); + return Curator.CURATE_ERROR; + } + // XXX End of escape + // Register any missing identifiers. try { identifierService.register(context, dso); diff --git a/dspace-api/src/main/java/org/dspace/identifier/IdentifierServiceImpl.java b/dspace-api/src/main/java/org/dspace/identifier/IdentifierServiceImpl.java index b98aea24fa08..e6dcfcdda693 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/IdentifierServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/identifier/IdentifierServiceImpl.java @@ -57,6 +57,11 @@ public void setProviders(List providers) { } } + @Override + public List getProviders() { + return this.providers; + } + /** * Reserves identifiers for the item * diff --git a/dspace-api/src/main/java/org/dspace/identifier/service/IdentifierService.java b/dspace-api/src/main/java/org/dspace/identifier/service/IdentifierService.java index 23005b657508..45bf3c6dea8a 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/service/IdentifierService.java +++ b/dspace-api/src/main/java/org/dspace/identifier/service/IdentifierService.java @@ -19,6 +19,7 @@ import org.dspace.identifier.IdentifierException; import org.dspace.identifier.IdentifierNotFoundException; import org.dspace.identifier.IdentifierNotResolvableException; +import org.dspace.identifier.IdentifierProvider; /** * @author Fabio Bolognesi (fabio at atmire dot com) @@ -194,4 +195,9 @@ void register(Context context, DSpaceObject dso, String identifier) void delete(Context context, DSpaceObject dso, String identifier) throws AuthorizeException, SQLException, IdentifierException; + /** + * Get List of currently enabled IdentifierProviders + * @return List of enabled IdentifierProvider objects. + */ + List getProviders(); } diff --git a/dspace-api/src/test/java/org/dspace/ctask/general/CreateMissingIdentifiersIT.java b/dspace-api/src/test/java/org/dspace/ctask/general/CreateMissingIdentifiersIT.java index 4934404c3b3e..3b50258a5a23 100644 --- a/dspace-api/src/test/java/org/dspace/ctask/general/CreateMissingIdentifiersIT.java +++ b/dspace-api/src/test/java/org/dspace/ctask/general/CreateMissingIdentifiersIT.java @@ -60,14 +60,7 @@ public void testPerform() .build(); /* - * Curate with default Handle Provider - */ - curator.curate(context, item); - int status = curator.getStatus(TASK_NAME); - assertEquals("Curation should succeed", Curator.CURATE_SUCCESS, status); - - /* - * Now install an incompatible provider to make the task fail. + * First, install an incompatible provider to make the task fail. */ registerProvider(VersionedHandleIdentifierProviderWithCanonicalHandles.class); @@ -81,5 +74,13 @@ public void testPerform() unregisterProvider(VersionedHandleIdentifierProviderWithCanonicalHandles.class); // Re-register the default provider (for later tests which may depend on it) registerProvider(VersionedHandleIdentifierProvider.class); + + /* + * Now, verify curate with default Handle Provider works + * (and that our re-registration of the default provider above was successful) + */ + curator.curate(context, item); + int status = curator.getStatus(TASK_NAME); + assertEquals("Curation should succeed", Curator.CURATE_SUCCESS, status); } } From 97a5439e3aa23bfde4f9741282d48c10a873b387 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Thu, 9 Jan 2025 14:09:24 +0100 Subject: [PATCH 081/178] switch IT search core to MockSolrSearchCore https://github.com/DSpace/DSpace/issues/10188 (cherry picked from commit 6d781e8f83a912863af8d998d982e030db64a2ce) --- .../org/dspace/content/VersioningWithRelationshipsIT.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/content/VersioningWithRelationshipsIT.java b/dspace-api/src/test/java/org/dspace/content/VersioningWithRelationshipsIT.java index 3acc4ca146ee..9ff452f7895f 100644 --- a/dspace-api/src/test/java/org/dspace/content/VersioningWithRelationshipsIT.java +++ b/dspace-api/src/test/java/org/dspace/content/VersioningWithRelationshipsIT.java @@ -59,7 +59,7 @@ import org.dspace.content.virtual.VirtualMetadataConfiguration; import org.dspace.content.virtual.VirtualMetadataPopulator; import org.dspace.core.Constants; -import org.dspace.discovery.SolrSearchCore; +import org.dspace.discovery.MockSolrSearchCore; import org.dspace.kernel.ServiceManager; import org.dspace.services.factory.DSpaceServicesFactory; import org.dspace.versioning.Version; @@ -79,8 +79,9 @@ public class VersioningWithRelationshipsIT extends AbstractIntegrationTestWithDa ContentServiceFactory.getInstance().getInstallItemService(); private final ItemService itemService = ContentServiceFactory.getInstance().getItemService(); - private final SolrSearchCore solrSearchCore = - DSpaceServicesFactory.getInstance().getServiceManager().getServicesByType(SolrSearchCore.class).get(0); + private final MockSolrSearchCore solrSearchCore = + DSpaceServicesFactory.getInstance().getServiceManager().getServiceByName(null, MockSolrSearchCore.class); + protected Community community; protected Collection collection; protected EntityType publicationEntityType; From 49b3deef77055a18daba21c0bf2812bb2daa3265 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Thu, 9 Jan 2025 15:42:25 -0600 Subject: [PATCH 082/178] Refactor AbstractIntegrationTestWithDatabase to use Builders to create test EPersons. (cherry picked from commit 0b8b7be22b88a411898bcb975e238e7cd96f40ab) --- .../AbstractIntegrationTestWithDatabase.java | 48 ++++++++----------- .../AbstractWebClientIntegrationTest.java | 16 +++++++ 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/AbstractIntegrationTestWithDatabase.java b/dspace-api/src/test/java/org/dspace/AbstractIntegrationTestWithDatabase.java index e27fb19a68eb..d5bb6ab8a4c8 100644 --- a/dspace-api/src/test/java/org/dspace/AbstractIntegrationTestWithDatabase.java +++ b/dspace-api/src/test/java/org/dspace/AbstractIntegrationTestWithDatabase.java @@ -17,8 +17,8 @@ import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; import org.dspace.authority.AuthoritySearchService; import org.dspace.authority.MockAuthoritySolrServiceImpl; -import org.dspace.authorize.AuthorizeException; import org.dspace.builder.AbstractBuilder; +import org.dspace.builder.EPersonBuilder; import org.dspace.content.Community; import org.dspace.core.Context; import org.dspace.core.I18nUtil; @@ -114,19 +114,16 @@ public void setUp() throws Exception { EPersonService ePersonService = EPersonServiceFactory.getInstance().getEPersonService(); eperson = ePersonService.findByEmail(context, "test@email.com"); if (eperson == null) { - // This EPerson creation should only happen once (i.e. for first test run) - log.info("Creating initial EPerson (email=test@email.com) for Unit Tests"); - eperson = ePersonService.create(context); - eperson.setFirstName(context, "first"); - eperson.setLastName(context, "last"); - eperson.setEmail("test@email.com"); - eperson.setCanLogIn(true); - eperson.setLanguage(context, I18nUtil.getDefaultLocale().getLanguage()); - ePersonService.setPassword(eperson, password); - // actually save the eperson to unit testing DB - ePersonService.update(context, eperson); + // Create test EPerson for usage in all tests + log.info("Creating Test EPerson (email=test@email.com) for Integration Tests"); + eperson = EPersonBuilder.createEPerson(context) + .withNameInMetadata("first", "last") + .withEmail("test@email.com") + .withCanLogin(true) + .withLanguage(I18nUtil.getDefaultLocale().getLanguage()) + .withPassword(password) + .build(); } - // Set our global test EPerson as the current user in DSpace context.setCurrentUser(eperson); @@ -135,26 +132,23 @@ public void setUp() throws Exception { admin = ePersonService.findByEmail(context, "admin@email.com"); if (admin == null) { - // This EPerson creation should only happen once (i.e. for first test run) - log.info("Creating initial EPerson (email=admin@email.com) for Unit Tests"); - admin = ePersonService.create(context); - admin.setFirstName(context, "first (admin)"); - admin.setLastName(context, "last (admin)"); - admin.setEmail("admin@email.com"); - admin.setCanLogIn(true); - admin.setLanguage(context, I18nUtil.getDefaultLocale().getLanguage()); - ePersonService.setPassword(admin, password); - // actually save the eperson to unit testing DB - ePersonService.update(context, admin); + // Create test Administrator for usage in all tests + log.info("Creating Test Admin EPerson (email=admin@email.com) for Integration Tests"); + admin = EPersonBuilder.createEPerson(context) + .withNameInMetadata("first (admin)", "last (admin)") + .withEmail("admin@email.com") + .withCanLogin(true) + .withLanguage(I18nUtil.getDefaultLocale().getLanguage()) + .withPassword(password) + .build(); + + // Add Test Administrator to the ADMIN group in test database GroupService groupService = EPersonServiceFactory.getInstance().getGroupService(); Group adminGroup = groupService.findByName(context, Group.ADMIN); groupService.addMember(context, adminGroup, admin); } context.restoreAuthSystemState(); - } catch (AuthorizeException ex) { - log.error("Error creating initial eperson or default groups", ex); - fail("Error creating initial eperson or default groups in AbstractUnitTest init()"); } catch (SQLException ex) { log.error(ex.getMessage(), ex); fail("SQL Error on AbstractUnitTest init()"); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractWebClientIntegrationTest.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractWebClientIntegrationTest.java index df6afcb16e32..fadebff7de5b 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractWebClientIntegrationTest.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/test/AbstractWebClientIntegrationTest.java @@ -12,6 +12,7 @@ import org.dspace.app.rest.Application; import org.dspace.app.rest.utils.DSpaceConfigurationInitializer; import org.dspace.app.rest.utils.DSpaceKernelInitializer; +import org.junit.Before; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -64,6 +65,21 @@ public class AbstractWebClientIntegrationTest extends AbstractIntegrationTestWit @Autowired protected ApplicationContext applicationContext; + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + // Because AbstractWebClientIntegrationTest starts a new webserver, we need to ensure *everything* created by + // the "super.setUp()" script is committed to the test database. Otherwise, the new webserver may not see the + // created test data in the database. + // NOTE: This commit() does NOT occur in AbstractIntegrationTestDatabase because it will remove newly created + // Hibernate entities from the current Hibernate session. For most ITs we don't want that as it may result + // in "could not initialize proxy - no Session" errors when using those entities in other tests (or other tests + // would need to reload each test entity back into the Hibernate session) + context.commit(); + } + /** * Get client TestRestTemplate for making HTTP requests to test webserver * @return TestRestTemplate From 5973c67eff7180496ee761a32fa1f9636b02c93e Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Tue, 9 Jul 2024 16:58:00 +0200 Subject: [PATCH 083/178] put DOIs in dc.identifier.doi (cherry picked from commit 3d1bef9d0e3fe820da9705427c129b5cb49d66fb) --- dspace/config/spring/api/scopus-integration.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace/config/spring/api/scopus-integration.xml b/dspace/config/spring/api/scopus-integration.xml index 47c5a4fbb900..6f7556574d09 100644 --- a/dspace/config/spring/api/scopus-integration.xml +++ b/dspace/config/spring/api/scopus-integration.xml @@ -128,7 +128,7 @@ - + @@ -264,4 +264,4 @@ - \ No newline at end of file + From 5d0e9871e9ec4cbacda577f9f2ae8cefcd961e27 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Tue, 9 Jul 2024 17:37:07 +0200 Subject: [PATCH 084/178] fix broken test (cherry picked from commit 2eff833fab361d4ed3c7b602b681823911f989af) --- .../dspace/app/rest/ScopusImportMetadataSourceServiceIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java index 7f6cb53ce400..e81f662ad5a9 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java @@ -124,7 +124,7 @@ private ArrayList getRecords() { ArrayList records = new ArrayList<>(); //define first record List metadatums = new ArrayList(); - MetadatumDTO doi = createMetadatumDTO("dc", "identifier", null, "10.3934/mine.2023004"); + MetadatumDTO doi = createMetadatumDTO("dc", "identifier", "doi", "10.3934/mine.2023004"); MetadatumDTO title = createMetadatumDTO("dc","title", null, "Hardy potential versus lower order terms in Dirichlet problems: regularizing effects†"); MetadatumDTO type = createMetadatumDTO("dc", "type", null, "Journal"); @@ -171,7 +171,7 @@ private ArrayList getRecords() { //define second record List metadatums2 = new ArrayList(); - MetadatumDTO doi2 = createMetadatumDTO("dc", "identifier", null, "10.3934/mine.2023001"); + MetadatumDTO doi2 = createMetadatumDTO("dc", "identifier", "doi", "10.3934/mine.2023001"); MetadatumDTO title2 = createMetadatumDTO("dc","title", null, "Large deviations for a binary collision model: energy evaporation†"); MetadatumDTO date2 = createMetadatumDTO("dc", "date", "issued", "2023-01-01"); From 09a722a029ebc6e50835c360af8977d97cb1c033 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Thu, 17 Oct 2024 20:17:06 +0200 Subject: [PATCH 085/178] use dc.relation.hasversion for externally generated DOIs (cherry picked from commit 29067b6572b538650e4686e983edafcc355d87c5) --- dspace/config/spring/api/scopus-integration.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace/config/spring/api/scopus-integration.xml b/dspace/config/spring/api/scopus-integration.xml index 6f7556574d09..d3e8150c74b1 100644 --- a/dspace/config/spring/api/scopus-integration.xml +++ b/dspace/config/spring/api/scopus-integration.xml @@ -128,7 +128,7 @@ - + From d109eac2a0ad170b656e36028d84fc294ae6eb35 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Thu, 17 Oct 2024 20:19:04 +0200 Subject: [PATCH 086/178] use dc.relation.hasversion instead of dc.identifier.doi (cherry picked from commit d61dc8d911a4d226362d9a554226da5c304ad722) --- .../dspace/app/rest/ScopusImportMetadataSourceServiceIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java index e81f662ad5a9..f9164ddc5994 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java @@ -124,7 +124,7 @@ private ArrayList getRecords() { ArrayList records = new ArrayList<>(); //define first record List metadatums = new ArrayList(); - MetadatumDTO doi = createMetadatumDTO("dc", "identifier", "doi", "10.3934/mine.2023004"); + MetadatumDTO doi = createMetadatumDTO("dc", "relation", "hasversion", "10.3934/mine.2023004"); MetadatumDTO title = createMetadatumDTO("dc","title", null, "Hardy potential versus lower order terms in Dirichlet problems: regularizing effects†"); MetadatumDTO type = createMetadatumDTO("dc", "type", null, "Journal"); @@ -171,7 +171,7 @@ private ArrayList getRecords() { //define second record List metadatums2 = new ArrayList(); - MetadatumDTO doi2 = createMetadatumDTO("dc", "identifier", "doi", "10.3934/mine.2023001"); + MetadatumDTO doi2 = createMetadatumDTO("dc", "relation", "hasversion", "10.3934/mine.2023001"); MetadatumDTO title2 = createMetadatumDTO("dc","title", null, "Large deviations for a binary collision model: energy evaporation†"); MetadatumDTO date2 = createMetadatumDTO("dc", "date", "issued", "2023-01-01"); From 62596ad86313053f56fa316fba62ff1156cbeb37 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Tue, 8 Oct 2024 13:42:07 +0200 Subject: [PATCH 087/178] option to exclude the submitter being indexed to solr in archived items https://github.com/DSpace/DSpace/issues/9660 --- .../indexobject/ItemIndexFactoryImpl.java | 8 +- .../org/dspace/discovery/DiscoveryIT.java | 109 ++++++++++++++++++ dspace/config/modules/discovery.cfg | 7 +- 3 files changed, 120 insertions(+), 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/ItemIndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/ItemIndexFactoryImpl.java index 7cdb8b93d80e..5fb2f779783c 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/ItemIndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/ItemIndexFactoryImpl.java @@ -154,9 +154,11 @@ public SolrInputDocument buildDocument(Context context, IndexableItem indexableI doc.addField("latestVersion", isLatestVersion(context, item)); EPerson submitter = item.getSubmitter(); - if (submitter != null) { - addFacetIndex(doc, "submitter", submitter.getID().toString(), - submitter.getFullName()); + if (submitter != null && !(DSpaceServicesFactory.getInstance().getConfigurationService().getBooleanProperty( + "discovery.index.item.submitter.enabled", false))) { + doc.addField("submitter_authority", submitter.getID().toString()); + } else if (submitter != null) { + addFacetIndex(doc, "submitter", submitter.getID().toString(), submitter.getFullName()); } // Add the item metadata diff --git a/dspace-api/src/test/java/org/dspace/discovery/DiscoveryIT.java b/dspace-api/src/test/java/org/dspace/discovery/DiscoveryIT.java index 55be531418ae..159f7a0ac5ae 100644 --- a/dspace-api/src/test/java/org/dspace/discovery/DiscoveryIT.java +++ b/dspace-api/src/test/java/org/dspace/discovery/DiscoveryIT.java @@ -8,6 +8,10 @@ package org.dspace.discovery; import static org.dspace.discovery.SolrServiceWorkspaceWorkflowRestrictionPlugin.DISCOVER_WORKSPACE_CONFIGURATION_NAME; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -21,6 +25,10 @@ import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; @@ -99,6 +107,9 @@ public class DiscoveryIT extends AbstractIntegrationTestWithDatabase { MetadataAuthorityService metadataAuthorityService = ContentAuthorityServiceFactory.getInstance() .getMetadataAuthorityService(); + MockSolrSearchCore solrSearchCore = DSpaceServicesFactory.getInstance().getServiceManager() + .getServiceByName(null, MockSolrSearchCore.class); + @Override @Before public void setUp() throws Exception { @@ -796,6 +807,104 @@ public void searchWithDefaultSortServiceTest() throws SearchServiceException { } } + /** + * Test designed to check if the submitter is not indexed in all in solr documents for items + * and the submitter authority is still indexed + * @throws SearchServiceException + */ + @Test + public void searchWithNoSubmitterTest() throws SearchServiceException { + + configurationService.setProperty("discovery.index.item.submitter.enabled", false); + DiscoveryConfiguration defaultConf = SearchUtils.getDiscoveryConfiguration(context, "default", null); + + // Populate the testing objects: create items in eperson's workspace and perform search in it + int numberItems = 10; + context.turnOffAuthorisationSystem(); + EPerson submitter = null; + try { + submitter = EPersonBuilder.createEPerson(context).withEmail("submitter@example.org") + .withNameInMetadata("Peter", "Funny").build(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + context.setCurrentUser(submitter); + Community community = CommunityBuilder.createCommunity(context).build(); + Collection collection = CollectionBuilder.createCollection(context, community).build(); + for (int i = 0; i < numberItems; i++) { + ItemBuilder.createItem(context, collection) + .withTitle("item " + i) + .build(); + } + context.restoreAuthSystemState(); + + // Build query with default parameters (except for workspaceConf) + QueryResponse result = null; + try { + result = solrSearchCore.getSolr().query(new SolrQuery(String.format( + "search.resourcetype:\"Item\""))); + } catch (SolrServerException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + assertEquals(result.getResults().size(), numberItems); + for (SolrDocument doc : result.getResults()) { + assertThat(doc.getFieldNames(), + not(hasItems("submitter_keyword", "submitter_ac", "submitter_acid", "submitter_filter"))); + assertThat(doc.getFieldNames(), hasItem("submitter_authority")); + } + } + + /** + * Test designed to check if the submitter is indexed in all in solr documents for items + * @throws SearchServiceException + */ + @Test + public void searchWithSubmitterTest() throws SearchServiceException { + + configurationService.setProperty("discovery.index.item.submitter.enabled", true); + DiscoveryConfiguration defaultConf = SearchUtils.getDiscoveryConfiguration(context, "default", null); + + // Populate the testing objects: create items in eperson's workspace and perform search in it + int numberItems = 10; + context.turnOffAuthorisationSystem(); + EPerson submitter = null; + try { + submitter = EPersonBuilder.createEPerson(context).withEmail("submitter@example.org") + .withNameInMetadata("Peter", "Funny").build(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + context.setCurrentUser(submitter); + Community community = CommunityBuilder.createCommunity(context).build(); + Collection collection = CollectionBuilder.createCollection(context, community).build(); + for (int i = 0; i < numberItems; i++) { + ItemBuilder.createItem(context, collection) + .withTitle("item " + i) + .build(); + } + context.restoreAuthSystemState(); + + // Build query with default parameters (except for workspaceConf) + QueryResponse result = null; + try { + result = solrSearchCore.getSolr().query(new SolrQuery(String.format( + "search.resourcetype:\"Item\""))); + } catch (SolrServerException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + assertEquals(result.getResults().size(), numberItems); + for (SolrDocument doc : result.getResults()) { + for (String fieldname : doc.getFieldNames()) { + assertThat(doc.getFieldNames(), hasItems("submitter_keyword","submitter_ac", "submitter_filter", + "submitter_authority")); + } + } + } + private void assertSearchQuery(String resourceType, int size) throws SearchServiceException { assertSearchQuery(resourceType, size, size, 0, -1); } diff --git a/dspace/config/modules/discovery.cfg b/dspace/config/modules/discovery.cfg index 72088ddc49fa..cd8e8636c2e3 100644 --- a/dspace/config/modules/discovery.cfg +++ b/dspace/config/modules/discovery.cfg @@ -24,6 +24,11 @@ discovery.search.server = ${solr.server}/${solr.multicorePrefix}search # discovery.index.ignore-authority = false discovery.index.projection=dc.title,dc.contributor.*,dc.date.issued +# Restricts the indexing of the submitter for archived items +# By default the submitter information from the corresponding eperson is not indexed. +# If you set this value to true, than the submitter information is indexed and you will need to reindex search core +# discovery.index.item.submitter.enabled = false + # Allow auto-reindexing. # If any database migrations are applied to your database (via Flyway), then a # reindex flag is always written to '[dspace]/solr/search/conf/reindex.flag'. @@ -48,4 +53,4 @@ discovery.facet.namedtype.workflow.pooled = 004workflow\n|||\nWaiting for Contro # Set the number of retry of a query when stale objects are found. # Set to -1 if stale objects should be ignored. Set to 0 if you want to avoid extra query but take the chance to cleanup # the index each time that stale objects are found. Default 3 -discovery.removestale.attempts = 3 \ No newline at end of file +discovery.removestale.attempts = 3 From e840ca27315ce8fb2772e8bcdf6c34c042a6cc99 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Mon, 28 Oct 2024 14:55:30 -0500 Subject: [PATCH 088/178] Create dependabot.yml --- .github/dependabot.yml | 115 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..6f4f3ae601a0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,115 @@ +#------------------- +# DSpace's dependabot rules. Enables maven updates for all dependencies on a weekly basis +# for main and any maintenance branches. Security updates only apply to main. +#------------------- +version: 2 +updates: + - package-ecosystem: "maven" + directory: "/" + schedule: + interval: "weekly" + # Allow up to 10 open PRs for dependencies + open-pull-requests-limit: 10 + # Group together some upgrades in a single PR + groups: + # Group together all Build Tools in a single PR + build-tools: + applies-to: version-updates + patterns: + - "org.apache.maven.plugins:*" + - "*:*-maven-plugin" + - "*:maven-*-plugin" + - "com.github.spotbugs:spotbugs" + - "com.google.code.findbugs:*" + - "com.google.errorprone:*" + - "com.puppycrawl.tools:checkstyle" + - "org.sonatype.plugins:*" + update-types: + - "minor" + - "patch" + test-tools: + applies-to: version-updates + patterns: + - "junit:*" + - "com.github.stefanbirker:system-rules" + - "com.h2database:*" + - "io.findify:s3mock*" + - "io.netty:*" + - "org.hamcrest:*" + - "org.mock-server:*" + - "org.mockito:*" + update-types: + - "minor" + - "patch" + # Group together all Apache Commons deps in a single PR + apache-commons: + applies-to: version-updates + patterns: + - "org.apache.commons:*" + - "commons-*:commons-*" + update-types: + - "minor" + - "patch" + # Group together all fasterxml deps in a single PR + fasterxml: + applies-to: version-updates + patterns: + - "com.fasterxml:*" + - "com.fasterxml.*:*" + update-types: + - "minor" + - "patch" + # Group together all Hibernate deps in a single PR + hibernate: + applies-to: version-updates + patterns: + - "org.hibernate.*:*" + update-types: + - "minor" + - "patch" + # Group together all Jakarta deps in a single PR + jakarta: + applies-to: version-updates + patterns: + - "jakarta.*:*" + - "org.eclipse.angus:jakarta.mail" + - "org.glassfish.jaxb:jaxb-runtime" + update-types: + - "minor" + - "patch" + # Group together all Google deps in a single PR + google-apis: + applies-to: version-updates + patterns: + - "com.google.apis:*" + - "com.google.api-client:*" + - "com.google.http-client:*" + - "com.google.oauth-client:*" + update-types: + - "minor" + - "patch" + # Group together all Spring deps in a single PR + spring: + applies-to: version-updates + patterns: + - "org.springframework:*" + - "org.springframework.*:*" + update-types: + - "minor" + - "patch" + # Group together all WebJARs deps in a single PR + webjars: + applies-to: version-updates + patterns: + - "org.webjars:*" + - "org.webjars.*:*" + update-types: + - "minor" + - "patch" + ignore: + # Don't try to auto-update any DSpace dependencies + - dependency-name: "org.dspace:*" + - dependency-name: "org.dspace.*:*" + # Ignore all major version updates for all dependencies. We'll only automate minor/patch updates. + - dependency-name: "*" + update-types: ["version-update:semver-major"] From fe28962e9672c01477059bcc2d2c8787a781dadf Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 29 Oct 2024 15:01:41 -0500 Subject: [PATCH 089/178] Exclude spring from build-tools group in dependabot.yml --- .github/dependabot.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6f4f3ae601a0..b6412b25b660 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -24,6 +24,9 @@ updates: - "com.google.errorprone:*" - "com.puppycrawl.tools:checkstyle" - "org.sonatype.plugins:*" + exclude-patterns: + # Exclude anything from Spring, as that is in a separate group + - "org.springframework.*:*" update-types: - "minor" - "patch" From 0b46a0963c5cfe8189fe3b8764eb6450ec41664a Mon Sep 17 00:00:00 2001 From: Jens Vannerum Date: Tue, 2 Apr 2024 11:15:49 +0200 Subject: [PATCH 090/178] 113811: cli logs should be written to a different file (cherry picked from commit d30468a09fbbd288df6cf2556e1a630f040801d1) --- dspace/bin/dspace | 2 + dspace/config/log4j2-cli.xml | 105 +++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 dspace/config/log4j2-cli.xml diff --git a/dspace/bin/dspace b/dspace/bin/dspace index 7d5d678d0f6e..24644aae9112 100644 --- a/dspace/bin/dspace +++ b/dspace/bin/dspace @@ -39,6 +39,8 @@ if [ "$JAVA_OPTS" = "" ]; then JAVA_OPTS="-Xmx256m -Dfile.encoding=UTF-8" fi +export JAVA_OPTS="$JAVA_OPTS -Dlog4j2.configurationFile=$DSPACEDIR/config/log4j2-cli.xml" + # Now invoke Java java $JAVA_OPTS \ -classpath $FULLPATH \ diff --git a/dspace/config/log4j2-cli.xml b/dspace/config/log4j2-cli.xml new file mode 100644 index 000000000000..24d36fd12efb --- /dev/null +++ b/dspace/config/log4j2-cli.xml @@ -0,0 +1,105 @@ + + + + + + + ${log4j:configParentLocation}/../log + + + INFO + + + INFO + + + + + + + + + yyyy-MM-dd + + + + + + + + + yyyy-MM-dd + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 04ec6381cba0472f4fdbc0918588180393a89c5d Mon Sep 17 00:00:00 2001 From: Kevin Van de Velde Date: Fri, 6 Sep 2024 10:15:08 +0200 Subject: [PATCH 091/178] Modifying it so that the cli file content is stored in a file using the date (cherry picked from commit 529c3a77c15f401fe623b2c9d04a438754a1d72d) --- dspace/config/log4j2-cli.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dspace/config/log4j2-cli.xml b/dspace/config/log4j2-cli.xml index 24d36fd12efb..73acab877f1f 100644 --- a/dspace/config/log4j2-cli.xml +++ b/dspace/config/log4j2-cli.xml @@ -25,8 +25,6 @@ From 1a6088388fa4bc0d01ecdea0b1fee7108a27bce5 Mon Sep 17 00:00:00 2001 From: Jens Vannerum Date: Wed, 4 Dec 2024 11:08:37 +0100 Subject: [PATCH 092/178] apply fix to windows env and remove duplicate logging for checksum checker (cherry picked from commit 9f39a3d6a53f9a815509d6a47abd176b07a6a86c) --- dspace/bin/dspace.bat | 3 +++ dspace/config/log4j2.xml | 27 --------------------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/dspace/bin/dspace.bat b/dspace/bin/dspace.bat index 2288127c447a..768b1061762d 100644 --- a/dspace/bin/dspace.bat +++ b/dspace/bin/dspace.bat @@ -38,6 +38,9 @@ REM If JAVA_OPTS specified, use those options REM Otherwise, default Java to using 256MB of memory if "%JAVA_OPTS%"=="" set "JAVA_OPTS=-Xmx256m -Dfile.encoding=UTF-8" +REM Add log4j2 configuration to JAVA_OPTS +set "JAVA_OPTS=%JAVA_OPTS% -Dlog4j2.configurationFile=%cd%\config\log4j2-cli.xml" + REM Execute Java java %JAVA_OPTS% -classpath "%DSPACE_CLASSPATH%" org.dspace.app.launcher.ScriptLauncher %* diff --git a/dspace/config/log4j2.xml b/dspace/config/log4j2.xml index 6e9a43e4f0fe..4f212d128dc6 100644 --- a/dspace/config/log4j2.xml +++ b/dspace/config/log4j2.xml @@ -44,26 +44,6 @@ --> - - - - - - yyyy-MM-dd - - - @@ -75,13 +55,6 @@ - - - - - From 1492ed7f61b5685e0e8e0f4365460887465bef88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 21:01:16 +0000 Subject: [PATCH 093/178] Bump the build-tools group with 21 updates Bumps the build-tools group with 21 updates: | Package | From | To | | --- | --- | --- | | [com.google.errorprone:error_prone_core](https://github.com/google/error-prone) | `2.10.0` | `2.36.0` | | [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) | `8.38` | `8.45.1` | | [com.github.spotbugs:spotbugs](https://github.com/spotbugs/spotbugs) | `4.8.2` | `4.8.6` | | [org.apache.maven.plugins:maven-enforcer-plugin](https://github.com/apache/maven-enforcer) | `3.0.0-M3` | `3.5.0` | | [org.apache.maven.plugins:maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) | `3.8.1` | `3.13.0` | | [org.apache.maven.plugins:maven-jar-plugin](https://github.com/apache/maven-jar-plugin) | `3.2.0` | `3.4.2` | | [org.apache.maven.plugins:maven-war-plugin](https://github.com/apache/maven-war-plugin) | `3.2.3` | `3.4.0` | | [org.apache.maven.plugins:maven-checkstyle-plugin](https://github.com/apache/maven-checkstyle-plugin) | `3.3.1` | `3.6.0` | | [com.github.spotbugs:spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) | `4.8.2.0` | `4.8.6.6` | | [org.apache.maven.plugins:maven-clean-plugin](https://github.com/apache/maven-clean-plugin) | `3.1.0` | `3.4.0` | | [org.apache.maven.plugins:maven-assembly-plugin](https://github.com/apache/maven-assembly-plugin) | `3.6.0` | `3.7.1` | | [org.apache.maven.plugins:maven-dependency-plugin](https://github.com/apache/maven-dependency-plugin) | `3.1.2` | `3.8.1` | | [org.apache.maven.plugins:maven-resources-plugin](https://github.com/apache/maven-resources-plugin) | `3.1.0` | `3.3.1` | | org.sonatype.plugins:nexus-staging-maven-plugin | `1.6.8` | `1.7.0` | | [org.apache.maven.plugins:maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) | `3.2.0` | `3.11.2` | | [org.apache.maven.plugins:maven-source-plugin](https://github.com/apache/maven-source-plugin) | `3.2.1` | `3.3.1` | | [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) | `0.8.5` | `0.8.12` | | [org.codehaus.mojo:xml-maven-plugin](https://github.com/mojohaus/xml-maven-plugin) | `1.0.2` | `1.1.0` | | [org.codehaus.mojo:license-maven-plugin](https://github.com/mojohaus/license-maven-plugin) | `2.0.0` | `2.5.0` | | [org.codehaus.mojo:build-helper-maven-plugin](https://github.com/mojohaus/build-helper-maven-plugin) | `3.4.0` | `3.6.0` | | [org.codehaus.mojo:buildnumber-maven-plugin](https://github.com/mojohaus/buildnumber-maven-plugin) | `3.2.0` | `3.2.1` | Updates `com.google.errorprone:error_prone_core` from 2.10.0 to 2.36.0 - [Release notes](https://github.com/google/error-prone/releases) - [Commits](https://github.com/google/error-prone/compare/v2.10.0...v2.36.0) Updates `com.puppycrawl.tools:checkstyle` from 8.38 to 8.45.1 - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-8.38...checkstyle-8.45.1) Updates `com.github.spotbugs:spotbugs` from 4.8.2 to 4.8.6 - [Release notes](https://github.com/spotbugs/spotbugs/releases) - [Changelog](https://github.com/spotbugs/spotbugs/blob/master/CHANGELOG.md) - [Commits](https://github.com/spotbugs/spotbugs/compare/4.8.2...4.8.6) Updates `org.apache.maven.plugins:maven-enforcer-plugin` from 3.0.0-M3 to 3.5.0 - [Release notes](https://github.com/apache/maven-enforcer/releases) - [Commits](https://github.com/apache/maven-enforcer/compare/enforcer-3.0.0-M3...enforcer-3.5.0) Updates `org.apache.maven.plugins:maven-compiler-plugin` from 3.8.1 to 3.13.0 - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.8.1...maven-compiler-plugin-3.13.0) Updates `org.apache.maven.plugins:maven-jar-plugin` from 3.2.0 to 3.4.2 - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.0...maven-jar-plugin-3.4.2) Updates `org.apache.maven.plugins:maven-war-plugin` from 3.2.3 to 3.4.0 - [Commits](https://github.com/apache/maven-war-plugin/compare/maven-war-plugin-3.2.3...maven-war-plugin-3.4.0) Updates `org.apache.maven.plugins:maven-checkstyle-plugin` from 3.3.1 to 3.6.0 - [Commits](https://github.com/apache/maven-checkstyle-plugin/compare/maven-checkstyle-plugin-3.3.1...maven-checkstyle-plugin-3.6.0) Updates `com.github.spotbugs:spotbugs-maven-plugin` from 4.8.2.0 to 4.8.6.6 - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.8.2.0...spotbugs-maven-plugin-4.8.6.6) Updates `org.apache.maven.plugins:maven-clean-plugin` from 3.1.0 to 3.4.0 - [Release notes](https://github.com/apache/maven-clean-plugin/releases) - [Commits](https://github.com/apache/maven-clean-plugin/compare/maven-clean-plugin-3.1.0...maven-clean-plugin-3.4.0) Updates `org.apache.maven.plugins:maven-assembly-plugin` from 3.6.0 to 3.7.1 - [Release notes](https://github.com/apache/maven-assembly-plugin/releases) - [Commits](https://github.com/apache/maven-assembly-plugin/compare/maven-assembly-plugin-3.6.0...maven-assembly-plugin-3.7.1) Updates `org.apache.maven.plugins:maven-dependency-plugin` from 3.1.2 to 3.8.1 - [Release notes](https://github.com/apache/maven-dependency-plugin/releases) - [Commits](https://github.com/apache/maven-dependency-plugin/compare/maven-dependency-plugin-3.1.2...maven-dependency-plugin-3.8.1) Updates `org.apache.maven.plugins:maven-resources-plugin` from 3.1.0 to 3.3.1 - [Release notes](https://github.com/apache/maven-resources-plugin/releases) - [Commits](https://github.com/apache/maven-resources-plugin/compare/maven-resources-plugin-3.1.0...maven-resources-plugin-3.3.1) Updates `org.sonatype.plugins:nexus-staging-maven-plugin` from 1.6.8 to 1.7.0 Updates `org.apache.maven.plugins:maven-javadoc-plugin` from 3.2.0 to 3.11.2 - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.2.0...maven-javadoc-plugin-3.11.2) Updates `org.apache.maven.plugins:maven-source-plugin` from 3.2.1 to 3.3.1 - [Release notes](https://github.com/apache/maven-source-plugin/releases) - [Commits](https://github.com/apache/maven-source-plugin/compare/maven-source-plugin-3.2.1...maven-source-plugin-3.3.1) Updates `org.jacoco:jacoco-maven-plugin` from 0.8.5 to 0.8.12 - [Release notes](https://github.com/jacoco/jacoco/releases) - [Commits](https://github.com/jacoco/jacoco/compare/v0.8.5...v0.8.12) Updates `org.codehaus.mojo:xml-maven-plugin` from 1.0.2 to 1.1.0 - [Release notes](https://github.com/mojohaus/xml-maven-plugin/releases) - [Commits](https://github.com/mojohaus/xml-maven-plugin/compare/xml-maven-plugin-1.0.2...1.1.0) Updates `org.codehaus.mojo:license-maven-plugin` from 2.0.0 to 2.5.0 - [Release notes](https://github.com/mojohaus/license-maven-plugin/releases) - [Commits](https://github.com/mojohaus/license-maven-plugin/compare/license-maven-plugin-2.0.0...2.5.0) Updates `org.codehaus.mojo:build-helper-maven-plugin` from 3.4.0 to 3.6.0 - [Release notes](https://github.com/mojohaus/build-helper-maven-plugin/releases) - [Commits](https://github.com/mojohaus/build-helper-maven-plugin/compare/3.4.0...3.6.0) Updates `org.codehaus.mojo:buildnumber-maven-plugin` from 3.2.0 to 3.2.1 - [Release notes](https://github.com/mojohaus/buildnumber-maven-plugin/releases) - [Commits](https://github.com/mojohaus/buildnumber-maven-plugin/compare/3.2.0...3.2.1) --- updated-dependencies: - dependency-name: com.google.errorprone:error_prone_core dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: com.github.spotbugs:spotbugs dependency-type: direct:production update-type: version-update:semver-patch dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-enforcer-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-war-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-checkstyle-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-clean-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-assembly-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-dependency-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-resources-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.sonatype.plugins:nexus-staging-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.apache.maven.plugins:maven-source-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.jacoco:jacoco-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch dependency-group: build-tools - dependency-name: org.codehaus.mojo:xml-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.codehaus.mojo:license-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.codehaus.mojo:build-helper-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor dependency-group: build-tools - dependency-name: org.codehaus.mojo:buildnumber-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch dependency-group: build-tools ... Signed-off-by: dependabot[bot] --- dspace-api/pom.xml | 4 ++-- pom.xml | 38 +++++++++++++++++++------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index c42895ab4c49..cf81408a2659 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -102,7 +102,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.4.0 + 3.6.0 validate @@ -116,7 +116,7 @@ org.codehaus.mojo buildnumber-maven-plugin - 3.2.0 + 3.2.1 UNKNOWN_REVISION diff --git a/pom.xml b/pom.xml index 50faf1f7372e..f496ed7e879d 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 8.11.3 3.10.8 - 2.10.0 + 2.36.0 2.16.0 2.16.0 @@ -89,7 +89,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + 3.5.0 enforce-java @@ -140,7 +140,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.13.0 11 @@ -173,7 +173,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.0 + 3.4.2 @@ -187,7 +187,7 @@ org.apache.maven.plugins maven-war-plugin - 3.2.3 + 3.4.0 false @@ -257,7 +257,7 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.3.1 + 3.6.0 verify-style @@ -287,14 +287,14 @@ com.puppycrawl.tools checkstyle - 8.38 + 8.45.1 com.github.spotbugs spotbugs-maven-plugin - 4.8.2.0 + 4.8.6.6 Max Low @@ -304,7 +304,7 @@ com.github.spotbugs spotbugs - 4.8.2 + 4.8.6 @@ -320,7 +320,7 @@ maven-clean-plugin - 3.1.0 + 3.4.0 @@ -334,17 +334,17 @@ maven-assembly-plugin - 3.6.0 + 3.7.1 org.apache.maven.plugins maven-dependency-plugin - 3.1.2 + 3.8.1 org.apache.maven.plugins maven-resources-plugin - 3.1.0 + 3.3.1 @@ -356,13 +356,13 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.8 + 1.7.0 org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.11.2 false @@ -372,7 +372,7 @@ org.apache.maven.plugins maven-source-plugin - 3.2.1 + 3.3.1 @@ -391,7 +391,7 @@ org.jacoco jacoco-maven-plugin - 0.8.5 + 0.8.12 @@ -471,7 +471,7 @@ org.codehaus.mojo xml-maven-plugin - 1.0.2 + 1.1.0 validate-ALL-xml-and-xsl @@ -679,7 +679,7 @@ org.codehaus.mojo license-maven-plugin - 2.0.0 + 2.5.0 false From d7b0e26179f9b0812afb722fd13efa00eae4fa57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 21:02:16 +0000 Subject: [PATCH 094/178] Bump the apache-commons group with 11 updates Bumps the apache-commons group with 11 updates: | Package | From | To | | --- | --- | --- | | commons-beanutils:commons-beanutils | `1.9.4` | `1.10.0` | | commons-cli:commons-cli | `1.6.0` | `1.9.0` | | [commons-codec:commons-codec](https://github.com/apache/commons-codec) | `1.16.0` | `1.17.2` | | org.apache.commons:commons-configuration2 | `2.10.1` | `2.11.0` | | org.apache.commons:commons-dbcp2 | `2.11.0` | `2.13.0` | | commons-io:commons-io | `2.15.1` | `2.18.0` | | org.apache.commons:commons-lang3 | `3.14.0` | `3.17.0` | | commons-logging:commons-logging | `1.3.0` | `1.3.4` | | org.apache.commons:commons-compress | `1.26.0` | `1.27.1` | | org.apache.commons:commons-text | `1.10.0` | `1.13.0` | | commons-validator:commons-validator | `1.7` | `1.9.0` | Updates `commons-beanutils:commons-beanutils` from 1.9.4 to 1.10.0 Updates `commons-cli:commons-cli` from 1.6.0 to 1.9.0 Updates `commons-codec:commons-codec` from 1.16.0 to 1.17.2 - [Changelog](https://github.com/apache/commons-codec/blob/master/RELEASE-NOTES.txt) - [Commits](https://github.com/apache/commons-codec/compare/rel/commons-codec-1.16.0...rel/commons-codec-1.17.2) Updates `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 Updates `org.apache.commons:commons-dbcp2` from 2.11.0 to 2.13.0 Updates `commons-io:commons-io` from 2.15.1 to 2.18.0 Updates `org.apache.commons:commons-lang3` from 3.14.0 to 3.17.0 Updates `commons-logging:commons-logging` from 1.3.0 to 1.3.4 Updates `org.apache.commons:commons-compress` from 1.26.0 to 1.27.1 Updates `org.apache.commons:commons-text` from 1.10.0 to 1.13.0 Updates `commons-validator:commons-validator` from 1.7 to 1.9.0 --- updated-dependencies: - dependency-name: commons-beanutils:commons-beanutils dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: commons-cli:commons-cli dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: commons-codec:commons-codec dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: org.apache.commons:commons-configuration2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: org.apache.commons:commons-dbcp2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: commons-io:commons-io dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: org.apache.commons:commons-lang3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: commons-logging:commons-logging dependency-type: direct:production update-type: version-update:semver-patch dependency-group: apache-commons - dependency-name: org.apache.commons:commons-compress dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: org.apache.commons:commons-text dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons - dependency-name: commons-validator:commons-validator dependency-type: direct:production update-type: version-update:semver-minor dependency-group: apache-commons ... Signed-off-by: dependabot[bot] --- pom.xml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 50faf1f7372e..242c03d1b178 100644 --- a/pom.xml +++ b/pom.xml @@ -1463,17 +1463,17 @@ commons-beanutils commons-beanutils - 1.9.4 + 1.10.0 commons-cli commons-cli - 1.6.0 + 1.9.0 commons-codec commons-codec - 1.16.0 + 1.17.2 org.apache.commons @@ -1483,12 +1483,12 @@ org.apache.commons commons-configuration2 - 2.10.1 + 2.11.0 org.apache.commons commons-dbcp2 - 2.11.0 + 2.13.0 commons-fileupload @@ -1498,24 +1498,24 @@ commons-io commons-io - 2.15.1 + 2.18.0 org.apache.commons commons-lang3 - 3.14.0 + 3.17.0 commons-logging commons-logging - 1.3.0 + 1.3.4 org.apache.commons commons-compress - 1.26.0 + 1.27.1 org.apache.commons @@ -1525,12 +1525,12 @@ org.apache.commons commons-text - 1.10.0 + 1.13.0 commons-validator commons-validator - 1.7 + 1.9.0 joda-time From 4a20a4c37fa43c33eb6ba11ea49422e15e786212 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 21:02:28 +0000 Subject: [PATCH 095/178] Bump the fasterxml group with 4 updates Bumps the fasterxml group with 4 updates: [com.fasterxml:classmate](https://github.com/FasterXML/java-classmate), [com.fasterxml.jackson.core:jackson-annotations](https://github.com/FasterXML/jackson), [com.fasterxml.jackson.core:jackson-core](https://github.com/FasterXML/jackson-core) and [com.fasterxml.jackson.core:jackson-databind](https://github.com/FasterXML/jackson). Updates `com.fasterxml:classmate` from 1.6.0 to 1.7.0 - [Commits](https://github.com/FasterXML/java-classmate/compare/classmate-1.6.0...classmate-1.7.0) Updates `com.fasterxml.jackson.core:jackson-annotations` from 2.16.0 to 2.18.2 - [Commits](https://github.com/FasterXML/jackson/commits) Updates `com.fasterxml.jackson.core:jackson-core` from 2.16.0 to 2.18.2 - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.16.0...jackson-core-2.18.2) Updates `com.fasterxml.jackson.core:jackson-core` from 2.16.0 to 2.18.2 - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.16.0...jackson-core-2.18.2) Updates `com.fasterxml.jackson.core:jackson-databind` from 2.16.0 to 2.18.2 - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml:classmate dependency-type: direct:production update-type: version-update:semver-minor dependency-group: fasterxml - dependency-name: com.fasterxml.jackson.core:jackson-annotations dependency-type: direct:production update-type: version-update:semver-minor dependency-group: fasterxml - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-type: direct:production update-type: version-update:semver-minor dependency-group: fasterxml - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-type: direct:production update-type: version-update:semver-minor dependency-group: fasterxml - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-minor dependency-group: fasterxml ... Signed-off-by: dependabot[bot] --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 50faf1f7372e..a7a87a7938ff 100644 --- a/pom.xml +++ b/pom.xml @@ -30,8 +30,8 @@ 3.10.8 2.10.0 - 2.16.0 - 2.16.0 + 2.18.2 + 2.18.2 1.3.2 2.3.1 2.3.9 @@ -1742,7 +1742,7 @@ com.fasterxml classmate - 1.6.0 + 1.7.0 com.fasterxml.jackson.core From bec1ee84e51e6a22766c58348791292fe9d6f36d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 21:03:40 +0000 Subject: [PATCH 096/178] Bump org.webjars.npm:json-editor__json-editor in the webjars group Bumps the webjars group with 1 update: [org.webjars.npm:json-editor__json-editor](https://github.com/json-editor/json-editor). Updates `org.webjars.npm:json-editor__json-editor` from 2.6.1 to 2.15.1 - [Changelog](https://github.com/json-editor/json-editor/blob/master/CHANGELOG.md) - [Commits](https://github.com/json-editor/json-editor/compare/2.6.1...2.15.1) --- updated-dependencies: - dependency-name: org.webjars.npm:json-editor__json-editor dependency-type: direct:production update-type: version-update:semver-minor dependency-group: webjars ... Signed-off-by: dependabot[bot] --- dspace-server-webapp/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-server-webapp/pom.xml b/dspace-server-webapp/pom.xml index 35fa473fc170..2715338ed0fa 100644 --- a/dspace-server-webapp/pom.xml +++ b/dspace-server-webapp/pom.xml @@ -345,7 +345,7 @@ org.webjars.npm json-editor__json-editor - 2.6.1 + 2.15.1 From d9945b27b94f372979ac3268d2c85887edb00fa3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 21:03:55 +0000 Subject: [PATCH 098/178] Bump jersey.version from 2.39.1 to 2.46 Bumps `jersey.version` from 2.39.1 to 2.46. Updates `org.glassfish.jersey.core:jersey-client` from 2.39.1 to 2.46 Updates `org.glassfish.jersey.inject:jersey-hk2` from 2.39.1 to 2.46 Updates `org.glassfish.jersey.core:jersey-server` from 2.39.1 to 2.46 Updates `org.glassfish.jersey.containers:jersey-container-servlet` from 2.39.1 to 2.46 Updates `org.glassfish.jersey.media:jersey-media-json-jackson` from 2.39.1 to 2.46 Updates `org.glassfish.jersey.media:jersey-media-jaxb` from 2.39.1 to 2.46 Updates `org.glassfish.jersey.ext:jersey-spring5` from 2.39.1 to 2.46 --- updated-dependencies: - dependency-name: org.glassfish.jersey.core:jersey-client dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.inject:jersey-hk2 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.core:jersey-server dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.containers:jersey-container-servlet dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.media:jersey-media-json-jackson dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.media:jersey-media-jaxb dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.ext:jersey-spring5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 50faf1f7372e..1e83cbbb9bb9 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ https://jena.apache.org/documentation/migrate_jena2_jena3.html --> 2.13.0 - 2.39.1 + 2.46 UTF-8 From 9d17009cf390ea403e439bc5287bf6c118c88836 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 21:04:17 +0000 Subject: [PATCH 099/178] Bump jetty.version from 9.4.54.v20240208 to 9.4.57.v20241219 Bumps `jetty.version` from 9.4.54.v20240208 to 9.4.57.v20241219. Updates `org.eclipse.jetty:jetty-server` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-alpn-java-server` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-deploy` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-http` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-io` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-servlet` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-servlets` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-util` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-webapp` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty:jetty-xml` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty.http2:http2-common` from 9.4.54.v20240208 to 9.4.57.v20241219 Updates `org.eclipse.jetty.http2:http2-server` from 9.4.54.v20240208 to 9.4.57.v20241219 --- updated-dependencies: - dependency-name: org.eclipse.jetty:jetty-server dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-alpn-java-server dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-deploy dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-http dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-io dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-servlet dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-servlets dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-util dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-webapp dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-xml dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty.http2:http2-common dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty.http2:http2-server dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 50faf1f7372e..6575c7e5277f 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ 2.3.9 1.1.1 - 9.4.54.v20240208 + 9.4.57.v20241219 2.23.1 2.0.31 1.19.0 From a01983c23027642163f7a8f88525e5638784f428 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 29 Oct 2024 17:02:14 -0500 Subject: [PATCH 100/178] Fix checkstyle.xml syntax for bump to 8.45.1 --- checkstyle.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkstyle.xml b/checkstyle.xml index e0fa808d83cb..a33fc4831950 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -92,7 +92,7 @@ For more information on CheckStyle configurations below, see: http://checkstyle. - + From eee743a72d779d7af940a0ecf83393f26c2b4de2 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 29 Oct 2024 17:02:46 -0500 Subject: [PATCH 101/178] Minor checkstyle fixes after bump to 8.45.1. All are indentation / spacing fixes which are more strict now. --- .../rest/AuthenticationRestController.java | 2 +- .../app/rest/model/AuthorizationRest.java | 6 +-- .../dspace/app/rest/model/BitstreamRest.java | 15 ++----- .../app/rest/model/BrowseIndexRest.java | 10 +---- .../org/dspace/app/rest/model/BundleRest.java | 15 ++----- .../app/rest/model/ClaimedTaskRest.java | 5 +-- .../dspace/app/rest/model/CollectionRest.java | 40 ++++------------- .../dspace/app/rest/model/CommunityRest.java | 25 +++-------- .../dspace/app/rest/model/EPersonRest.java | 5 +-- .../dspace/app/rest/model/EntityTypeRest.java | 5 +-- .../app/rest/model/ExternalSourceRest.java | 5 +-- .../org/dspace/app/rest/model/GroupRest.java | 15 ++----- .../org/dspace/app/rest/model/ItemRest.java | 45 ++++--------------- .../app/rest/model/OrcidHistoryRest.java | 2 +- .../dspace/app/rest/model/PoolTaskRest.java | 5 +-- .../dspace/app/rest/model/ProcessRest.java | 15 ++----- .../app/rest/model/RelationshipRest.java | 5 +-- .../app/rest/model/ResearcherProfileRest.java | 4 +- .../app/rest/model/VersionHistoryRest.java | 10 +---- .../dspace/app/rest/model/VersionRest.java | 10 +---- .../model/VocabularyEntryDetailsRest.java | 6 +-- .../dspace/app/rest/model/VocabularyRest.java | 4 +- .../rest/model/WorkflowDefinitionRest.java | 10 +---- .../app/rest/model/WorkflowItemRest.java | 20 ++------- .../app/rest/model/WorkflowStepRest.java | 5 +-- .../app/rest/model/WorkspaceItemRest.java | 20 ++------- .../org/dspace/app/rest/utils/RegexUtils.java | 2 +- .../app/rest/matcher/RegistrationMatcher.java | 2 +- 28 files changed, 70 insertions(+), 243 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/AuthenticationRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/AuthenticationRestController.java index e8b8eb8e70da..2fbc116843ca 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/AuthenticationRestController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/AuthenticationRestController.java @@ -224,7 +224,7 @@ private AuthenticationTokenResource shortLivedTokenResponse(HttpServletRequest r * @return ResponseEntity */ @RequestMapping(value = "/login", method = { RequestMethod.GET, RequestMethod.PUT, RequestMethod.PATCH, - RequestMethod.DELETE }) + RequestMethod.DELETE }) public ResponseEntity login() { return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).body("Only POST is allowed for login requests."); } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/AuthorizationRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/AuthorizationRest.java index 95f288831303..7aec2cb4b76e 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/AuthorizationRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/AuthorizationRest.java @@ -18,9 +18,9 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest(method = "getEperson", name = AuthorizationRest.EPERSON), - @LinkRest(method = "getFeature", name = AuthorizationRest.FEATURE), - @LinkRest(method = "getObject", name = AuthorizationRest.OBJECT) + @LinkRest(method = "getEperson", name = AuthorizationRest.EPERSON), + @LinkRest(method = "getFeature", name = AuthorizationRest.FEATURE), + @LinkRest(method = "getObject", name = AuthorizationRest.OBJECT) }) public class AuthorizationRest extends BaseObjectRest { public static final String NAME = "authorization"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BitstreamRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BitstreamRest.java index 8e9efc2680b7..77b558c0faab 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BitstreamRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BitstreamRest.java @@ -16,18 +16,9 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = BitstreamRest.BUNDLE, - method = "getBundle" - ), - @LinkRest( - name = BitstreamRest.FORMAT, - method = "getFormat" - ), - @LinkRest( - name = BitstreamRest.THUMBNAIL, - method = "getThumbnail" - ) + @LinkRest(name = BitstreamRest.BUNDLE, method = "getBundle"), + @LinkRest(name = BitstreamRest.FORMAT, method = "getFormat"), + @LinkRest(name = BitstreamRest.THUMBNAIL, method = "getThumbnail") }) public class BitstreamRest extends DSpaceObjectRest { public static final String PLURAL_NAME = "bitstreams"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BrowseIndexRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BrowseIndexRest.java index f7978f00fdf5..176b268e4be1 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BrowseIndexRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BrowseIndexRest.java @@ -20,14 +20,8 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = BrowseIndexRest.LINK_ITEMS, - method = "listBrowseItems" - ), - @LinkRest( - name = BrowseIndexRest.LINK_ENTRIES, - method = "listBrowseEntries" - ) + @LinkRest(name = BrowseIndexRest.LINK_ITEMS, method = "listBrowseItems"), + @LinkRest(name = BrowseIndexRest.LINK_ENTRIES, method = "listBrowseEntries") }) public class BrowseIndexRest extends BaseObjectRest { private static final long serialVersionUID = -4870333170249999559L; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BundleRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BundleRest.java index dd4a80d488a8..7ef1ceed92c9 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BundleRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/BundleRest.java @@ -16,18 +16,9 @@ * @author Jelle Pelgrims (jelle.pelgrims at atmire.com) */ @LinksRest(links = { - @LinkRest( - name = BundleRest.ITEM, - method = "getItem" - ), - @LinkRest( - name = BundleRest.BITSTREAMS, - method = "getBitstreams" - ), - @LinkRest( - name = BundleRest.PRIMARY_BITSTREAM, - method = "getPrimaryBitstream" - ) + @LinkRest(name = BundleRest.ITEM, method = "getItem"), + @LinkRest(name = BundleRest.BITSTREAMS, method = "getBitstreams"), + @LinkRest(name = BundleRest.PRIMARY_BITSTREAM, method = "getPrimaryBitstream") }) public class BundleRest extends DSpaceObjectRest { public static final String NAME = "bundle"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ClaimedTaskRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ClaimedTaskRest.java index dd97fef44d0b..d9c907697573 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ClaimedTaskRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ClaimedTaskRest.java @@ -16,10 +16,7 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = ClaimedTaskRest.STEP, - method = "getStep" - ) + @LinkRest(name = ClaimedTaskRest.STEP, method = "getStep") }) public class ClaimedTaskRest extends BaseObjectRest { public static final String NAME = "claimedtask"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/CollectionRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/CollectionRest.java index 3f5ae3bb34c2..28ff9bcceb6b 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/CollectionRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/CollectionRest.java @@ -15,38 +15,14 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = CollectionRest.LICENSE, - method = "getLicense" - ), - @LinkRest( - name = CollectionRest.LOGO, - method = "getLogo" - ), - @LinkRest( - name = CollectionRest.MAPPED_ITEMS, - method = "getMappedItems" - ), - @LinkRest( - name = CollectionRest.PARENT_COMMUNITY, - method = "getParentCommunity" - ), - @LinkRest( - name = CollectionRest.ADMIN_GROUP, - method = "getAdminGroup" - ), - @LinkRest( - name = CollectionRest.SUBMITTERS_GROUP, - method = "getSubmittersGroup" - ), - @LinkRest( - name = CollectionRest.ITEM_READ_GROUP, - method = "getItemReadGroup" - ), - @LinkRest( - name = CollectionRest.BITSTREAM_READ_GROUP, - method = "getBitstreamReadGroup" - ), + @LinkRest(name = CollectionRest.LICENSE, method = "getLicense"), + @LinkRest(name = CollectionRest.LOGO, method = "getLogo"), + @LinkRest(name = CollectionRest.MAPPED_ITEMS, method = "getMappedItems"), + @LinkRest(name = CollectionRest.PARENT_COMMUNITY, method = "getParentCommunity"), + @LinkRest(name = CollectionRest.ADMIN_GROUP, method = "getAdminGroup"), + @LinkRest(name = CollectionRest.SUBMITTERS_GROUP, method = "getSubmittersGroup"), + @LinkRest(name = CollectionRest.ITEM_READ_GROUP, method = "getItemReadGroup"), + @LinkRest(name = CollectionRest.BITSTREAM_READ_GROUP, method = "getBitstreamReadGroup"), }) public class CollectionRest extends DSpaceObjectRest { public static final String NAME = "collection"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/CommunityRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/CommunityRest.java index 86dc4b2c3900..08758ef9cc27 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/CommunityRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/CommunityRest.java @@ -15,26 +15,11 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = CommunityRest.COLLECTIONS, - method = "getCollections" - ), - @LinkRest( - name = CommunityRest.LOGO, - method = "getLogo" - ), - @LinkRest( - name = CommunityRest.SUBCOMMUNITIES, - method = "getSubcommunities" - ), - @LinkRest( - name = CommunityRest.PARENT_COMMUNITY, - method = "getParentCommunity" - ), - @LinkRest( - name = CommunityRest.ADMIN_GROUP, - method = "getAdminGroup" - ) + @LinkRest(name = CommunityRest.COLLECTIONS, method = "getCollections"), + @LinkRest(name = CommunityRest.LOGO, method = "getLogo"), + @LinkRest(name = CommunityRest.SUBCOMMUNITIES, method = "getSubcommunities"), + @LinkRest(name = CommunityRest.PARENT_COMMUNITY, method = "getParentCommunity"), + @LinkRest(name = CommunityRest.ADMIN_GROUP, method = "getAdminGroup") }) public class CommunityRest extends DSpaceObjectRest { public static final String NAME = "community"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/EPersonRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/EPersonRest.java index 7b4c683322a9..0c4f4d7b290b 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/EPersonRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/EPersonRest.java @@ -20,10 +20,7 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = EPersonRest.GROUPS, - method = "getGroups" - ) + @LinkRest(name = EPersonRest.GROUPS, method = "getGroups") }) public class EPersonRest extends DSpaceObjectRest { public static final String NAME = "eperson"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/EntityTypeRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/EntityTypeRest.java index f777b3a29c18..f62ef2767ea9 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/EntityTypeRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/EntityTypeRest.java @@ -15,10 +15,7 @@ * Refer to {@link org.dspace.content.EntityType} for explanation of the properties */ @LinksRest(links = { - @LinkRest( - name = EntityTypeRest.RELATION_SHIP_TYPES, - method = "getEntityTypeRelationship" - ) + @LinkRest(name = EntityTypeRest.RELATION_SHIP_TYPES, method = "getEntityTypeRelationship") }) public class EntityTypeRest extends BaseObjectRest { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ExternalSourceRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ExternalSourceRest.java index 639c2cf72e2e..2cd3712bdd53 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ExternalSourceRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ExternalSourceRest.java @@ -13,10 +13,7 @@ * This class serves as a REST representation for an External Source */ @LinksRest(links = { - @LinkRest( - name = ExternalSourceRest.ENTITY_TYPES, - method = "getSupportedEntityTypes" - ) + @LinkRest(name = ExternalSourceRest.ENTITY_TYPES, method = "getSupportedEntityTypes") }) public class ExternalSourceRest extends BaseObjectRest { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/GroupRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/GroupRest.java index 3f60b2d61fd6..332f5a5055fb 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/GroupRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/GroupRest.java @@ -18,18 +18,9 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) @LinksRest(links = { - @LinkRest( - name = GroupRest.SUBGROUPS, - method = "getGroups" - ), - @LinkRest( - name = GroupRest.EPERSONS, - method = "getMembers" - ), - @LinkRest( - name = GroupRest.OBJECT, - method = "getParentObject" - ) + @LinkRest(name = GroupRest.SUBGROUPS, method = "getGroups"), + @LinkRest(name = GroupRest.EPERSONS, method = "getMembers"), + @LinkRest(name = GroupRest.OBJECT, method = "getParentObject") }) public class GroupRest extends DSpaceObjectRest { public static final String NAME = "group"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ItemRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ItemRest.java index 1254ef8f9372..bd0530be5dc4 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ItemRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ItemRest.java @@ -17,42 +17,15 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = ItemRest.ACCESS_STATUS, - method = "getAccessStatus" - ), - @LinkRest( - name = ItemRest.BUNDLES, - method = "getBundles" - ), - @LinkRest( - name = ItemRest.IDENTIFIERS, - method = "getIdentifiers" - ), - @LinkRest( - name = ItemRest.MAPPED_COLLECTIONS, - method = "getMappedCollections" - ), - @LinkRest( - name = ItemRest.OWNING_COLLECTION, - method = "getOwningCollection" - ), - @LinkRest( - name = ItemRest.RELATIONSHIPS, - method = "getRelationships" - ), - @LinkRest( - name = ItemRest.VERSION, - method = "getItemVersion" - ), - @LinkRest( - name = ItemRest.TEMPLATE_ITEM_OF, - method = "getTemplateItemOf" - ), - @LinkRest( - name = ItemRest.THUMBNAIL, - method = "getThumbnail" - ) + @LinkRest(name = ItemRest.ACCESS_STATUS, method = "getAccessStatus"), + @LinkRest(name = ItemRest.BUNDLES, method = "getBundles"), + @LinkRest(name = ItemRest.IDENTIFIERS, method = "getIdentifiers"), + @LinkRest(name = ItemRest.MAPPED_COLLECTIONS, method = "getMappedCollections"), + @LinkRest(name = ItemRest.OWNING_COLLECTION, method = "getOwningCollection"), + @LinkRest(name = ItemRest.RELATIONSHIPS, method = "getRelationships"), + @LinkRest(name = ItemRest.VERSION, method = "getItemVersion"), + @LinkRest(name = ItemRest.TEMPLATE_ITEM_OF, method = "getTemplateItemOf"), + @LinkRest(name = ItemRest.THUMBNAIL, method = "getThumbnail") }) public class ItemRest extends DSpaceObjectRest { public static final String NAME = "item"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/OrcidHistoryRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/OrcidHistoryRest.java index 02e0f4706208..abfc0a31dc6a 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/OrcidHistoryRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/OrcidHistoryRest.java @@ -38,7 +38,7 @@ public class OrcidHistoryRest extends BaseObjectRest { private String responseMessage; - public OrcidHistoryRest(){} + public OrcidHistoryRest() {} @Override @JsonProperty(access = JsonProperty.Access.READ_ONLY) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/PoolTaskRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/PoolTaskRest.java index c32c2c95783a..320558f94671 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/PoolTaskRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/PoolTaskRest.java @@ -17,10 +17,7 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = PoolTaskRest.STEP, - method = "getStep" - ) + @LinkRest(name = PoolTaskRest.STEP, method = "getStep") }) public class PoolTaskRest extends BaseObjectRest { public static final String NAME = "pooltask"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ProcessRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ProcessRest.java index 8216e1617195..f7d8088d8a30 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ProcessRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ProcessRest.java @@ -21,18 +21,9 @@ * This class serves as a REST representation for the {@link Process} class */ @LinksRest(links = { - @LinkRest( - name = ProcessRest.FILES, - method = "getFilesFromProcess" - ), - @LinkRest( - name = ProcessRest.FILE_TYPES, - method = "getFileTypesFromProcess" - ), - @LinkRest( - name = ProcessRest.OUTPUT, - method = "getOutputFromProcess" - ) + @LinkRest(name = ProcessRest.FILES, method = "getFilesFromProcess"), + @LinkRest(name = ProcessRest.FILE_TYPES, method = "getFileTypesFromProcess"), + @LinkRest(name = ProcessRest.OUTPUT, method = "getOutputFromProcess") }) public class ProcessRest extends BaseObjectRest { public static final String NAME = "process"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/RelationshipRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/RelationshipRest.java index dd35a0726e9d..d4050cda9a34 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/RelationshipRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/RelationshipRest.java @@ -19,10 +19,7 @@ * Refer to {@link org.dspace.content.Relationship} for explanation about the properties */ @LinksRest(links = { - @LinkRest( - name = RelationshipRest.RELATIONSHIP_TYPE, - method = "getRelationshipType" - ) + @LinkRest(name = RelationshipRest.RELATIONSHIP_TYPE, method = "getRelationshipType") }) public class RelationshipRest extends BaseObjectRest { public static final String NAME = "relationship"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ResearcherProfileRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ResearcherProfileRest.java index 4224cfeeb924..1ebde12e854f 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ResearcherProfileRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ResearcherProfileRest.java @@ -20,8 +20,8 @@ * */ @LinksRest(links = { - @LinkRest(name = ResearcherProfileRest.ITEM, method = "getItem"), - @LinkRest(name = ResearcherProfileRest.EPERSON, method = "getEPerson") + @LinkRest(name = ResearcherProfileRest.ITEM, method = "getItem"), + @LinkRest(name = ResearcherProfileRest.EPERSON, method = "getEPerson") }) public class ResearcherProfileRest extends BaseObjectRest { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VersionHistoryRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VersionHistoryRest.java index e93e131aadb7..51d926082e33 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VersionHistoryRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VersionHistoryRest.java @@ -13,14 +13,8 @@ * The REST object for the {@link org.dspace.versioning.VersionHistory} object */ @LinksRest(links = { - @LinkRest( - name = VersionHistoryRest.VERSIONS, - method = "getVersions" - ), - @LinkRest( - name = VersionHistoryRest.DRAFT_VERSION, - method = "getDraftVersion" - ) + @LinkRest(name = VersionHistoryRest.VERSIONS, method = "getVersions"), + @LinkRest(name = VersionHistoryRest.DRAFT_VERSION, method = "getDraftVersion") }) public class VersionHistoryRest extends BaseObjectRest { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VersionRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VersionRest.java index abdc64295fdc..b4a7ce2b016c 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VersionRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VersionRest.java @@ -16,14 +16,8 @@ * The REST object for the {@link org.dspace.versioning.Version} objects */ @LinksRest(links = { - @LinkRest( - name = VersionRest.VERSION_HISTORY, - method = "getVersionHistory" - ), - @LinkRest( - name = VersionRest.ITEM, - method = "getVersionItem" - ) + @LinkRest(name = VersionRest.VERSION_HISTORY, method = "getVersionHistory"), + @LinkRest(name = VersionRest.ITEM, method = "getVersionItem") }) public class VersionRest extends BaseObjectRest { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VocabularyEntryDetailsRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VocabularyEntryDetailsRest.java index 30e5eb71cbff..964c67dbbe8e 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VocabularyEntryDetailsRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VocabularyEntryDetailsRest.java @@ -18,9 +18,9 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest(name = VocabularyEntryDetailsRest.PARENT, method = "getParent"), - @LinkRest(name = VocabularyEntryDetailsRest.CHILDREN, method = "getChildren") - }) + @LinkRest(name = VocabularyEntryDetailsRest.PARENT, method = "getParent"), + @LinkRest(name = VocabularyEntryDetailsRest.CHILDREN, method = "getChildren") +}) public class VocabularyEntryDetailsRest extends BaseObjectRest { public static final String PLURAL_NAME = "vocabularyEntryDetails"; public static final String NAME = "vocabularyEntryDetail"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VocabularyRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VocabularyRest.java index cc848b945b2f..45d13076acb2 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VocabularyRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/VocabularyRest.java @@ -15,9 +15,7 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest(name = VocabularyRest.ENTRIES, - method = "filter" - ), + @LinkRest(name = VocabularyRest.ENTRIES, method = "filter"), }) public class VocabularyRest extends BaseObjectRest { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowDefinitionRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowDefinitionRest.java index 7c2de7071bde..9ea4e2ae8c22 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowDefinitionRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowDefinitionRest.java @@ -18,14 +18,8 @@ * @author Maria Verdonck (Atmire) on 11/12/2019 */ @LinksRest(links = { - @LinkRest( - name = WorkflowDefinitionRest.COLLECTIONS_MAPPED_TO, - method = "getCollections" - ), - @LinkRest( - name = WorkflowDefinitionRest.STEPS, - method = "getSteps" - ) + @LinkRest(name = WorkflowDefinitionRest.COLLECTIONS_MAPPED_TO, method = "getCollections"), + @LinkRest(name = WorkflowDefinitionRest.STEPS, method = "getSteps") }) public class WorkflowDefinitionRest extends BaseObjectRest { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowItemRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowItemRest.java index 4a840a0bb77b..e94cfd54644d 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowItemRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowItemRest.java @@ -15,22 +15,10 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = WorkflowItemRest.STEP, - method = "getStep" - ), - @LinkRest( - name = WorkflowItemRest.SUBMITTER, - method = "getWorkflowItemSubmitter" - ), - @LinkRest( - name = WorkflowItemRest.ITEM, - method = "getWorkflowItemItem" - ), - @LinkRest( - name = WorkflowItemRest.COLLECTION, - method = "getWorkflowItemCollection" - ) + @LinkRest(name = WorkflowItemRest.STEP, method = "getStep"), + @LinkRest(name = WorkflowItemRest.SUBMITTER, method = "getWorkflowItemSubmitter"), + @LinkRest(name = WorkflowItemRest.ITEM, method = "getWorkflowItemItem"), + @LinkRest(name = WorkflowItemRest.COLLECTION, method = "getWorkflowItemCollection") }) public class WorkflowItemRest extends AInprogressSubmissionRest { public static final String NAME = "workflowitem"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowStepRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowStepRest.java index 648cffbca80d..91128fd05a7d 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowStepRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkflowStepRest.java @@ -18,10 +18,7 @@ * @author Maria Verdonck (Atmire) on 10/01/2020 */ @LinksRest(links = { - @LinkRest( - name = WorkflowStepRest.ACTIONS, - method = "getActions" - ), + @LinkRest(name = WorkflowStepRest.ACTIONS, method = "getActions"), }) public class WorkflowStepRest extends BaseObjectRest { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkspaceItemRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkspaceItemRest.java index b40d1a4494f4..7693771fb9f2 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkspaceItemRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/WorkspaceItemRest.java @@ -15,22 +15,10 @@ * @author Andrea Bollini (andrea.bollini at 4science.it) */ @LinksRest(links = { - @LinkRest( - name = WorkspaceItemRest.SUPERVISION_ORDERS, - method = "getSupervisionOrders" - ), - @LinkRest( - name = WorkspaceItemRest.SUBMITTER, - method = "getWorkspaceItemSubmitter" - ), - @LinkRest( - name = WorkspaceItemRest.ITEM, - method = "getWorkspaceItemItem" - ), - @LinkRest( - name = WorkspaceItemRest.COLLECTION, - method = "getWorkspaceItemCollection" - ) + @LinkRest(name = WorkspaceItemRest.SUPERVISION_ORDERS, method = "getSupervisionOrders"), + @LinkRest(name = WorkspaceItemRest.SUBMITTER, method = "getWorkspaceItemSubmitter"), + @LinkRest(name = WorkspaceItemRest.ITEM, method = "getWorkspaceItemItem"), + @LinkRest(name = WorkspaceItemRest.COLLECTION, method = "getWorkspaceItemCollection") }) public class WorkspaceItemRest extends AInprogressSubmissionRest { public static final String NAME = "workspaceitem"; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/RegexUtils.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/RegexUtils.java index 8739f6b8d57f..856e36457057 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/RegexUtils.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/RegexUtils.java @@ -13,7 +13,7 @@ */ public class RegexUtils { - private RegexUtils(){} + private RegexUtils() {} /** * Regular expression in the request mapping to accept UUID as identifier diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/RegistrationMatcher.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/RegistrationMatcher.java index a154091a2eff..2a4cee8375be 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/RegistrationMatcher.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/RegistrationMatcher.java @@ -17,7 +17,7 @@ public class RegistrationMatcher { - private RegistrationMatcher(){} + private RegistrationMatcher() {} public static Matcher matchRegistration(String email, UUID epersonUuid) { return allOf( From 5d7b42603d5d68bbffef0226fca3116e7029e401 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 3 Dec 2024 09:34:02 -0600 Subject: [PATCH 102/178] Add newly required "should-stop" flag to errorprone config. See https://errorprone.info/docs/installation --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index f496ed7e879d..f3b09f3a9dd9 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,7 @@ true -XDcompilePolicy=simple + --should-stop=ifError=FLOW -Xplugin:ErrorProne -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED From f7ee509423591c33621e9b6abf80135c08f16522 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 3 Dec 2024 09:44:56 -0600 Subject: [PATCH 103/178] Fix duplicate code warning from errorprone. This "else if" clause is the same as the "else" and can be removed --- .../src/main/java/org/dspace/discovery/SolrServiceImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java index 9339b574b578..34efe96ae7f8 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java @@ -1445,8 +1445,6 @@ protected String transformFacetField(DiscoverFacetField facetFieldConfig, String } else { return field + "_acid"; } - } else if (facetFieldConfig.getType().equals(DiscoveryConfigurationParameters.TYPE_STANDARD)) { - return field; } else { return field; } From 1581c737e2a84fa217c334b2022b80f2fdd2ff15 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 30 Oct 2024 14:49:04 -0500 Subject: [PATCH 104/178] Update jsoneditor.js reference in Hal Browser --- dspace-server-webapp/src/main/webapp/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/main/webapp/index.html b/dspace-server-webapp/src/main/webapp/index.html index c780286107d8..0b80f806767e 100644 --- a/dspace-server-webapp/src/main/webapp/index.html +++ b/dspace-server-webapp/src/main/webapp/index.html @@ -321,7 +321,7 @@

    Embedded Resources

    - +