Merge pull request #402 from Breeding-Insight/bug/BI-2309

BI-2308 - Field Book BrAPI Integration - Importing Fields

src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java

@@ -1,5 +1,6 @@
 package org.breedinginsight.brapi.v2;
 
+import io.micronaut.context.annotation.Property;
 import io.micronaut.http.HttpHeaders;
 import io.micronaut.http.HttpResponse;
 import io.micronaut.http.HttpStatus;
@@ -18,6 +19,7 @@
 import org.brapi.v2.model.BrAPIIndexPagination;
 import org.brapi.v2.model.BrAPIMetadata;
 import org.brapi.v2.model.BrAPIStatus;
+import org.brapi.v2.model.core.BrAPITrial;
 import org.brapi.v2.model.germ.*;
 import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest;
 import org.brapi.v2.model.germ.response.*;
@@ -31,6 +33,7 @@
 import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
 import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO;
 import org.breedinginsight.brapi.v2.model.request.query.GermplasmQuery;
+import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
 import org.breedinginsight.model.Program;
 import org.breedinginsight.services.ProgramService;
 import org.breedinginsight.utilities.Utilities;
@@ -49,11 +52,14 @@
 import javax.inject.Inject;
 import javax.validation.Valid;
 import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 @Slf4j
 @Controller("/${micronaut.bi.api.version}")
 @Secured(SecurityRule.IS_AUTHENTICATED)
 public class BrAPIGermplasmController {
+    private final String referenceSource;
 
     private final BrAPIGermplasmService germplasmService;
     private final GermplasmQueryMapper germplasmQueryMapper;
@@ -67,13 +73,15 @@ public class BrAPIGermplasmController {
 
 
     @Inject
-    public BrAPIGermplasmController(BrAPIGermplasmService germplasmService,
+    public BrAPIGermplasmController(@Property(name = "brapi.server.reference-source") String referenceSource,
+                                    BrAPIGermplasmService germplasmService,
                                     GermplasmQueryMapper germplasmQueryMapper,
                                     ProgramDAO programDAO,
                                     BrAPIGermplasmDAO germplasmDAO,
                                     GenotypeService genoService,
                                     BrAPIEndpointProvider brAPIEndpointProvider,
                                     ProgramService programService) {
+        this.referenceSource = referenceSource;
         this.germplasmService = germplasmService;
         this.germplasmQueryMapper = germplasmQueryMapper;
         this.programDAO = programDAO;
@@ -84,12 +92,13 @@ public BrAPIGermplasmController(BrAPIGermplasmService germplasmService,
     }
 
     // NOTE: bypasses cache and makes api request directly to brapi service
+    // Needs to convert DeltaBreed UUIDs to BrAPI Service DbIds and back
     @Post("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/search/germplasm")
     @Produces(MediaType.APPLICATION_JSON)
     @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES})
     public HttpResponse<Object> searchGermplasm(
             @PathVariable("programId") UUID programId,
-            @Body BrAPIGermplasmSearchRequest body) throws ApiException {
+            @Body BrAPIGermplasmSearchRequest body) throws ApiException, DoesNotExistException {
 
         log.debug("searchGermplasm: fetching germplasm by filters");
 
@@ -105,19 +114,84 @@ public HttpResponse<Object> searchGermplasm(
         String extRefId = program.get().getId().toString();
         body.externalReferenceIds(List.of(extRefId));
 
+        // convert request filter dbIds from DeltaBreed UUID to BrAPI service dbIds
+        List<String> convertedDbIds = germplasmService.getGermplasmDbIdsForUUIDs(program.get().getId(), body.getGermplasmDbIds());
+        body.setGermplasmDbIds(convertedDbIds);
+
         ApiResponse<Pair<Optional<BrAPIGermplasmListResponse>, Optional<BrAPIAcceptedSearchResponse>>> brapiGermplasm;
         brapiGermplasm = brAPIEndpointProvider
                 .get(programDAO.getCoreClient(program.get().getId()), GermplasmApi.class)
                 .searchGermplasmPost(body);
 
+        return getObjectHttpResponse(brapiGermplasm, program.get().getKey());
+    }
+
+    // NOTE: bypasses cache and makes api request directly to brapi service
+    // Needs to convert BrAPIService dbIds to DeltaBreed UUIDs
+    @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/search/germplasm/{searchResultId}{?queryParams*}")
+    @Produces(MediaType.APPLICATION_JSON)
+    @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES})
+    public HttpResponse<Object> getSearchResult(
+            @PathVariable("programId") UUID programId,
+            @PathVariable("searchResultId") UUID searchResultId,
+            @QueryValue @QueryValid(using = GermplasmQueryMapper.class) @Valid GermplasmQuery queryParams) throws ApiException {
+
+        log.debug("getGermplasmResult: getting results");
+
+        Optional<Program> program = programService.getById(programId);
+        if(program.isEmpty()) {
+            log.warn("Program id: " + programId + " not found");
+            return HttpResponse.notFound();
+        }
+
+        ApiResponse<Pair<Optional<BrAPIGermplasmListResponse>, Optional<BrAPIAcceptedSearchResponse>>> brapiGermplasm;
+        brapiGermplasm = brAPIEndpointProvider
+                .get(programDAO.getCoreClient(program.get().getId()), GermplasmApi.class)
+                .searchGermplasmSearchResultsDbIdGet(searchResultId.toString(), queryParams.getPage(), queryParams.getPageSize());
+
+        return getObjectHttpResponse(brapiGermplasm, program.get().getKey());
+    }
+
+    private HttpResponse<Object> getObjectHttpResponse(ApiResponse<Pair<Optional<BrAPIGermplasmListResponse>, Optional<BrAPIAcceptedSearchResponse>>> brapiGermplasm,
+                                                       String programKey) throws ApiException {
         if (brapiGermplasm.getBody().getLeft().isPresent()) {
-            return HttpResponse.ok(brapiGermplasm.getBody().getLeft().get());
+            // convert dbIds to DeltaBreed UUID
+            BrAPIGermplasmListResponse response = brapiGermplasm.getBody().getLeft().get();
+            List<BrAPIGermplasm> germplasm = response.getResult().getData();
+            //germplasm.forEach(g -> setDbIdsAndStripProgramKeys(g, programKey));
+            batchProcessGermplasm(germplasm, programKey);
+            return HttpResponse.ok(response);
         } else if (brapiGermplasm.getBody().getRight().isPresent()) {
             return HttpResponse.ok(brapiGermplasm.getBody().getRight().get());
         } else {
             throw new ApiException("Expected search response");
         }
+    }
 
+    /**
+     * Keep dbIds in DeltaBreed UUID context and strip program keys from synonyms and pedigree string for Field Book display
+     * @param germplasmList
+     * @param programKey
+     */
+    private void batchProcessGermplasm(List<BrAPIGermplasm> germplasmList, String programKey) throws IllegalStateException {
+        // Prepare a regex pattern for program key removal
+        Pattern programKeyPattern = Utilities.getRegexPatternMatchAllProgramKeysAnyAccession(programKey);
+        germplasmList.parallelStream().forEach(germplasm -> {
+            // Set dbId
+            germplasm.germplasmDbId(Utilities.getExternalReference(germplasm.getExternalReferences(), "breedinginsight.org")
+                    .orElseThrow(() -> new IllegalStateException("No BI external reference found"))
+                    .getReferenceId());
+            // Process synonyms
+            if (germplasm.getSynonyms() != null) {
+                germplasm.getSynonyms().forEach(synonym -> {
+                    synonym.setSynonym(Utilities.removeProgramKey(synonym.getSynonym(), programKey, germplasm.getAccessionNumber()));
+                });
+            }
+            // Process pedigree
+            if (germplasm.getPedigree() != null) {
+                germplasm.setPedigree(programKeyPattern.matcher(germplasm.getPedigree()).replaceAll(""));
+            }
+        });
     }
 
     @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/germplasm{?queryParams*}")

src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java

@@ -95,42 +95,36 @@ public HttpResponse<Response<DataResponse<List<BrAPIStudy>>>> getStudies(
             @PathVariable("programId") UUID programId,
             @QueryValue @QueryValid(using = StudyQueryMapper.class) @Valid StudyQuery queryParams) {
         try {
+            Optional<Program> program = programService.getById(programId);
+            if (program.isEmpty()) { return HttpResponse.notFound(); }
+
+            queryParams.setSortField(studyQueryMapper.getDefaultSortField());
+            queryParams.setSortOrder(studyQueryMapper.getDefaultSortOrder());
+            SearchRequest searchRequest = queryParams.constructSearchRequest();
             log.debug("fetching studies for program: " + programId);
-            List<BrAPIStudy> studies;
 
+            // If the program user is an experimental collaborator, filter results for only authorized studies.
             Optional<ProgramUser> experimentalCollaborator = programUserService.getIfExperimentalCollaborator(programId, securityService.getUser().getId());
-            // If the program user is an experimental collaborator, filter results.
             if (experimentalCollaborator.isPresent()) {
-                Optional<Program> program = programService.getById(programId);
-                if (program.isEmpty()) {
-                    return HttpResponse.notFound();
-                }
-
-                List<UUID> experimentIds = experimentalCollaboratorService.getAuthorizedExperimentIds(experimentalCollaborator.get().getId());
-                studies = studyService.getStudiesByExperimentIds(program.get(), experimentIds)
+                List<UUID> authorizedExperimentIds = experimentalCollaboratorService.getAuthorizedExperimentIds(experimentalCollaborator.get().getId());
+                List<BrAPIStudy> authorizedStudies = studyService.getStudiesByExperimentIds(program.get(), authorizedExperimentIds)
                         .stream()
                         .peek(this::setDbIds)
                         .collect(Collectors.toList());
-            } else {
-                studies = studyService.getStudies(programId)
+                return ResponseUtils.getBrapiQueryResponse(authorizedStudies, studyQueryMapper, queryParams, searchRequest);
+            }
+
+            List<BrAPIStudy> studies = studyService.getStudies(programId)
                         .stream()
                         .peek(this::setDbIds)
                         .collect(Collectors.toList());
-            }
-
-            queryParams.setSortField(studyQueryMapper.getDefaultSortField());
-            queryParams.setSortOrder(studyQueryMapper.getDefaultSortOrder());
-            SearchRequest searchRequest = queryParams.constructSearchRequest();
             return ResponseUtils.getBrapiQueryResponse(studies, studyQueryMapper, queryParams, searchRequest);
         } catch (ApiException e) {
             log.info(e.getMessage(), e);
             return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving study");
         } catch (IllegalArgumentException e) {
             log.info(e.getMessage(), e);
             return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, "Error parsing requested date format");
-        } catch (DoesNotExistException e) {
-            log.info(e.getMessage(), e);
-            return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage());
         }
     }
 

src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java

@@ -95,23 +95,21 @@ public HttpResponse<Response<DataResponse<List<BrAPITrial>>>> getExperiments(
             @PathVariable("programId") UUID programId,
             @QueryValue @QueryValid(using = ExperimentQueryMapper.class) @Valid ExperimentQuery queryParams) {
         try {
-            List<BrAPITrial> experiments = new ArrayList<>();
+            Optional<Program> program = programService.getById(programId);
+            if (program.isEmpty()) { return HttpResponse.notFound(); }
+
+            SearchRequest searchRequest = queryParams.constructSearchRequest();
             log.debug("fetching trials for program: " + programId);
 
+            // If the program user is an experimental collaborator, filter results for only authorized experiments.
             Optional<ProgramUser> experimentalCollaborator = programUserService.getIfExperimentalCollaborator(programId, securityService.getUser().getId());
-            // If the program user is an experimental collaborator, filter results.
             if (experimentalCollaborator.isPresent()) {
-                Optional<Program> program = programService.getById(programId);
-                if (program.isEmpty()) {
-                    return HttpResponse.notFound();
-                }
                 List<UUID> experimentIds = experimentalCollaboratorService.getAuthorizedExperimentIds(experimentalCollaborator.get().getId());
-                experiments = experimentService.getTrialsByExperimentIds(program.get(), experimentIds).stream().peek(this::setDbIds).collect(Collectors.toList());
-            } else {
-                experiments = experimentService.getExperiments(programId).stream().peek(this::setDbIds).collect(Collectors.toList());
+                List<BrAPITrial> authorizedExperiments = experimentService.getTrialsByExperimentIds(program.get(), experimentIds).stream().peek(this::setDbIds).collect(Collectors.toList());
+                return ResponseUtils.getBrapiQueryResponse(authorizedExperiments, experimentQueryMapper, queryParams, searchRequest);
             }
 
-            SearchRequest searchRequest = queryParams.constructSearchRequest();
+            List<BrAPITrial> experiments = experimentService.getExperiments(programId).stream().peek(this::setDbIds).collect(Collectors.toList());
             return ResponseUtils.getBrapiQueryResponse(experiments, experimentQueryMapper, queryParams, searchRequest);
         } catch (ApiException e) {
             log.info(e.getMessage(), e);

src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java

@@ -345,6 +345,22 @@ public BrAPIGermplasm getGermplasmByUUID(String germplasmId, UUID programId) thr
         return germplasm;
     }
 
+    public List<String> getGermplasmDbIdsForUUIDs(List<String> germplasmUUIDs, UUID programId) throws ApiException, DoesNotExistException {
+        Map<String, BrAPIGermplasm> cache = programGermplasmCache.get(programId);
+        List<String> germplasmList = new ArrayList<>();
+        if (cache != null) {
+            // not using streams because want to throw checked exception
+            for (String germplasmUUID : germplasmUUIDs) {
+                BrAPIGermplasm germplasm = cache.get(germplasmUUID);
+                if (germplasm == null) {
+                    throw new DoesNotExistException("UUID for this germplasm does not exist: " + germplasmUUID);
+                }
+                germplasmList.add(germplasm.getGermplasmDbId());
+            }
+        }
+        return germplasmList;
+    }
+
     public Optional<BrAPIGermplasm> getGermplasmByDBID(String germplasmDbId, UUID programId) throws ApiException {
         Map<String, BrAPIGermplasm> cache = programGermplasmCache.get(programId);
         //key is UUID, want to filter by DBID

src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java

@@ -68,6 +68,14 @@ public BrAPIGermplasm getGermplasmByUUID(UUID programId, String germplasmId) thr
         }
     }
 
+    public List<String> getGermplasmDbIdsForUUIDs(UUID programId, List<String> germplasmUUIDs) throws DoesNotExistException {
+        try {
+            return germplasmDAO.getGermplasmDbIdsForUUIDs(germplasmUUIDs, programId);
+        } catch (ApiException e) {
+            throw new InternalServerException(e.getMessage(), e);
+        }
+    }
+
     public Optional<BrAPIGermplasm> getGermplasmByDBID(UUID programId, String germplasmId) throws ApiException {
         return germplasmDAO.getGermplasmByDBID(germplasmId, programId);
     }

src/main/java/org/breedinginsight/services/ProgramUserService.java

@@ -281,16 +281,13 @@ public boolean existsAndActive(UUID programId, UUID userId) {
      * @param userId the user ID.
      * @return an Optional that unwraps to a program user if it is an Experimental Collaborator, Optional.empty() otherwise.
      */
-    public Optional<ProgramUser> getIfExperimentalCollaborator(UUID programId, UUID userId) throws DoesNotExistException {
-        Optional<ProgramUser> programUser = getProgramUserbyId(programId, userId);
-        if (programUser.isEmpty()) {
-            throw new DoesNotExistException("Program User does not exist");
-        }
-        boolean isExperimentalCollaborator = programUser.get().getRoles().stream().anyMatch(x -> ProgramSecuredRole.getEnum(x.getDomain()).equals(ProgramSecuredRole.EXPERIMENTAL_COLLABORATOR));
-        if (isExperimentalCollaborator) {
-            return programUser;
-        }
-        return Optional.empty();
+    public Optional<ProgramUser> getIfExperimentalCollaborator(UUID programId, UUID userId) {
+        return getProgramUserbyId(programId, userId)
+                .flatMap(user -> user.getRoles().stream()
+                        .anyMatch(role -> ProgramSecuredRole.getEnum(role.getDomain()).equals(ProgramSecuredRole.EXPERIMENTAL_COLLABORATOR))
+                        ? Optional.of(user)
+                        : Optional.empty()
+                );
     }
 
     public List<ProgramUser> getProgramUsersByRole(UUID programId, UUID roleId) throws DoesNotExistException {

src/main/java/org/breedinginsight/utilities/Utilities.java

@@ -32,6 +32,7 @@
 import java.util.Optional;
 import java.util.UUID;
 import java.util.function.Function;
+import java.util.regex.Pattern;
 
 public class Utilities {
 
@@ -97,6 +98,10 @@ public static String removeProgramKeyAnyAccession(String str, String programKey)
         return str.replaceAll("\\[" + programKey + "-.*\\]", "").trim();
     }
 
+    public static Pattern getRegexPatternMatchAllProgramKeysAnyAccession(String programKey) {
+        return Pattern.compile(String.format("\\s*\\[%s-.*?\\]\\s*", programKey));
+    }
+
     /**
      * Remove program key from a string. Returns a new value instead of altering original string.
      *
@@ -162,8 +167,21 @@ public static Object formatBrapiObjForDisplay(Object brapiInstance, Class brapiC
         return brapiInstance;
     }
 
+    /**
+     * \s*: Matches zero or more whitespace characters before the opening bracket.
+     * \[: Matches the opening square bracket [. The backslash is used to escape the special meaning of [ in regex.
+     * .*?: Matches any character (except newline) zero or more times, non-greedily.
+     * . matches any character except newline.
+     * * means "zero or more times".
+     * ? makes the matching non-greedy, so it stops at the first closing bracket.
+     * \]: Matches the closing square bracket ]. Again, the backslash is used to escape it.
+     * \s*: Matches zero or more whitespace characters after the closing bracket.
+     * @param original
+     * @param programKey
+     * @return
+     */
     public static String removeProgramKeyAndUnknownAdditionalData(String original, String programKey) {
-        String keyValueRegEx = String.format(" \\[%s\\-.*\\]", programKey);
+        String keyValueRegEx = String.format("\\s*\\[%s-.*?\\]\\s*", programKey);
         String stripped =  original.replaceAll(keyValueRegEx, "");
         return stripped;
     }

src/main/resources/version.properties

@@ -15,4 +15,4 @@
 #
 
 version=v0.10.0+821
-versionInfo=https://github.com/Breeding-Insight/bi-api/commit/83acbff581f091ab19ecec2ea97c9a111be78fc5
+versionInfo=https://github.com/Breeding-Insight/bi-api/commit/83acbff581f091ab19ecec2ea97c9a111be78fc5