From 4f86ab46183afaaae1f3dee2598eee715325d90f Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Fri, 20 Aug 2021 08:57:01 +0200 Subject: [PATCH 1/6] Avoid assertTrue --- .../importer/util/GrobidServiceTest.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java index b4a0f9e8f0d..c1c4b563466 100644 --- a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java +++ b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java @@ -40,23 +40,24 @@ public static void setup() { @Test public void processValidCitationTest() throws IOException { + String expectedResponse = + "@article{-1,\n" + + " author = {Derwing, Tracey and Rossiter, Marian and Munro, Murray},\n" + + " title = {Teaching Native Speakers to Listen to Foreign-accented Speech},\n" + + " journal = {Journal of Multilingual and Multicultural Development},\n" + + " publisher = {Informa UK Limited},\n" + + " date = {2002-09},\n" + + " year = {2002},\n" + + " month = {9},\n" + + " pages = {245-259},\n" + + " volume = {23},\n" + + " number = {4},\n" + + " doi = {10.1080/01434630208666468}\n" + + "}\n"; String response = grobidService.processCitation("Derwing, T. M., Rossiter, M. J., & Munro, " + "M. J. (2002). Teaching native speakers to listen to foreign-accented speech. " + "Journal of Multilingual and Multicultural Development, 23(4), 245-259.", GrobidService.ConsolidateCitations.WITH_METADATA); - String[] responseRows = response.split("\n"); - assertNotNull(response); - assertEquals('@', response.charAt(0)); - assertTrue(responseRows[1].contains("author") && responseRows[1].contains("Derwing, Tracey and Rossiter, Marian and Munro, Murray")); - assertTrue(responseRows[2].contains("title") && responseRows[2].contains("Teaching Native Speakers to Listen to Foreign-accented Speech")); - assertTrue(responseRows[3].contains("journal") && responseRows[3].contains("Journal of Multilingual and Multicultural")); - assertTrue(responseRows[4].contains("publisher") && responseRows[4].contains("Informa UK Limited")); - assertTrue(responseRows[5].contains("date") && responseRows[5].contains("2002-09")); - assertTrue(responseRows[6].contains("year") && responseRows[6].contains("2002")); - assertTrue(responseRows[7].contains("month") && responseRows[7].contains("9")); - assertTrue(responseRows[8].contains("pages") && responseRows[8].contains("245-259")); - assertTrue(responseRows[9].contains("volume") && responseRows[9].contains("23")); - assertTrue(responseRows[10].contains("number") && responseRows[10].contains("4")); - assertTrue(responseRows[11].contains("doi") && responseRows[11].contains("10.1080/01434630208666468")); + assertEquals(expectedResponse, response); } @Test From 78fe5fa7bda26a4a993070487d9e2aaf47ab5f90 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Fri, 20 Aug 2021 09:28:05 +0200 Subject: [PATCH 2/6] Checkstyle --- .../java/org/jabref/logic/importer/util/GrobidServiceTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java index c1c4b563466..38fdaac91d0 100644 --- a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java +++ b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java @@ -21,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; From d435cefff2652cbaf186448a9ee0b84e7fc0ad99 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Fri, 20 Aug 2021 10:41:44 +0200 Subject: [PATCH 3/6] Refactor processCitation --- .../fetcher/GrobidCitationFetcher.java | 19 ++------ .../logic/importer/util/GrobidService.java | 9 ++-- .../fetcher/GrobidCitationFetcherTest.java | 6 ++- .../importer/util/GrobidServiceTest.java | 45 ++++++++++--------- 4 files changed, 35 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index 7955e206fd7..7634a67cfd6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -12,10 +12,8 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.SearchBasedFetcher; -import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.importer.util.GrobidService; import org.jabref.model.entry.BibEntry; -import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; @@ -45,29 +43,20 @@ public GrobidCitationFetcher(ImportFormatPreferences importFormatPreferences) { * * @return A BibTeX string if extraction is successful */ - private Optional parseUsingGrobid(String plainText) throws RuntimeException { + private Optional parseUsingGrobid(String plainText) throws RuntimeException { try { - return Optional.of(grobidService.processCitation(plainText, GrobidService.ConsolidateCitations.WITH_METADATA)); + return grobidService.processCitation(plainText, importFormatPreferences, GrobidService.ConsolidateCitations.WITH_METADATA); } catch (SocketTimeoutException e) { String msg = "Connection timed out."; LOGGER.debug(msg, e); throw new RuntimeException(msg, e); - } catch (IOException e) { + } catch (IOException | ParseException e) { String msg = "Could not process citation. " + e.getMessage(); LOGGER.debug(msg, e); return Optional.empty(); } } - private Optional parseBibToBibEntry(String bibtexString) { - try { - return BibtexParser.singleFromString(bibtexString, - importFormatPreferences, new DummyFileUpdateMonitor()); - } catch (ParseException e) { - return Optional.empty(); - } - } - @Override public String getName() { return "GROBID"; @@ -82,8 +71,6 @@ public List performSearch(String searchQuery) throws FetcherException .filter(str -> !str.isBlank()) .map(this::parseUsingGrobid) .flatMap(Optional::stream) - .map(this::parseBibToBibEntry) - .flatMap(Optional::stream) .collect(Collectors.toList()); } catch (RuntimeException e) { throw new FetcherException(e.getMessage(), e.getCause()); diff --git a/src/main/java/org/jabref/logic/importer/util/GrobidService.java b/src/main/java/org/jabref/logic/importer/util/GrobidService.java index 3259a44869a..8d99a52f66c 100644 --- a/src/main/java/org/jabref/logic/importer/util/GrobidService.java +++ b/src/main/java/org/jabref/logic/importer/util/GrobidService.java @@ -4,6 +4,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Optional; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; @@ -46,12 +47,12 @@ public GrobidService(String grobidServerURL) { } /** - * Calls the Grobid server for converting the citation into BibTeX + * Calls the Grobid server for converting the citation into a BibEntry * - * @return A plain BibTeX string (generated by the Grobid server) + * @return A BibEntry for the String * @throws IOException if an I/O excecption during the call ocurred or no BibTeX entry could be determiend */ - public String processCitation(String rawCitation, ConsolidateCitations consolidateCitations) throws IOException { + public Optional processCitation(String rawCitation, ImportFormatPreferences importFormatPreferences, ConsolidateCitations consolidateCitations) throws IOException, ParseException { Connection.Response response = Jsoup.connect(grobidServerURL + "/api/processCitation") .header("Accept", MediaTypes.APPLICATION_BIBTEX) .data("citations", rawCitation) @@ -66,7 +67,7 @@ public String processCitation(String rawCitation, ConsolidateCitations consolida throw new IOException("The GROBID server response does not contain anything."); } - return httpResponse; + return BibtexParser.singleFromString(httpResponse, importFormatPreferences, new DummyFileUpdateMonitor()); } public List processPDF(Path filePath, ImportFormatPreferences importFormatPreferences) throws IOException, ParseException { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java index a8b4f5267d4..f5e6a0c55c0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java @@ -8,12 +8,14 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.util.GrobidService; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -116,9 +118,9 @@ public void grobidPerformSearchWithInvalidDataTest(String invalidInput) throws F } @Test - public void performSearchThrowsExceptionInCaseOfConnectionIssues() throws IOException { + public void performSearchThrowsExceptionInCaseOfConnectionIssues() throws IOException, ParseException { GrobidService grobidServiceMock = mock(GrobidService.class); - when(grobidServiceMock.processCitation(anyString(), any())).thenThrow(new SocketTimeoutException("Timeout")); + when(grobidServiceMock.processCitation(anyString(), any(), any())).thenThrow(new SocketTimeoutException("Timeout")); grobidCitationFetcher = new GrobidCitationFetcher(importFormatPreferences, grobidServiceMock); assertThrows(FetcherException.class, () -> { diff --git a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java index 38fdaac91d0..8c9c595c15f 100644 --- a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java +++ b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java @@ -12,12 +12,14 @@ import org.jabref.logic.importer.fileformat.PdfGrobidImporterTest; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.Answers; +import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -38,37 +40,36 @@ public static void setup() { } @Test - public void processValidCitationTest() throws IOException { - String expectedResponse = - "@article{-1,\n" + - " author = {Derwing, Tracey and Rossiter, Marian and Munro, Murray},\n" + - " title = {Teaching Native Speakers to Listen to Foreign-accented Speech},\n" + - " journal = {Journal of Multilingual and Multicultural Development},\n" + - " publisher = {Informa UK Limited},\n" + - " date = {2002-09},\n" + - " year = {2002},\n" + - " month = {9},\n" + - " pages = {245-259},\n" + - " volume = {23},\n" + - " number = {4},\n" + - " doi = {10.1080/01434630208666468}\n" + - "}\n"; - String response = grobidService.processCitation("Derwing, T. M., Rossiter, M. J., & Munro, " + + public void processValidCitationTest() throws IOException, ParseException { + BibEntry exampleBibEntry = new BibEntry(StandardEntryType.Article).withCitationKey("-1") + .withField(StandardField.AUTHOR, "Derwing, Tracey and Rossiter, Marian and Munro, Murray") + .withField(StandardField.TITLE, "Teaching Native Speakers to Listen to Foreign-accented Speech") + .withField(StandardField.JOURNAL, "Journal of Multilingual and Multicultural Development") + .withField(StandardField.DOI, "10.1080/01434630208666468") + .withField(StandardField.DATE, "2002-09") + .withField(StandardField.YEAR, "2002") + .withField(StandardField.MONTH, "9") + .withField(StandardField.PAGES, "245-259") + .withField(StandardField.VOLUME, "23") + .withField(StandardField.PUBLISHER, "Informa UK Limited") + .withField(StandardField.NUMBER, "4"); + Optional response = grobidService.processCitation("Derwing, T. M., Rossiter, M. J., & Munro, " + "M. J. (2002). Teaching native speakers to listen to foreign-accented speech. " + - "Journal of Multilingual and Multicultural Development, 23(4), 245-259.", GrobidService.ConsolidateCitations.WITH_METADATA); - assertEquals(expectedResponse, response); + "Journal of Multilingual and Multicultural Development, 23(4), 245-259.", importFormatPreferences, GrobidService.ConsolidateCitations.WITH_METADATA); + assertTrue(response.isPresent()); + assertEquals(exampleBibEntry, response.get()); } @Test - public void processEmptyStringTest() throws IOException { - String response = grobidService.processCitation(" ", GrobidService.ConsolidateCitations.WITH_METADATA); + public void processEmptyStringTest() throws IOException, ParseException { + Optional response = grobidService.processCitation(" ", importFormatPreferences, GrobidService.ConsolidateCitations.WITH_METADATA); assertNotNull(response); - assertEquals("", response); + assertTrue(response.isEmpty()); } @Test public void processInvalidCitationTest() { - assertThrows(IOException.class, () -> grobidService.processCitation("iiiiiiiiiiiiiiiiiiiiiiii", GrobidService.ConsolidateCitations.WITH_METADATA)); + assertThrows(IOException.class, () -> grobidService.processCitation("iiiiiiiiiiiiiiiiiiiiiiii", importFormatPreferences, GrobidService.ConsolidateCitations.WITH_METADATA)); } @Test From 0e60ff44fd4179ce1c5216cf519f0b2d0dd43052 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Fri, 20 Aug 2021 10:45:06 +0200 Subject: [PATCH 4/6] Checkstyle --- .../jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java index f5e6a0c55c0..6303152440f 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java @@ -15,7 +15,6 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; -import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; From 83e3b119a6d1aa6fc45168043353275fe380ca98 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Fri, 20 Aug 2021 11:25:15 +0200 Subject: [PATCH 5/6] Use assertEquals --- .../java/org/jabref/logic/importer/util/GrobidServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java index 8c9c595c15f..105027ed367 100644 --- a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java +++ b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java @@ -64,7 +64,7 @@ public void processValidCitationTest() throws IOException, ParseException { public void processEmptyStringTest() throws IOException, ParseException { Optional response = grobidService.processCitation(" ", importFormatPreferences, GrobidService.ConsolidateCitations.WITH_METADATA); assertNotNull(response); - assertTrue(response.isEmpty()); + assertEquals(Optional.empty(), response); } @Test From ef68a21d91b3786c46d5da8a2dc34dee66ee0ab4 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Fri, 20 Aug 2021 11:53:39 +0200 Subject: [PATCH 6/6] Use correct junit API --- .../java/org/jabref/logic/importer/util/GrobidServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java index 105027ed367..7c7d4b1b0a8 100644 --- a/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java +++ b/src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java @@ -19,10 +19,10 @@ import org.junit.jupiter.api.Test; import org.mockito.Answers; -import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when;