Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DOI matching in duplicate check #6897

Merged
merged 14 commits into from
Sep 29, 2020
Merged

DOI matching in duplicate check #6897

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
6c8b7d7
Add DOI matching with not full coincidence.
KunAndrew Aug 8, 2020
25e34f0
issue 6707
KunAndrew Aug 11, 2020
8e4714b
issue 6707
KunAndrew Aug 17, 2020
b9f412c
issue 6707
KunAndrew Sep 8, 2020
8a02474
issue 6707
KunAndrew Sep 9, 2020
304b99f
Merge branch 'fix-for-issue-6707' of https://github.com/KunAndrew/jab…
Siedlerchr Sep 9, 2020
3fc1ffa
remove unnecessary cast
Siedlerchr Sep 9, 2020
d8c08a2
Update src/main/java/org/jabref/logic/database/DuplicateCheck.java
Siedlerchr Sep 26, 2020
0aee547
Update src/main/java/org/jabref/logic/database/DuplicateCheck.java
Siedlerchr Sep 26, 2020
074856d
Update src/main/java/org/jabref/model/entry/identifier/DOI.java
Siedlerchr Sep 26, 2020
1102851
Merge remote-tracking branch 'upstream/master' into KunAndrew-fix-for…
Siedlerchr Sep 26, 2020
0fa5691
simplify code
Siedlerchr Sep 26, 2020
298ec85
add changelog entry
Siedlerchr Sep 26, 2020
3920cd3
Merge branch 'master' into KunAndrew-fix-for-issue-6707
Siedlerchr Sep 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions src/main/java/org/jabref/logic/database/DuplicateCheck.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import org.jabref.model.entry.field.FieldProperty;
import org.jabref.model.entry.field.OrFields;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.entry.identifier.DOI;
import org.jabref.model.entry.identifier.ISBN;

import com.google.common.collect.Sets;
import org.slf4j.Logger;
Expand Down Expand Up @@ -57,6 +59,9 @@ public class DuplicateCheck {
DuplicateCheck.FIELD_WEIGHTS.put(StandardField.EDITOR, 2.5);
DuplicateCheck.FIELD_WEIGHTS.put(StandardField.TITLE, 3.);
DuplicateCheck.FIELD_WEIGHTS.put(StandardField.JOURNAL, 2.);
DuplicateCheck.FIELD_WEIGHTS.put(StandardField.NOTE, 0.1);
DuplicateCheck.FIELD_WEIGHTS.put(StandardField.COMMENT, 0.1);
DuplicateCheck.FIELD_WEIGHTS.put(StandardField.DOI, 3.);
}

private final BibEntryTypesManager entryTypesManager;
Expand Down Expand Up @@ -303,6 +308,19 @@ public boolean isDuplicate(final BibEntry one, final BibEntry two, final BibData
return true;
}

// check DOI
Optional<DOI> oneDOI = one.getDOI();
Optional<DOI> twoDOI = two.getDOI();
if (oneDOI.isPresent() && twoDOI.isPresent()) {
return oneDOI.get().getDOI().equalsIgnoreCase(twoDOI.get().getDOI());
}
// check ISBN
Optional<ISBN> oneISBN = one.getISBN();
Optional<ISBN> twoISBN = two.getISBN();
if (oneISBN.isPresent() && twoISBN.isPresent()) {
return oneISBN.get().equals(twoISBN.get());
}

if (haveDifferentEntryType(one, two) ||
haveDifferentEditions(one, two) ||
haveDifferentChaptersOrPagesOfTheSameBook(one, two)) {
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/org/jabref/model/entry/BibEntry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.jabref.model.entry.field.OrFields;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.entry.identifier.DOI;
import org.jabref.model.entry.identifier.ISBN;
import org.jabref.model.entry.types.EntryType;
import org.jabref.model.entry.types.IEEETranEntryType;
import org.jabref.model.entry.types.StandardEntryType;
Expand Down Expand Up @@ -478,6 +479,10 @@ public Optional<DOI> getDOI() {
return getField(StandardField.DOI).flatMap(DOI::parse);
}

public Optional<ISBN> getISBN() {
return getField(StandardField.ISBN).flatMap(ISBN::parse);
}

/**
* Return the LaTeX-free contents of the given field or its alias an an Optional
* <p>
Expand Down
5 changes: 2 additions & 3 deletions src/main/java/org/jabref/model/entry/identifier/DOI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,12 @@ public DOI(String doi) {
Matcher matcher = EXACT_DOI_PATT.matcher(trimmedDoi);
if (matcher.find()) {
// match only group \1
this.doi = matcher.group(1);
this.doi = matcher.group(1).replaceAll("[^\\w,/,:,-,.,-]|[_]", "");
Copy link
Member Author

@Siedlerchr Siedlerchr Sep 10, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@KunAndrew I think you need to adjust the DOI regex pattern:
One CrossRef test fails, because of the missing underscore:
The DOI cannot be resolved wihtout underscore
org.opentest4j.AssertionFailedError: expected: <10.1007/11538394_20> but was: <10.1007/1153839420>

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There should be a comment added which characters are removed and why. Without any comment, this is a "magic number" which should be extraced to a constant.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed this addtional regex because it was not really making sense. The DOI class already trims DOIs and also removes other encoding components.

} else {
// Short DOI
Matcher shortDoiMatcher = EXACT_SHORT_DOI_PATT.matcher(trimmedDoi);
if (shortDoiMatcher.find()) {
this.doi = shortDoiMatcher.group(1);
this.doi = shortDoiMatcher.group(1).replaceAll("[^\\w,/,:,-,.,-]|[_]", "");
isShortDoi = true;
} else {
throw new IllegalArgumentException(trimmedDoi + " is not a valid DOI/Short DOI.");
Expand Down Expand Up @@ -253,5 +253,4 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hash(doi.toLowerCase(Locale.ENGLISH));
}

}
18 changes: 18 additions & 0 deletions src/main/java/org/jabref/model/entry/identifier/ISBN.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.net.URI;
import java.net.URISyntaxException;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Matcher;
Expand Down Expand Up @@ -106,4 +107,21 @@ public Optional<URI> getExternalURI() {
return Optional.empty();
}
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if ((o == null) || (getClass() != o.getClass())) {
return false;
}
ISBN other = (ISBN) o;
return isbnString.equalsIgnoreCase(other.isbnString);
}

@Override
public int hashCode() {
return Objects.hash(isbnString.toLowerCase(Locale.ENGLISH));
}
}
30 changes: 30 additions & 0 deletions src/test/java/org/jabref/logic/database/DuplicateCheckTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,36 @@ public void twoEntriesWithSameDoiButDifferentTypesAreDuplicates() {
assertTrue(duplicateChecker.isDuplicate(simpleArticle, duplicateWithDifferentType, BibDatabaseMode.BIBTEX));
}

@Test
public void twoEntriesWithSameDoiButDifferentTypesAreDuplicates2() {
simpleArticle.setField(StandardField.DOI, "10.1016/j.is.2004.02.002");
unrelatedArticle.setField(StandardField.DOI, "10.1016/j.is.2004.02.0\\02");
BibEntry duplicateWithDifferentType = unrelatedArticle;
duplicateWithDifferentType.setType(StandardEntryType.InCollection);

assertTrue(duplicateChecker.isDuplicate(simpleArticle, duplicateWithDifferentType, BibDatabaseMode.BIBTEX));
}

@Test
public void twoEntriesWithSameDoiButDifferentTypesAreDuplicates3() {
simpleArticle.setField(StandardField.DOI, "10.1016/j.is.2004.02.002");
unrelatedArticle.setField(StandardField.DOI, "10.1016/j.is.2004.02.0_02");
BibEntry duplicateWithDifferentType = unrelatedArticle;
duplicateWithDifferentType.setType(StandardEntryType.InCollection);

assertTrue(duplicateChecker.isDuplicate(simpleArticle, duplicateWithDifferentType, BibDatabaseMode.BIBTEX));
}

@Test
public void twoEntriesWithSameISBNButDifferentTypesAreDuplicates() {
simpleArticle.setField(StandardField.ISBN, "0-123456-47-9");
unrelatedArticle.setField(StandardField.ISBN, "0-123456-47-9");
BibEntry duplicateWithDifferentType = unrelatedArticle;
duplicateWithDifferentType.setType(StandardEntryType.InCollection);

assertTrue(duplicateChecker.isDuplicate(simpleArticle, duplicateWithDifferentType, BibDatabaseMode.BIBTEX));
}

@Test
public void twoInbooksWithDifferentChaptersAreNotDuplicates() {
twoEntriesWithDifferentSpecificFieldsAreNotDuplicates(simpleInbook, StandardField.CHAPTER,
Expand Down