import citavi knowledge items (JabRef#9033)

Co-authored-by: Oliver Kopp <kopp.dev@gmail.com>

.gitattributes

@@ -13,5 +13,9 @@ gradlew text eol=lf
 # .bib files have to be written using OS specific line endings to enable our tests working
 *.bib text !eol
 
+# Citavi needs to be LF line ending
+# This overwrites the setting of "*.bib"
+Citavi*.bib eol=lf
+
 # disable after a release (otherwise, duplicate CHANGELOG.md entries will be generated)
 # CHANGELOG.md merge=union

CHANGELOG.md

@@ -13,6 +13,8 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 
 ### Changed
 
+- We improved the Citavi Importer to also import so called Knowledge-items into the field `comment` of the corresponding entry [#9025](https://github.com/JabRef/jabref/issues/9025)
+- We removed wrapping of string constants when writing to a `.bib` file.
 - We changed the button label from "Return to JabRef" to "Return to library" to better indicate the purpose of the action.
 
 ### Fixed

src/main/java/org/jabref/logic/bibtex/FieldContentFormatter.java

@@ -5,10 +5,8 @@
 import java.util.Set;
 import java.util.regex.Pattern;
 
-import org.jabref.logic.util.OS;
 import org.jabref.model.entry.field.Field;
 import org.jabref.model.entry.field.StandardField;
-import org.jabref.model.strings.StringUtil;
 
 /**
  * This class provides the reformatting needed when reading BibTeX fields formatted
@@ -43,7 +41,10 @@ public FieldContentFormatter(FieldContentFormatterPreferences preferences) {
      */
     public String format(String fieldContent, Field field) {
         if (multiLineFields.contains(field)) {
-            return StringUtil.unifyLineBreaks(fieldContent, OS.NEWLINE);
+            // Keep the field as is.
+            // Newlines are normalized at org.jabref.logic.exporter.BibWriter
+            // Alternative: StringUtil.unifyLineBreaks(fieldContent, OS.NEWLINE)
+            return fieldContent;
         }
 
         return WHITESPACE.matcher(fieldContent).replaceAll(" ");

src/main/java/org/jabref/logic/bibtex/FieldWriter.java

@@ -1,8 +1,6 @@
 package org.jabref.logic.bibtex;
 
-import org.jabref.logic.util.OS;
 import org.jabref.model.entry.field.Field;
-import org.jabref.model.strings.StringUtil;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -204,7 +202,6 @@ private void writeText(StringBuilder stringBuilder, String text, int startPos, i
      */
     private void writeStringLabel(StringBuilder stringBuilder, String text, int startPos, int endPos, boolean isFirst, boolean isLast) {
         String line = (isFirst ? "" : " # ") + text.substring(startPos, endPos) + (isLast ? "" : " # ");
-        String wrappedLine = StringUtil.wrap(line, preferences.getLineLength(), OS.NEWLINE);
-        stringBuilder.append(wrappedLine);
+        stringBuilder.append(line);
     }
 }

src/main/java/org/jabref/logic/bibtex/FieldWriterPreferences.java

@@ -8,7 +8,6 @@ public class FieldWriterPreferences {
 
     private final boolean resolveStrings;
     private final List<Field> resolveStringsForFields;
-    private final int lineLength = 65; // Constant
     private final FieldContentFormatterPreferences fieldContentFormatterPreferences;
 
     /**
@@ -29,10 +28,6 @@ public List<Field> getResolveStringsForFields() {
         return resolveStringsForFields;
     }
 
-    public int getLineLength() {
-        return lineLength;
-    }
-
     public FieldContentFormatterPreferences getFieldContentFormatterPreferences() {
         return fieldContentFormatterPreferences;
     }

src/main/java/org/jabref/logic/cleanup/Cleanups.java

@@ -67,9 +67,8 @@ public static List<FieldFormatterCleanup> parse(String formatterString) {
         int startIndex = 0;
 
         // first remove all newlines for easier parsing
-        String remainingString = formatterString;
+        String remainingString = StringUtil.unifyLineBreaks(formatterString, "");
 
-        remainingString = StringUtil.unifyLineBreaks(remainingString, "");
         try {
             while (startIndex < formatterString.length()) {
                 // read the field name

src/main/java/org/jabref/logic/cleanup/FieldFormatterCleanup.java

@@ -56,9 +56,6 @@ private List<FieldChange> cleanupSingleField(Field fieldKey, BibEntry entry) {
 
         // Run formatter
         String newValue = formatter.format(oldValue);
-        if (formatter instanceof NormalizeNewlinesFormatter) {
-            newValue = oldValue;
-        }
 
         if (oldValue.equals(newValue)) {
             return Collections.emptyList();

src/main/java/org/jabref/logic/exporter/BibDatabaseWriter.java

@@ -28,7 +28,6 @@
 import org.jabref.logic.citationkeypattern.GlobalCitationKeyPattern;
 import org.jabref.logic.cleanup.FieldFormatterCleanup;
 import org.jabref.logic.cleanup.FieldFormatterCleanups;
-import org.jabref.logic.cleanup.NormalizeNewlinesFormatter;
 import org.jabref.logic.formatter.bibtexfields.TrimWhitespaceFormatter;
 import org.jabref.model.FieldChange;
 import org.jabref.model.database.BibDatabase;
@@ -79,9 +78,9 @@ private static List<FieldChange> applySaveActions(List<BibEntry> toChange, MetaD
             }
         });
 
-        // Run a couple of standard cleanups
+        // Run standard cleanups
         List<FieldFormatterCleanup> preSaveCleanups =
-                Stream.of(new TrimWhitespaceFormatter(), new NormalizeNewlinesFormatter())
+                Stream.of(new TrimWhitespaceFormatter())
                       .map(formatter -> new FieldFormatterCleanup(InternalField.INTERNAL_ALL_FIELD, formatter))
                       .collect(Collectors.toList());
         for (FieldFormatterCleanup formatter : preSaveCleanups) {

src/main/java/org/jabref/logic/formatter/bibtexfields/TrimWhitespaceFormatter.java

@@ -6,7 +6,7 @@
 import org.jabref.logic.l10n.Localization;
 
 /**
- * Trim all whitespace characters(defined in java) in the string.
+ * Trim all whitespace characters (as defined in Java) in the beginning and at the end of the string.
  */
 public class TrimWhitespaceFormatter extends Formatter {
 

src/main/java/org/jabref/logic/importer/fileformat/BibtexParser.java

@@ -634,7 +634,7 @@ private String parseFieldContent(Field field) throws IOException {
                 value.append(fieldContentFormatter.format(text, field));
             } else if (character == '{') {
                 // Value is a string enclosed in brackets. There can be pairs
-                // of brackets inside of a field, so we need to count the
+                // of brackets inside a field, so we need to count the
                 // brackets to know when the string is finished.
                 StringBuilder text = parseBracketedTextExactly();
                 value.append(fieldContentFormatter.format(text, field));

src/main/java/org/jabref/logic/importer/fileformat/CitaviXmlImporter.java

@@ -33,6 +33,8 @@
 import org.jabref.logic.importer.Parser;
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.importer.fileformat.citavi.CitaviExchangeData;
+import org.jabref.logic.importer.fileformat.citavi.CitaviExchangeData.KnowledgeItems;
+import org.jabref.logic.importer.fileformat.citavi.CitaviExchangeData.KnowledgeItems.KnowledgeItem;
 import org.jabref.logic.importer.fileformat.citavi.CitaviExchangeData.Persons.Person;
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.Author;
@@ -71,6 +73,7 @@ public class CitaviXmlImporter extends Importer implements Parser {
     private CitaviExchangeData.Persons persons;
     private CitaviExchangeData.Keywords keywords;
     private CitaviExchangeData.Publishers publishers;
+    private KnowledgeItems knowledgeItems;
 
     private CitaviExchangeData.ReferenceAuthors refAuthors;
     private CitaviExchangeData.ReferenceEditors refEditors;
@@ -114,7 +117,6 @@ public boolean isRecognizedFormat(Path filePath) throws IOException {
                 if (str.toLowerCase(Locale.ROOT).contains("citaviexchangedata")) {
                     return true;
                 }
-
                 i++;
             }
         }
@@ -123,12 +125,11 @@ public boolean isRecognizedFormat(Path filePath) throws IOException {
 
     @Override
     public ParserResult importDatabase(Path filePath) throws IOException {
-
         try (BufferedReader reader = getReaderFromZip(filePath)) {
             Object unmarshalledObject = unmarshallRoot(reader);
 
             if (unmarshalledObject instanceof CitaviExchangeData) {
-                // Check whether we have an article set, an article, a book article or a book article set
+                // Check whether we have an article set, an article, a book article, or a book article set
                 CitaviExchangeData data = (CitaviExchangeData) unmarshalledObject;
                 List<BibEntry> bibEntries = parseDataList(data);
 
@@ -148,6 +149,7 @@ private List<BibEntry> parseDataList(CitaviExchangeData data) {
         persons = data.getPersons();
         keywords = data.getKeywords();
         publishers = data.getPublishers();
+        knowledgeItems = data.getKnowledgeItems();
 
         refAuthors = data.getReferenceAuthors();
         refEditors = data.getReferenceEditors();
@@ -199,14 +201,15 @@ private BibEntry parseData(CitaviExchangeData.References.Reference data) {
         Optional.ofNullable(data.getVolume())
                 .ifPresent(value -> entry.setField(StandardField.VOLUME, clean(value)));
         Optional.ofNullable(getAuthorName(data))
-                .ifPresent(value -> entry.setField(StandardField.AUTHOR, value));
+                .ifPresent(value -> entry.setField(StandardField.AUTHOR, clean(value)));
         Optional.ofNullable(getEditorName(data))
-                .ifPresent(value -> entry.setField(StandardField.EDITOR, value));
+                .ifPresent(value -> entry.setField(StandardField.EDITOR, clean(value)));
         Optional.ofNullable(getKeywords(data))
-                .ifPresent(value -> entry.setField(StandardField.KEYWORDS, value));
+                .ifPresent(value -> entry.setField(StandardField.KEYWORDS, clean(value)));
         Optional.ofNullable(getPublisher(data))
-                .ifPresent(value -> entry.setField(StandardField.PUBLISHER, value));
-
+                .ifPresent(value -> entry.setField(StandardField.PUBLISHER, clean(value)));
+        Optional.ofNullable(getKnowledgeItem(data))
+                .ifPresent(value -> entry.setField(StandardField.COMMENT, StringUtil.unifyLineBreaks(value, "\n")));
         return entry;
     }
 
@@ -359,6 +362,20 @@ private String getPublisher(CitaviExchangeData.References.Reference data) {
         return this.refIdWithPublishers.get(data.getId());
     }
 
+    private String getKnowledgeItem(CitaviExchangeData.References.Reference data) {
+        Optional<KnowledgeItem> knowledgeItem = knowledgeItems.getKnowledgeItem().stream().filter(p -> data.getId().equals(p.getReferenceID())).findFirst();
+
+        StringBuilder comment = new StringBuilder();
+        Optional<String> title = knowledgeItem.map(item -> item.getCoreStatement());
+        title.ifPresent(t -> comment.append("# ").append(t).append("\n\n"));
+        Optional<String> text = knowledgeItem.map(item -> item.getText());
+        text.ifPresent(t -> comment.append(t).append("\n\n"));
+        Optional<Integer> pages = knowledgeItem.map(item -> item.getPageRangeNumber()).filter(range -> range != -1);
+        pages.ifPresent(p -> comment.append("page range: ").append(p));
+
+        return comment.toString();
+    }
+
     private void initUnmarshaller() throws JAXBException {
         if (unmarshaller == null) {
             // Lazy init because this is expensive

src/main/java/org/jabref/model/entry/CanonicalBibEntry.java

@@ -17,12 +17,16 @@ private CanonicalBibEntry() {
     }
 
     /**
-     * This returns a canonical BibTeX serialization. Serializes all fields, even the JabRef internal ones. Does NOT
-     * serialize "KEY_FIELD" as field, but as key
+     * This returns a canonical BibTeX serialization.
+     * The result is close to BibTeX, but not a valid BibTeX representation in all cases
      *
      * <ul>
-     *     <li>Special characters such as "{" or "&" are NOT escaped, but written as</li>
-     *     <li>String constants are not handled. That means, <code>month = apr</code> in a bib file gets <code>month = {#apr#}</code>. This indicates that the month field is correctly stored</li>
+     *     <li>Serializes all fields, even the JabRef internal ones.</li>
+     *     <li>Does NOT serialize "KEY_FIELD" as field, but as key.</li>
+     *     <li>Special characters such as "{" or "&" are NOT escaped, but written as is.</li>
+     *     <li>New lines are written as is.</li>
+     *     <li>String constants are not handled. That means, <code>month = apr</code> in a bib file gets <code>month = {#apr#}</code>.
+     *         This indicates that the month field is correctly stored.</li>
      * </ul>
      */
     public static String getCanonicalRepresentation(BibEntry entry) {
@@ -53,7 +57,7 @@ public static String getCanonicalRepresentation(BibEntry entry) {
         // generate field entries
         StringJoiner sj = new StringJoiner(",\n", "", "\n");
         for (String fieldName : sortedFields) {
-            String line = String.format("  %s = {%s}", fieldName, String.valueOf(mapFieldToValue.get(fieldName)).replaceAll("\\r\\n", "\n"));
+            String line = String.format("  %s = {%s}", fieldName, mapFieldToValue.get(fieldName));
             sj.add(line);
         }
 

src/main/resources/l10n/JabRef_en.properties

@@ -2078,9 +2078,6 @@ Dismiss=Dismiss
 Mark\ all\ changes\ as\ accepted=Mark all changes as accepted
 Unmark\ all\ changes=Unmark all changes
 
-Normalize\ newline\ characters=Normalize newline characters
-Normalizes\ all\ newline\ characters\ in\ the\ field\ content.=Normalizes all newline characters in the field content.
-
 Independent=Independent
 Intersection=Intersection
 Union=Union