Merge remote-tracking branch 'upstream/master' into writebackupToTemp

* upstream/master:
  Update run configuration for IDE (#6854)
  Use personal token for commenting on GitHub PRs (otherwise the action fails)
  Add chenyuheng
  Find for unlinked local files: redefine keyboard shorcut (#6850)
  "Find unlinked files": rename window (#6849)
  "Find unlinked files": removal on entry in en l10 (#6852)
  Fix/use format conversion for all imports (#6833)
  FIx unit tests in master (#6845)

.github/workflows/tests.yml

@@ -257,7 +257,7 @@ jobs:
         uses: unsplash/comment-on-pr@master
         if: steps.authors_check.outputs.newauthor == 'true'
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN_COMMENT_ON_PR }}
         with:
           msg: "${{ steps.authors_check.outputs.message }}"
           check_for_duplicate_msg: true

docs/getting-into-the-code/code-howtos.md

@@ -30,6 +30,7 @@ Principles:
                Localization.lang("Something went wrong...", ioe);
        }
     ```
+
 * Never, ever throw and catch `Exception` or `Throwable`
 * Errors should only be logged when they are finally caught \(i.e., logged only once\). See **Logging** for details.
 * If the Exception message is intended to be shown to the User in the UI \(see below\) provide also a localizedMessage \(see `JabRefException`\).
@@ -51,7 +52,7 @@ TODO: Usage of status bar and Swing Dialogs
 
 ## Using the EventSystem
 
-### What the EventSystem is used for?
+### What the EventSystem is used for
 
 Many times there is a need to provide an object on many locations simultaneously. This design pattern is quite similar to Java's Observer, but it is much simplier and readable while having the same functional sense.
 
@@ -368,7 +369,7 @@ The following expressions can be used in FXML attributes, according to the [offi
 
 All radio buttons that should be grouped together need to have a ToggleGroup defined in the FXML code Example:
 
-```markup
+```xml
 <VBox>
             <fx:define>
                 <ToggleGroup fx:id="citeToggleGroup"/>
@@ -383,4 +384,3 @@ All radio buttons that should be grouped together need to have a ToggleGroup def
             </children>
 </VBox>
 ```
-

docs/getting-into-the-code/guidelines-for-setting-up-a-local-workspace.md

@@ -222,6 +222,7 @@ Make sure your Eclipse installation us up to date, Eclipse JEE 2020-03 or newer
         --add-exports javafx.web/com.sun.webkit=org.controlsfx.controls
         --add-exports javafx.graphics/com.sun.javafx.css=org.controlsfx.controls
         --add-exports javafx.controls/com.sun.javafx.scene.control.behavior=com.jfoenix
+        --add-exports javafx.graphics/com.sun.javafx.stage=com.jfoenix
         --add-exports com.oracle.truffle.regex/com.oracle.truffle.regex=org.graalvm.truffle
         --patch-module org.jabref=build/resources/main
      ```

scripts/generate-authors.sh

@@ -58,5 +58,5 @@ cd "$(dirname "$(readlink -f "$BASH_SOURCE")")/.."
 
     # co-authors
     coauthors=$(git log -i --grep=co-authored-by  | grep -i "co-authored-by" | sed "s/.*co-authored-by: \(.*\)/\1/I" | sed "s/ <.*//")
-	echo -e "$authors\n$(git log --format='%aN')\n$coauthors" | grep -v "\[bot\]" | grep -v "JabRef" | grep -v "Siedlerchr" | grep -v "^Christoph$" | grep -v "^Mootez$" | grep -v "oscargus" | grep -v "dependabot" | grep -v "github actions" | grep -v "igorsteinmacher" | grep -v "halirutan" | grep -v "matthiasgeiger" | grep -v "Gitter Badger" | grep -v "gdstewart" | LC_ALL=C.UTF-8 sort --unique --ignore-case
+	echo -e "$authors\n$(git log --format='%aN')\n$coauthors" | grep -v "\[bot\]" | grep -v "JabRef" | grep -v "Siedlerchr" | grep -v "^Christoph$" | grep -v "^Mootez$" | grep -v "oscargus" | grep -v "dependabot" | grep -v "github actions" | grep -v "igorsteinmacher" | grep -v "halirutan" | grep -v "matthiasgeiger" | grep -v "Gitter Badger" | grep -v "gdstewart" | grep -v "chenyuheng" | LC_ALL=C.UTF-8 sort --unique --ignore-case
 } > AUTHORS

src/main/java/org/jabref/gui/BasePanel.java

@@ -614,7 +614,7 @@ public void copy() {
     }
 
     public void paste() {
-        mainTable.paste();
+        mainTable.paste(this.bibDatabaseContext.getMode());
     }
 
     public void cut() {

src/main/java/org/jabref/gui/EntryTypeViewModel.java

@@ -20,6 +20,7 @@
 import org.jabref.logic.database.DuplicateCheck;
 import org.jabref.logic.importer.FetcherException;
 import org.jabref.logic.importer.IdBasedFetcher;
+import org.jabref.logic.importer.ImportCleanup;
 import org.jabref.logic.importer.WebFetchers;
 import org.jabref.logic.importer.fetcher.DoiFetcher;
 import org.jabref.logic.l10n.Localization;
@@ -147,6 +148,8 @@ public void runFetcherWorker() {
             Optional<BibEntry> result = fetcherWorker.getValue();
             if (result.isPresent()) {
                 final BibEntry entry = result.get();
+                ImportCleanup cleanup = new ImportCleanup(basePanel.getBibDatabaseContext().getMode());
+                cleanup.doPostCleanup(entry);
                 Optional<BibEntry> duplicate = new DuplicateCheck(Globals.entryTypesManager).containsDuplicate(basePanel.getDatabase(), entry, basePanel.getBibDatabaseContext().getMode());
                 if ((duplicate.isPresent())) {
                     DuplicateResolverDialog dialog = new DuplicateResolverDialog(entry, duplicate.get(), DuplicateResolverDialog.DuplicateResolverType.IMPORT_CHECK, basePanel.getBibDatabaseContext(), stateManager);

src/main/java/org/jabref/gui/JabRefFrame.java

@@ -125,6 +125,7 @@
 import org.jabref.logic.autosaveandbackup.BackupManager;
 import org.jabref.logic.citationstyle.CitationStyleOutputFormat;
 import org.jabref.logic.importer.IdFetcher;
+import org.jabref.logic.importer.ImportCleanup;
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.importer.WebFetchers;
 import org.jabref.logic.l10n.Localization;
@@ -1189,6 +1190,8 @@ private boolean readyForAutosave(BibDatabaseContext context) {
      */
     private void addImportedEntries(final BasePanel panel, final ParserResult parserResult) {
         BackgroundTask<ParserResult> task = BackgroundTask.wrap(() -> parserResult);
+        ImportCleanup cleanup = new ImportCleanup(panel.getBibDatabaseContext().getMode());
+        cleanup.doPostCleanup(parserResult.getDatabase().getEntries());
         ImportEntriesDialog dialog = new ImportEntriesDialog(panel.getBibDatabaseContext(), task);
         dialog.setTitle(Localization.lang("Import"));
         dialog.showAndWait();

src/main/java/org/jabref/gui/entryeditor/RelatedArticlesTab.java

@@ -22,8 +22,11 @@
 import org.jabref.gui.DialogService;
 import org.jabref.gui.desktop.JabRefDesktop;
 import org.jabref.gui.util.BackgroundTask;
+import org.jabref.logic.importer.ImportCleanup;
 import org.jabref.logic.importer.fetcher.MrDLibFetcher;
 import org.jabref.logic.l10n.Localization;
+import org.jabref.model.database.BibDatabase;
+import org.jabref.model.database.BibDatabaseModeDetection;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.preferences.JabRefPreferences;
@@ -67,6 +70,8 @@ private StackPane getRelatedArticlesPane(BibEntry entry) {
                 .wrap(() -> fetcher.performSearch(entry))
                 .onRunning(() -> progress.setVisible(true))
                 .onSuccess(relatedArticles -> {
+                    ImportCleanup cleanup = new ImportCleanup(BibDatabaseModeDetection.inferMode(new BibDatabase(List.of(entry))));
+                    cleanup.doPostCleanup(relatedArticles);
                     progress.setVisible(false);
                     root.getChildren().add(getRelatedArticleInfo(relatedArticles, fetcher));
                 })

src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesDialog.java

@@ -57,7 +57,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * GUI Dialog for the feature "Find unlinked files".
+ * GUI Dialog for the feature "Search for unlinked local files".
  */
 public class FindUnlinkedFilesDialog extends BaseDialog<Boolean> {
 
@@ -77,7 +77,7 @@ public class FindUnlinkedFilesDialog extends BaseDialog<Boolean> {
 
     public FindUnlinkedFilesDialog(BibDatabaseContext database, DialogService dialogService, CountingUndoManager undoManager) {
         super();
-        this.setTitle(Localization.lang("Find unlinked files"));
+        this.setTitle(Localization.lang("Search for unlinked local files"));
         this.dialogService = dialogService;
 
         databaseContext = database;

src/main/java/org/jabref/gui/externalfiles/ImportHandler.java

@@ -15,6 +15,7 @@
 import org.jabref.gui.undo.UndoableInsertEntries;
 import org.jabref.logic.citationkeypattern.CitationKeyGenerator;
 import org.jabref.logic.externalfiles.ExternalFilesContentImporter;
+import org.jabref.logic.importer.ImportCleanup;
 import org.jabref.logic.util.UpdateField;
 import org.jabref.logic.util.io.FileUtil;
 import org.jabref.model.FieldChange;
@@ -107,7 +108,8 @@ private BibEntry createEmptyEntryWithLink(Path file) {
     public void importEntries(List<BibEntry> entries) {
         // TODO: Add undo/redo
         // undoManager.addEdit(new UndoableInsertEntries(panel.getDatabase(), entries));
-
+        ImportCleanup cleanup = new ImportCleanup(database.getMode());
+        cleanup.doPostCleanup(entries);
         database.getDatabase().insertEntries(entries);
 
         // Set owner/timestamp

src/main/java/org/jabref/gui/journals/ManageJournalAbbreviationsViewModel.java

@@ -20,7 +20,6 @@
 import org.jabref.gui.util.FileDialogConfiguration;
 import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.journals.Abbreviation;
-import org.jabref.logic.journals.JournalAbbreviationLoader;
 import org.jabref.logic.journals.JournalAbbreviationPreferences;
 import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.logic.l10n.Localization;
@@ -109,7 +108,7 @@ public SimpleBooleanProperty isLoadingProperty() {
      */
     void addBuiltInList() {
         BackgroundTask
-                .wrap(JournalAbbreviationLoader::getBuiltInAbbreviations)
+                .wrap(journalAbbreviationRepository::getAllLoaded)
                 .onRunning(() -> isLoading.setValue(true))
                 .onSuccess(result -> {
                     isLoading.setValue(false);
@@ -271,10 +270,6 @@ private void setCurrentAbbreviationNameAndAbbreviationIfValid(String name, Strin
         shouldWriteLists = true;
     }
 
-    private void setCurrentAbbreviationNameAndAbbreviationIfValid(String name, String abbreviation) {
-        setCurrentAbbreviationNameAndAbbreviationIfValid(name, abbreviation, "");
-    }
-
     /**
      * Method to delete the abbreviation set in the currentAbbreviation property. The currentAbbreviationProperty will
      * be set to the previous or next abbreviation in the abbreviations property if applicable. Else it will be set to
@@ -336,7 +331,7 @@ public void selectLastJournalFile() {
     /**
      * This method first saves all external files to its internal list, then writes all abbreviations to their files and
      * finally updates the abbreviations auto complete. It basically calls {@link #saveExternalFilesList()}, {@link
-     * #saveJournalAbbreviationFiles() } and finally {@link JournalAbbreviationLoader#update(JournalAbbreviationPreferences)}.
+     * #saveJournalAbbreviationFiles() }}.
      */
     public void save() {
         BackgroundTask.wrap(() -> {

src/main/java/org/jabref/gui/keyboard/KeyBinding.java

@@ -37,7 +37,7 @@ public enum KeyBinding {
     ENTRY_EDITOR_PREVIOUS_PANEL("Entry editor, previous panel", Localization.lang("Entry editor, previous panel"), "ctrl+shift+TAB", KeyBindingCategory.VIEW),
     FILE_LIST_EDITOR_MOVE_ENTRY_DOWN("File list editor, move entry down", Localization.lang("File list editor, move entry down"), "ctrl+DOWN", KeyBindingCategory.VIEW),
     FILE_LIST_EDITOR_MOVE_ENTRY_UP("File list editor, move entry up", Localization.lang("File list editor, move entry up"), "ctrl+UP", KeyBindingCategory.VIEW),
-    FIND_UNLINKED_FILES("Find unlinked files", Localization.lang("Find unlinked files"), "shift+F7", KeyBindingCategory.QUALITY),
+    FIND_UNLINKED_FILES("Search for unlinked local files", Localization.lang("Search for unlinked local files"), "shift+F7", KeyBindingCategory.QUALITY),
     FOCUS_ENTRY_TABLE("Focus entry table", Localization.lang("Focus entry table"), "alt+1", KeyBindingCategory.VIEW),
     HELP("Help", Localization.lang("Help"), "F1", KeyBindingCategory.FILE),
     IMPORT_INTO_CURRENT_DATABASE("Import into current library", Localization.lang("Import into current library"), "ctrl+I", KeyBindingCategory.FILE),

src/main/java/org/jabref/gui/maintable/MainTable.java

@@ -40,9 +40,11 @@
 import org.jabref.gui.util.CustomLocalDragboard;
 import org.jabref.gui.util.DefaultTaskExecutor;
 import org.jabref.gui.util.ViewModelTableRowFactory;
+import org.jabref.logic.importer.ImportCleanup;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.util.OS;
 import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.database.BibDatabaseMode;
 import org.jabref.model.database.event.EntriesAddedEvent;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.preferences.PreferencesService;
@@ -157,7 +159,7 @@ public MainTable(MainTableDataModel model,
      * The {@link MainTable} will scroll to the cell with the same starting column value and typed string
      *
      * @param sortedColumn The sorted column in {@link MainTable}
-     * @param keyEvent The pressed character
+     * @param keyEvent     The pressed character
      */
 
     private void jumpToSearchKey(TableColumn<BibEntryTableViewModel, ?> sortedColumn, KeyEvent keyEvent) {
@@ -271,14 +273,15 @@ private void clearAndSelectLast() {
         scrollTo(getItems().size() - 1);
     }
 
-    public void paste() {
+    public void paste(BibDatabaseMode bibDatabaseMode) {
         // Find entries in clipboard
         List<BibEntry> entriesToAdd = Globals.clipboardManager.extractData();
+        ImportCleanup cleanup = new ImportCleanup(bibDatabaseMode);
+        cleanup.doPostCleanup(entriesToAdd);
         panel.insertEntries(entriesToAdd);
         if (!entriesToAdd.isEmpty()) {
             this.requestFocus();
         }
-
     }
 
     private void handleOnDragOver(TableRow<BibEntryTableViewModel> row, BibEntryTableViewModel item, DragEvent event) {

src/main/java/org/jabref/gui/mergeentries/FetchAndMergeEntry.java

@@ -18,6 +18,7 @@
 import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.importer.EntryBasedFetcher;
 import org.jabref.logic.importer.IdBasedFetcher;
+import org.jabref.logic.importer.ImportCleanup;
 import org.jabref.logic.importer.WebFetcher;
 import org.jabref.logic.importer.WebFetchers;
 import org.jabref.logic.l10n.Localization;
@@ -64,6 +65,8 @@ public void fetchAndMerge(BibEntry entry, List<Field> fields) {
                 if (fetcher.isPresent()) {
                     BackgroundTask.wrap(() -> fetcher.get().performSearchById(fieldContent.get()))
                                   .onSuccess(fetchedEntry -> {
+                                      ImportCleanup cleanup = new ImportCleanup(panel.getBibDatabaseContext().getMode());
+                                      cleanup.doPostCleanup(entry);
                                       String type = field.getDisplayName();
                                       if (fetchedEntry.isPresent()) {
                                           showMergeDialog(entry, fetchedEntry.get(), fetcher.get());
@@ -147,6 +150,8 @@ public void fetchAndMerge(BibEntry entry, EntryBasedFetcher fetcher) {
         BackgroundTask.wrap(() -> fetcher.performSearch(entry).stream().findFirst())
                       .onSuccess(fetchedEntry -> {
                           if (fetchedEntry.isPresent()) {
+                              ImportCleanup cleanup = new ImportCleanup(panel.getBibDatabaseContext().getMode());
+                              cleanup.doPostCleanup(fetchedEntry.get());
                               showMergeDialog(entry, fetchedEntry.get(), fetcher);
                           } else {
                               dialogService.notify(Localization.lang("Could not find any bibliographic information."));

src/main/java/org/jabref/logic/journals/JournalAbbreviationLoader.java

@@ -1,13 +1,11 @@
 package org.jabref.logic.journals;
 
 import java.io.IOException;
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -16,32 +14,13 @@ public class JournalAbbreviationLoader {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(JournalAbbreviationLoader.class);
 
-    private static final String JOURNALS_FILE_BUILTIN = "/journals/journalList.csv";
-
-    public static List<Abbreviation> getBuiltInAbbreviations() {
-        return readJournalListFromResource(JOURNALS_FILE_BUILTIN);
-    }
-
-    private static List<Abbreviation> readJournalListFromResource(String resource) {
-        AbbreviationParser parser = new AbbreviationParser();
-        parser.readJournalListFromResource(Objects.requireNonNull(resource));
-        return parser.getAbbreviations();
-    }
-
     public static List<Abbreviation> readJournalListFromFile(Path file) throws IOException {
         LOGGER.debug(String.format("Reading journal list from file %s", file));
         AbbreviationParser parser = new AbbreviationParser();
         parser.readJournalListFromFile(file);
         return parser.getAbbreviations();
     }
 
-    private static List<Abbreviation> readJournalListFromFile(Path file, Charset encoding) throws IOException {
-        LOGGER.debug(String.format("Reading journal list from file %s", file));
-        AbbreviationParser parser = new AbbreviationParser();
-        parser.readJournalListFromFile(file, Objects.requireNonNull(encoding));
-        return parser.getAbbreviations();
-    }
-
     public static JournalAbbreviationRepository loadRepository(JournalAbbreviationPreferences journalAbbreviationPreferences) {
         JournalAbbreviationRepository repository;
         // Initialize with built-in list

src/main/java/org/jabref/logic/journals/JournalAbbreviationRepository.java

@@ -7,6 +7,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.h2.mvstore.MVMap;
 import org.h2.mvstore.MVStore;
@@ -65,8 +66,7 @@ public boolean isAbbreviatedName(String journalName) {
         String journal = journalName.trim();
 
         return customAbbreviations.stream().anyMatch(abbreviation -> isMatchedAbbreviated(journal, abbreviation))
-                ||
-                abbreviationToFull.containsKey(journal);
+                || abbreviationToFull.containsKey(journal);
     }
 
     /**
@@ -125,4 +125,9 @@ public Optional<String> getShortestUniqueAbbreviation(String text) {
     public Set<String> getFullNames() {
         return fullToAbbreviation.keySet();
     }
+
+    public List<Abbreviation> getAllLoaded() {
+        return fullToAbbreviation.entrySet().stream().map(entry ->
+                new Abbreviation(entry.getKey(), entry.getValue())).collect(Collectors.toList());
+    }
 }

src/main/resources/l10n/JabRef_en.properties

@@ -1128,7 +1128,6 @@ Resetting\ preference\ key\ '%0'=Resetting preference key '%0'
 Unknown\ preference\ key\ '%0'=Unknown preference key '%0'
 Unable\ to\ clear\ preferences.=Unable to clear preferences.
 
-Find\ unlinked\ files=Find unlinked files
 Unselect\ all=Unselect all
 Expand\ all=Expand all
 Collapse\ all=Collapse all
@@ -2267,3 +2266,8 @@ Removes\ digits.=Removes digits.
 The\ query\ cannot\ contain\ a\ year\ and\ year-range\ field.=The query cannot contain a year and year-range field.
 This\ query\ uses\ unsupported\ fields.=This query uses unsupported fields.
 This\ query\ uses\ unsupported\ syntax.=This query uses unsupported syntax.
+
+Check\ Proxy\ Setting=Check Proxy Setting
+Check\ connection=Check connection
+Connection\ failed\!=Connection failed\!
+Connection\ successful\!=Connection successful\!

src/test/java/org/jabref/gui/externalfiles/AutoSetFileLinksUtilTest.java

@@ -27,7 +27,7 @@ public class AutoSetFileLinksUtilTest {
 
     private final FilePreferences fileDirPrefs = mock(FilePreferences.class);
     private final AutoLinkPreferences autoLinkPrefs =
-            new AutoLinkPreferences(AutoLinkPreferences.CitationKeyDependency.REGEX, "", ';');
+            new AutoLinkPreferences(AutoLinkPreferences.CitationKeyDependency.START, "", ';');
     private final BibDatabaseContext databaseContext = mock(BibDatabaseContext.class);
     private final ExternalFileTypes externalFileTypes = mock(ExternalFileTypes.class);
     private final BibEntry entry = new BibEntry(StandardEntryType.Article);