Keep source urls on file download (#10855)

* Add source URL to the linked files and update it when a file is initially downloaded * Add context menu action to redownload a file * Add main menu tools action to redownload missing files Refactor redownload code into an action * Don't change the local link name when redownloading, the link should already have an appropriate file name, so use that for the downloaded file. * Update CHANGELOG.md * revert method change * Revert changes in property files * Improve extension check * Fix issue where action would only be executed once per jabref execution. * Make string empty check more readable * Fix test for FileFieldParser * Small refactoring and language fix * Refactor LinkedFileViewModel to use the DownloadLinkedFileAction for downloading Change test to use that too. * Fix rewrite check * Change file field format to be backwards compatible with old versions with mediaType. Recover/add tests for the file field. * Small language fix and reduced public fingerprint * Extract downloadProgress * DownloadLinkedFileActionTest Co-authored-by: Christoph <siedlerkiller@gmail.com> * Fixed checkstyle * Revert "Change file field format to be backwards compatible with old versions with mediaType." This reverts commit 2230310. * Reapply and modify good tests --------- Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Co-authored-by: Christoph <siedlerkiller@gmail.com>
JabRef · Feb 18, 2024 · 2777ebc · 2777ebc
1 parent 4194a09
commit 2777ebc
Show file tree

Hide file tree

Showing 19 changed files with 611 additions and 246 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We added a new group icon column to the main table showing the icons of the entry's groups. [#10801](https://github.com/JabRef/jabref/pull/10801)
 - We added ability to jump to an entry in the command line using `-j CITATIONKEY`. [koppor#540](https://github.com/koppor/jabref/issues/540)
 - We added a new boolean to the style files for Openoffice/Libreoffice integration to switch between ZERO_WIDTH_SPACE (default) and no space. [#10843](https://github.com/JabRef/jabref/pull/10843)
+- We added the possibility to redownload files that had been present but are no longer in the specified location. [#10848](https://github.com/JabRef/jabref/issues/10848)
 
 ### Changed
 

diff --git a/src/main/java/org/jabref/gui/MainMenu.java b/src/main/java/org/jabref/gui/MainMenu.java
@@ -44,6 +44,7 @@
 import org.jabref.gui.integrity.IntegrityCheckAction;
 import org.jabref.gui.journals.AbbreviateAction;
 import org.jabref.gui.libraryproperties.LibraryPropertiesAction;
+import org.jabref.gui.linkedfile.RedownloadMissingFilesAction;
 import org.jabref.gui.mergeentries.MergeEntriesAction;
 import org.jabref.gui.preferences.ShowPreferencesAction;
 import org.jabref.gui.preview.CopyCitationAction;
@@ -287,7 +288,11 @@ private void createMenu() {
 
                 new SeparatorMenuItem(),
 
-                factory.createMenuItem(StandardActions.REBUILD_FULLTEXT_SEARCH_INDEX, new RebuildFulltextSearchIndexAction(stateManager, frame::getCurrentLibraryTab, dialogService, preferencesService.getFilePreferences(), taskExecutor))
+                factory.createMenuItem(StandardActions.REBUILD_FULLTEXT_SEARCH_INDEX, new RebuildFulltextSearchIndexAction(stateManager, frame::getCurrentLibraryTab, dialogService, preferencesService.getFilePreferences(), taskExecutor)),
+
+                new SeparatorMenuItem(),
+
+                factory.createMenuItem(StandardActions.REDOWNLOAD_MISSING_FILES, new RedownloadMissingFilesAction(stateManager, dialogService, preferencesService.getFilePreferences(), taskExecutor))
         );
         SidePaneType webSearchPane = SidePaneType.WEB_SEARCH;
         SidePaneType groupsPane = SidePaneType.GROUPS;

diff --git a/src/main/java/org/jabref/gui/actions/StandardActions.java b/src/main/java/org/jabref/gui/actions/StandardActions.java
@@ -30,6 +30,7 @@ public enum StandardActions implements Action {
     SEND_AS_EMAIL(Localization.lang("As Email")),
     SEND_TO_KINDLE(Localization.lang("To Kindle")),
     REBUILD_FULLTEXT_SEARCH_INDEX(Localization.lang("Rebuild fulltext search index"), IconTheme.JabRefIcons.FILE),
+    REDOWNLOAD_MISSING_FILES(Localization.lang("Redownload missing files"), IconTheme.JabRefIcons.DOWNLOAD),
     OPEN_EXTERNAL_FILE(Localization.lang("Open file"), IconTheme.JabRefIcons.FILE, KeyBinding.OPEN_FILE),
     OPEN_URL(Localization.lang("Open URL or DOI"), IconTheme.JabRefIcons.WWW, KeyBinding.OPEN_URL_OR_DOI),
     SEARCH_SHORTSCIENCE(Localization.lang("Search ShortScience")),
@@ -149,6 +150,7 @@ public enum StandardActions implements Action {
 
     EDIT_FILE_LINK(Localization.lang("Edit"), IconTheme.JabRefIcons.EDIT, KeyBinding.EDIT_ENTRY),
     DOWNLOAD_FILE(Localization.lang("Download file"), IconTheme.JabRefIcons.DOWNLOAD_FILE),
+    REDOWNLOAD_FILE(Localization.lang("Redownload file"), IconTheme.JabRefIcons.DOWNLOAD_FILE),
     RENAME_FILE_TO_PATTERN(Localization.lang("Rename file to defined pattern"), IconTheme.JabRefIcons.AUTO_RENAME),
     RENAME_FILE_TO_NAME(Localization.lang("Rename file to a given name"), IconTheme.JabRefIcons.RENAME, KeyBinding.REPLACE_STRING),
     MOVE_FILE_TO_FOLDER(Localization.lang("Move file to file directory"), IconTheme.JabRefIcons.MOVE_TO_FOLDER),

diff --git a/src/main/java/org/jabref/gui/externalfiles/FileDownloadTask.java b/src/main/java/org/jabref/gui/externalfiles/FileDownloadTask.java
diff --git a/src/main/java/org/jabref/gui/fieldeditors/LinkedFileViewModel.java b/src/main/java/org/jabref/gui/fieldeditors/LinkedFileViewModel.java
@@ -1,7 +1,6 @@
 package org.jabref.gui.fieldeditors;
 
 import java.io.IOException;
-import java.net.MalformedURLException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -11,11 +10,6 @@
 import java.util.function.BiPredicate;
 import java.util.function.Supplier;
 
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLHandshakeException;
-import javax.net.ssl.SSLSocketFactory;
-
 import javafx.beans.Observable;
 import javafx.beans.binding.Bindings;
 import javafx.beans.binding.ObjectBinding;
@@ -32,20 +26,16 @@
 import org.jabref.gui.AbstractViewModel;
 import org.jabref.gui.DialogService;
 import org.jabref.gui.desktop.JabRefDesktop;
-import org.jabref.gui.externalfiles.FileDownloadTask;
 import org.jabref.gui.externalfiletype.ExternalFileType;
 import org.jabref.gui.externalfiletype.ExternalFileTypes;
-import org.jabref.gui.externalfiletype.StandardExternalFileType;
 import org.jabref.gui.icon.IconTheme;
 import org.jabref.gui.icon.JabRefIcon;
+import org.jabref.gui.linkedfile.DownloadLinkedFileAction;
 import org.jabref.gui.linkedfile.LinkedFileEditDialogView;
 import org.jabref.gui.mergeentries.MultiMergeEntriesView;
-import org.jabref.gui.util.BackgroundTask;
 import org.jabref.gui.util.ControlHelper;
 import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.externalfiles.LinkedFileHandler;
-import org.jabref.logic.importer.FetcherClientException;
-import org.jabref.logic.importer.FetcherServerException;
 import org.jabref.logic.importer.Importer;
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.importer.fileformat.PdfContentImporter;
@@ -54,8 +44,6 @@
 import org.jabref.logic.importer.fileformat.PdfVerbatimBibTextImporter;
 import org.jabref.logic.importer.fileformat.PdfXmpImporter;
 import org.jabref.logic.l10n.Localization;
-import org.jabref.logic.net.URLDownload;
-import org.jabref.logic.util.io.FileNameUniqueness;
 import org.jabref.logic.util.io.FileUtil;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
@@ -421,146 +409,47 @@ public void edit() {
             this.linkedFile.setLink(file.getLink());
             this.linkedFile.setDescription(file.getDescription());
             this.linkedFile.setFileType(file.getFileType());
+            this.linkedFile.setSourceURL(file.getSourceUrl());
         });
     }
 
-    public void download() {
-        if (!linkedFile.isOnlineLink()) {
-            throw new UnsupportedOperationException("In order to download the file it has to be an online link");
-        }
-        try {
-            Optional<Path> targetDirectory = databaseContext.getFirstExistingFileDir(preferencesService.getFilePreferences());
-            if (targetDirectory.isEmpty()) {
-                dialogService.showErrorDialogAndWait(Localization.lang("Download file"), Localization.lang("File directory is not set or does not exist!"));
-                return;
-            }
-
-            URLDownload urlDownload = new URLDownload(linkedFile.getLink());
-            if (!checkSSLHandshake(urlDownload)) {
-                return;
-            }
-
-            BackgroundTask<Path> downloadTask = prepareDownloadTask(targetDirectory.get(), urlDownload);
-            downloadTask.onSuccess(destination -> {
-                boolean isDuplicate;
-                try {
-                    isDuplicate = FileNameUniqueness.isDuplicatedFile(targetDirectory.get(), destination.getFileName(), dialogService);
-                } catch (IOException e) {
-                    LOGGER.error("FileNameUniqueness.isDuplicatedFile failed", e);
-                    return;
-                }
-
-                if (!isDuplicate) {
-                    // we need to call LinkedFileViewModel#fromFile, because we need to make the path relative to the configured directories
-                    LinkedFile newLinkedFile = LinkedFilesEditorViewModel.fromFile(
-                            destination,
-                            databaseContext.getFileDirectories(preferencesService.getFilePreferences()),
-                            preferencesService.getFilePreferences());
-                    entry.replaceDownloadedFile(linkedFile.getLink(), newLinkedFile);
-
-                    // Notify in bar when the file type is HTML.
-                    if (newLinkedFile.getFileType().equals(StandardExternalFileType.URL.getName())) {
-                        dialogService.notify(Localization.lang("Downloaded website as an HTML file."));
-                        LOGGER.debug("Downloaded website {} as an HTML file at {}", linkedFile.getLink(), destination);
-                    }
-                }
-            });
-            downloadProgress.bind(downloadTask.workDonePercentageProperty());
-            downloadTask.titleProperty().set(Localization.lang("Downloading"));
-            downloadTask.messageProperty().set(
-                    Localization.lang("Fulltext for") + ": " + entry.getCitationKey().orElse(Localization.lang("New entry")));
-            downloadTask.showToUser(true);
-            downloadTask.onFailure(ex -> {
-                LOGGER.error("Error downloading from URL: " + urlDownload, ex);
-                String fetcherExceptionMessage = ex.getMessage();
-                int statusCode;
-                if (ex instanceof FetcherClientException clientException) {
-                    statusCode = clientException.getStatusCode();
-                    if (statusCode == 401) {
-                        dialogService.showInformationDialogAndWait(Localization.lang("Failed to download from URL"), Localization.lang("401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage));
-                    } else if (statusCode == 403) {
-                        dialogService.showInformationDialogAndWait(Localization.lang("Failed to download from URL"), Localization.lang("403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage));
-                    } else if (statusCode == 404) {
-                        dialogService.showInformationDialogAndWait(Localization.lang("Failed to download from URL"), Localization.lang("404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage));
-                    }
-                } else if (ex instanceof FetcherServerException serverException) {
-                    statusCode = serverException.getStatusCode();
-                    dialogService.showInformationDialogAndWait(Localization.lang("Failed to download from URL"),
-                                Localization.lang("Error downloading from URL. Cause is likely the server side. HTTP Error %0 \n %1 \nURL: %2 \nPlease try again later or contact the server administrator.", statusCode, fetcherExceptionMessage, urlDownload.getSource()));
-                    } else {
-                        dialogService.showErrorDialogAndWait(Localization.lang("Failed to download from URL"), Localization.lang("Error message: %0 \nURL: %1 \nPlease check the URL and try again.", fetcherExceptionMessage, urlDownload.getSource()));
-                    }
-                });
-                taskExecutor.execute(downloadTask);
-            } catch (MalformedURLException exception) {
-                dialogService.showErrorDialogAndWait(Localization.lang("Invalid URL"), exception);
-            }
+    public void redownload() {
+        LOGGER.info("Redownloading file from " + linkedFile.getSourceUrl());
+        if (linkedFile.getSourceUrl().isEmpty() || !LinkedFile.isOnlineLink(linkedFile.getSourceUrl())) {
+            throw new UnsupportedOperationException("In order to download the file, the source url has to be an online link");
         }
 
-        public boolean checkSSLHandshake(URLDownload urlDownload) {
-            try {
-                urlDownload.canBeReached();
-            } catch (kong.unirest.UnirestException ex) {
-                if (ex.getCause() instanceof SSLHandshakeException) {
-                    if (dialogService.showConfirmationDialogAndWait(Localization.lang("Download file"),
-                            Localization.lang("Unable to find valid certification path to requested target(%0), download anyway?",
-                                    urlDownload.getSource().toString()))) {
-                        return true;
-                    } else {
-                        dialogService.notify(Localization.lang("Download operation canceled."));
-                        return false;
-                    }
-                } else {
-                    LOGGER.error("Error while checking if the file can be downloaded", ex);
-                    dialogService.notify(Localization.lang("Error downloading"));
-                    return false;
-                }
-            }
-            return true;
-        }
+        String fileName = Path.of(linkedFile.getLink()).getFileName().toString();
 
-    public BackgroundTask<Path> prepareDownloadTask(Path targetDirectory, URLDownload urlDownload) {
-        SSLSocketFactory defaultSSLSocketFactory = HttpsURLConnection.getDefaultSSLSocketFactory();
-        HostnameVerifier defaultHostnameVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
-        return BackgroundTask
-                .wrap(() -> {
-                    Optional<ExternalFileType> suggestedType = inferFileType(urlDownload);
-                    ExternalFileType externalFileType = suggestedType.orElse(StandardExternalFileType.PDF);
-
-                    String suggestedName = linkedFileHandler.getSuggestedFileName(externalFileType.getExtension());
-                    String fulltextDir = FileUtil.createDirNameFromPattern(databaseContext.getDatabase(), entry, preferencesService.getFilePreferences().getFileDirectoryPattern());
-                    suggestedName = FileNameUniqueness.getNonOverWritingFileName(targetDirectory.resolve(fulltextDir), suggestedName);
-                    return targetDirectory.resolve(fulltextDir).resolve(suggestedName);
-                })
-                .then(destination -> new FileDownloadTask(urlDownload.getSource(), destination))
-                .onFailure(ex -> LOGGER.error("Error in download", ex))
-                .onFinished(() -> URLDownload.setSSLVerification(defaultSSLSocketFactory, defaultHostnameVerifier));
-    }
-
-    private Optional<ExternalFileType> inferFileType(URLDownload urlDownload) {
-        Optional<ExternalFileType> suggestedType = inferFileTypeFromMimeType(urlDownload);
-
-        // If we did not find a file type from the MIME type, try based on extension:
-        if (suggestedType.isEmpty()) {
-            suggestedType = inferFileTypeFromURL(urlDownload.getSource().toExternalForm());
-        }
-        return suggestedType;
+        DownloadLinkedFileAction downloadLinkedFileAction = new DownloadLinkedFileAction(
+                databaseContext,
+                entry,
+                linkedFile,
+                linkedFile.getSourceUrl(),
+                dialogService,
+                preferencesService.getFilePreferences(),
+                taskExecutor,
+                fileName);
+        downloadProgress.bind(downloadLinkedFileAction.downloadProgress());
+        downloadLinkedFileAction.execute();
     }
 
-    private Optional<ExternalFileType> inferFileTypeFromMimeType(URLDownload urlDownload) {
-        String mimeType = urlDownload.getMimeType();
-
-        if (mimeType != null) {
-            LOGGER.debug("MIME Type suggested: " + mimeType);
-            return ExternalFileTypes.getExternalFileTypeByMimeType(mimeType, preferencesService.getFilePreferences());
-        } else {
-            return Optional.empty();
+    public void download() {
+        LOGGER.info("Downloading file from " + linkedFile.getSourceUrl());
+        if (!linkedFile.isOnlineLink()) {
+            throw new UnsupportedOperationException("In order to download the file it has to be an online link");
         }
-    }
 
-    private Optional<ExternalFileType> inferFileTypeFromURL(String url) {
-        return URLUtil.getSuffix(url, preferencesService.getFilePreferences())
-                      .flatMap(extension -> ExternalFileTypes.getExternalFileTypeByExt(extension, preferencesService.getFilePreferences()));
+        DownloadLinkedFileAction downloadLinkedFileAction = new DownloadLinkedFileAction(
+                databaseContext,
+                entry,
+                linkedFile,
+                linkedFile.getLink(),
+                dialogService,
+                preferencesService.getFilePreferences(),
+                taskExecutor);
+        downloadProgress.bind(downloadLinkedFileAction.downloadProgress());
+        downloadLinkedFileAction.execute();
     }
 
     public LinkedFile getFile() {