Merge pull request #3690 from jeremylong/timeout

NVD Rate Limiting Fix?

ant/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>6.3.3-SNAPSHOT</version>
+        <version>6.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>dependency-check-ant</artifactId>

ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java

@@ -92,6 +92,10 @@ public class Update extends Purge {
      * Base Data Mirror URL for CVE JSON files.
      */
     private String cveUrlBase;
+    /**
+     * The wait time in milliseconds between downloads from the NVD.
+     */
+    private String cveWaitTime;
     /**
      * The number of hours to wait before re-checking for updates.
      */
@@ -341,6 +345,24 @@ public void setCveUrlBase(String cveUrlBase) {
         this.cveUrlBase = cveUrlBase;
     }
 
+    /**
+     * Get the value of cveUrlBase.
+     *
+     * @return the value of cveUrlBase
+     */
+    public String getCveWaitTime() {
+        return cveWaitTime;
+    }
+
+    /**
+     * Set the value of cveWaitTime.
+     *
+     * @param cveWaitTime new value of cveWaitTime
+     */
+    public void setCveWaitTime(String cveWaitTime) {
+        this.cveWaitTime = cveWaitTime;
+    }
+
     /**
      * Get the value of cveValidForHours.
      *
@@ -413,11 +435,12 @@ protected void populateSettings() throws BuildException {
         getSettings().setStringIfNotEmpty(Settings.KEYS.DB_USER, databaseUser);
         getSettings().setStringIfNotEmpty(Settings.KEYS.DB_PASSWORD, databasePassword);
 
-        String cveModifiedJson = Optional.ofNullable(cveUrlModified)
-          .filter(url -> !url.isEmpty())
-          .orElseGet(this::getDefaultCveUrlModified);
+        final String cveModifiedJson = Optional.ofNullable(cveUrlModified)
+                .filter(url -> !url.isEmpty())
+                .orElseGet(this::getDefaultCveUrlModified);
         getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveModifiedJson);
         getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase);
+        getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, cveWaitTime);
         if (cveValidForHours != null) {
             if (cveValidForHours >= 0) {
                 getSettings().setInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours);
@@ -428,7 +451,7 @@ protected void populateSettings() throws BuildException {
     }
 
     private String getDefaultCveUrlModified() {
-      return CveUrlParser.newInstance(getSettings())
-          .getDefaultCveUrlModified(cveUrlBase);
+        return CveUrlParser.newInstance(getSettings())
+                .getDefaultCveUrlModified(cveUrlBase);
     }
 }

ant/src/site/markdown/config-update.md

@@ -36,6 +36,7 @@ Property             | Description
 ---------------------|----------------------------------------------------------------------------------------------------------------------|------------------
 cveUrlModified       | URL for the modified CVE JSON data feed. When mirroring the NVD you must mirror the *.json.gz and the *.meta files. Optional if your custom cveUrlBase is just a domain name change.  | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz
 cveUrlBase           | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year.                                  | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz
+cveWaitTime          | The time in milliseconds to wait between downloads from the NVD.                                                     | 4000
 dataDirectory        | Data directory that is used to store the local copy of the NVD. This should generally not be changed.                | data
 databaseDriverName   | The name of the database driver. Example: org.h2.Driver.                                                             |  
 databaseDriverPath   | The path to the database driver JAR file; only used if the driver is not in the class path.                          |  

archetype/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2017 Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>6.3.3-SNAPSHOT</version>
+        <version>6.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>dependency-check-plugin</artifactId>
     <name>Dependency-Check Plugin Archetype</name>

cli/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2012 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>6.3.3-SNAPSHOT</version>
+        <version>6.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>dependency-check-cli</artifactId>

cli/src/main/java/org/owasp/dependencycheck/App.java

@@ -390,6 +390,7 @@ private void runUpdateOnly() throws UpdateException, DatabaseException {
             engine.doUpdates();
         }
     }
+
     //CSOFF: MethodLength
     /**
      * Updates the global Settings.
@@ -584,10 +585,12 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException {
                 cli.getStringArgument(CliParser.ARGUMENT.PATH_TO_CORE));
         settings.setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON,
                 cli.getStringArgument(CliParser.ARGUMENT.CVE_BASE_URL));
+        settings.setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME,
+                cli.getStringArgument(CliParser.ARGUMENT.CVE_DOWNLOAD_WAIT_TIME));
 
-        String cveModifiedJson = Optional.ofNullable(cli.getStringArgument(CliParser.ARGUMENT.CVE_MODIFIED_URL))
-            .filter(arg -> !arg.isEmpty())
-            .orElseGet(() -> getDefaultCveUrlModified(cli));
+        final String cveModifiedJson = Optional.ofNullable(cli.getStringArgument(CliParser.ARGUMENT.CVE_MODIFIED_URL))
+                .filter(arg -> !arg.isEmpty())
+                .orElseGet(() -> getDefaultCveUrlModified(cli));
         settings.setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON,
                 cveModifiedJson);
 
@@ -598,8 +601,8 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException {
     }
 
     private String getDefaultCveUrlModified(CliParser cli) {
-      return CveUrlParser.newInstance(settings)
-          .getDefaultCveUrlModified(cli.getStringArgument(CliParser.ARGUMENT.CVE_BASE_URL));
+        return CveUrlParser.newInstance(settings)
+                .getDefaultCveUrlModified(cli.getStringArgument(CliParser.ARGUMENT.CVE_BASE_URL));
     }
 
     //CSON: MethodLength

cli/src/main/java/org/owasp/dependencycheck/CliParser.java

@@ -307,6 +307,8 @@ private void addAdvancedOptions(final Options options) {
                         "Base URL for each year’s CVE files (json.gz), the %d will be replaced with the year."))
                 .addOption(newOptionWithArg(ARGUMENT.CVE_MODIFIED_URL, "url",
                         "URL for the modified CVE (json.gz)."))
+                .addOption(newOptionWithArg(ARGUMENT.CVE_DOWNLOAD_WAIT_TIME, "milliseconds",
+                        "Time in milliseconds to wait between downloading from the NVD."))
                 .addOption(newOptionWithArg(ARGUMENT.CVE_USER, "user",
                         "Credentials for basic authentication to the CVE data."))
                 .addOption(newOptionWithArg(ARGUMENT.CVE_PASSWORD, "password",
@@ -1046,6 +1048,10 @@ public static class ARGUMENT {
          * The CLI argument name for setting the URL for the CVE Data Files.
          */
         public static final String CVE_BASE_URL = "cveUrlBase";
+        /**
+         * The time in milliseconds to wait between downloading NVD CVE data.
+         */
+        public static final String CVE_DOWNLOAD_WAIT_TIME = "cveDownloadWait";
         /**
          * The short CLI argument name for setting the location of the data
          * directory.
@@ -1163,7 +1169,7 @@ public static class ARGUMENT {
          * Disables the swift package manager analyzer.
          */
         public static final String DISABLE_SWIFT = "disableSwiftPackageManagerAnalyzer";
-                /**
+        /**
          * Disables the swift package resolved analyzer.
          */
         public static final String DISABLE_SWIFT_RESOLVED = "disableSwiftPackageResolvedAnalyzer";

cli/src/main/resources/completion-for-dependency-check.sh

@@ -24,6 +24,7 @@ _odc_completions()
             --cveValidForHours
             --cveUser <user>
             --cvePassword <password>
+            --cveDownloadWait <milliseconds>
         -d --data
             --dbDriverName
             --dbDriverPath

core/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>6.3.3-SNAPSHOT</version>
+        <version>6.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>dependency-check-core</artifactId>

core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java

@@ -340,7 +340,6 @@ protected final MetaProperties getMetaFile(String url) throws UpdateException {
             final URL u = new URL(metaUrl);
             final Downloader d = new Downloader(settings);
             final String content = d.fetchContent(u, true, Settings.KEYS.CVE_USER, Settings.KEYS.CVE_PASSWORD);
-            Thread.sleep(2000);
             return new MetaProperties(content);
         } catch (MalformedURLException ex) {
             throw new UpdateException("Meta file url is invalid: " + metaUrl, ex);
@@ -352,9 +351,6 @@ protected final MetaProperties getMetaFile(String url) throws UpdateException {
             throw new UpdateException("Unable to download meta file: " + metaUrl + "; received 429 -- too many requests", ex);
         } catch (ResourceNotFoundException ex) {
             throw new UpdateException("Unable to download meta file: " + metaUrl + "; received 404 -- resource not found", ex);
-        } catch (InterruptedException ex) {
-            Thread.interrupted();
-            throw new UpdateException("The download of the meta file was interupted: " + metaUrl, ex);
         }
     }
 
@@ -399,10 +395,12 @@ protected final List<NvdCveInfo> getUpdatesNeeded() throws UpdateException {
                         final int start = settings.getInt(Settings.KEYS.CVE_START_YEAR);
                         final int end = Calendar.getInstance().get(Calendar.YEAR);
                         final String baseUrl = settings.getString(Settings.KEYS.CVE_BASE_JSON);
+                        final long waitTime = settings.getInt(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, 4000);
                         for (int i = start; i <= end; i++) {
                             try {
                                 url = String.format(baseUrl, i);
                                 final MetaProperties meta = getMetaFile(url);
+                                Thread.sleep(waitTime);
                                 final long currentTimestamp = getPropertyInSeconds(DatabaseProperties.LAST_UPDATED_BASE + i);
 
                                 if (currentTimestamp < meta.getLastModifiedDate()) {
@@ -421,6 +419,9 @@ protected final List<NvdCveInfo> getUpdatesNeeded() throws UpdateException {
                                 } else {
                                     throw ex;
                                 }
+                            } catch (InterruptedException ex) {
+                                Thread.interrupted();
+                                throw new UpdateException("The download of the meta file was interupted: " + url, ex);
                             }
                         }
                     }

core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java

@@ -143,8 +143,8 @@ public Future<ProcessTask> call() throws Exception {
             }
             final ProcessTask task = new ProcessTask(cveDB, this, settings);
             final Future<ProcessTask> val = this.processorService.submit(task);
-
-            Thread.sleep(2000);
+            final long waitTime = settings.getInt(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, 4000);
+            Thread.sleep(waitTime);
             return val;
 
         } catch (Throwable ex) {

core/src/main/resources/dependencycheck.properties

@@ -59,6 +59,8 @@ cve.url.modified.validfordays=7
 cve.check.validforhours=4
 #first year to pull data from the URLs below
 cve.startyear=2002
+# the time in milliseconds to wait between downloads from the NVD
+cve.download.waittime=4000
 #the original URL and modified URL should be the same; this is used to detect if we are using an internal NVD CVE copy
 cve.url.original=https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz
 cve.url.modified=https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz

maven/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>6.3.3-SNAPSHOT</version>
+        <version>6.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>dependency-check-maven</artifactId>
     <packaging>maven-plugin</packaging>

maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java

@@ -833,6 +833,12 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
     @SuppressWarnings("CanBeFinal")
     @Parameter(property = "cveUrlBase")
     private String cveUrlBase;
+    /**
+     * The wait timeout between downloading from the NVD.
+     */
+    @SuppressWarnings("CanBeFinal")
+    @Parameter(property = "cveWaitTime")
+    private String cveWaitTime;
     /**
      * The username to use when connecting to the CVE-URL.
      */
@@ -2050,6 +2056,7 @@ protected void populateSettings() {
             .orElseGet(this::getDefaultCveUrlModified);
         settings.setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveModifiedJson);
         settings.setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase);
+        settings.setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, cveWaitTime);
         settings.setIntIfNotNull(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours);
         settings.setBooleanIfNotNull(Settings.KEYS.PRETTY_PRINT, prettyPrint);
         artifactScopeExcluded = new ArtifactScopeExcluded(skipTestScope, skipProvidedScope, skipSystemScope, skipRuntimeScope);

maven/src/site/markdown/configuration.md

@@ -131,11 +131,12 @@ may be the cveUrl properties, which can be used to host a mirror of the NVD with
 
 Property                 | Description                                                                                                                                | Default Value                                                       |
 -------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|
-cveUrlModified           | URL for the modified CVE JSON data feed.  When mirroring the NVD you must mirror the *.json.gz and the *.meta files. Optional if your custom cveUrlBase is just a domain name change.                       | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz |
+cveUrlModified           | URL for the modified CVE JSON data feed.  When mirroring the NVD you must mirror the *.json.gz and the *.meta files. Optional if your custom cveUrlBase is just a domain name change. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz |
 cveUrlBase               | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year.                                                        | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz       |
 cveServerId              | The id of a server defined in the settings.xml that configures the credentials (username and password) for accessing the cveUrl.           |                                                                |
 cveUser                  | The username used when connecting to the cveUrl. Must be empty if cveServerId is specified and should be used.                             |                                                                |
 cvePassword              | The password used when connecting to the cveUrl. Must be empty if cveServerId is specified and should be used.                             |                                                                |
+cveWaitTime              | The time in milliseconds to wait between downloads from the NVD.                                                                           | 4000                                                                |
 suppressionFileServerId  | The id of a server defined in the settings.xml that configures the credentials (username and password) for accessing the suppressionFiles. |                                                                |
 suppressionFileUser      | The username used when connecting to the suppressionFiles. Must be empty if suppressionFileServerId is specified and should be used.       |                                                                |
 suppressionFilePassword  | The password used when connecting to the suppressionFiles. Must be empty if suppressionFileServerId is specified and should be used.       |                                                                |

pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2012 - Jeremy Long
 
     <groupId>org.owasp</groupId>
     <artifactId>dependency-check-parent</artifactId>
-    <version>6.3.3-SNAPSHOT</version>
+    <version>6.4.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <modules>

src/site/markdown/dependency-check-gradle/configuration-aggregate.md

@@ -84,8 +84,9 @@ Note, if ANY of the cve configuration group are set - they should all be set to
 
 Config Group | Property          | Description                                                                                                  | Default Value                                                       |
 -------------|-------------------|--------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|
-cve          | urlModified    | URL for the modified CVE JSON data feed.                                                                        | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz |
-cve          | urlBase        | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year.                             | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz       |
+cve          | urlModified       | URL for the modified CVE JSON data feed.                                                                     | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz |
+cve          | urlBase           | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year.                          | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz       |
+cve          | waitTime          | The time in milliseconds to wait between downloads from the NVD.                                             | 4000                                                                |
 data         | directory         | Sets the data directory to hold SQL CVEs contents. This should generally not be changed.                     |                                                                |
 data         | driver            | The name of the database driver. Example: org.h2.Driver.                                                     |                                                                |
 data         | driverPath        | The path to the database driver JAR file; only used if the driver is not in the class path.                  |                                                                |