update htsjdk to 2.21.0 (#6250)

* update htsjdk 2.20.3 -> 2.21.0

build.gradle

@@ -57,7 +57,7 @@ repositories {
     mavenLocal()
 }
 
-final htsjdkVersion = System.getProperty('htsjdk.version','2.20.3')
+final htsjdkVersion = System.getProperty('htsjdk.version','2.21.0')
 final picardVersion = System.getProperty('picard.version','2.21.1')
 final barclayVersion = System.getProperty('barclay.version','2.1.0')
 final sparkVersion = System.getProperty('spark.version', '2.4.3')

src/main/java/org/broadinstitute/hellbender/exceptions/UserException.java

@@ -1,6 +1,7 @@
 package org.broadinstitute.hellbender.exceptions;
 
 import htsjdk.samtools.SAMSequenceDictionary;
+import htsjdk.samtools.SAMSequenceRecord;
 import htsjdk.tribble.Feature;
 import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
 import org.broadinstitute.hellbender.tools.walkers.variantutils.ValidateVariants;
@@ -12,6 +13,7 @@
 import java.io.File;
 import java.nio.file.Path;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * <p/>
@@ -339,12 +341,6 @@ public MissingContigInSequenceDictionary(String contigName, SAMSequenceDictionar
         }
     }
 
-    /**
-     * <p/>
-     * Class UserException.MalformedFile
-     * <p/>
-     * For errors parsing files
-     */
     public static class IncompatibleSequenceDictionaries extends UserException {
         private static final long serialVersionUID = 0L;
 
@@ -373,6 +369,21 @@ public LexicographicallySortedSequenceDictionary(String name, SAMSequenceDiction
         }
     }
 
+    public static class SequenceDictionaryIsMissingContigLengths extends UserException {
+        private static final long serialVersionUID = 0L;
+
+        public SequenceDictionaryIsMissingContigLengths(String source, SAMSequenceDictionary dict){
+            super("GATK SequenceDictionaryValidation requires all contigs in the dictionary to have lengths associated with them.  " +
+                    "\nOne or more contigs in the dictionary from " + source + " are missing contig lengths." +
+                    "\nThe following contigs are missing lengths: " +
+                    dict.getSequences().stream()
+                            .filter( s -> s.getSequenceLength() == SAMSequenceRecord.UNKNOWN_SEQUENCE_LENGTH)
+                            .map(SAMSequenceRecord::getSequenceName)
+                            .limit(20)
+                            .collect(Collectors.joining(",")));
+        }
+    }
+
     public static final class Require2BitReferenceForBroadcast extends BadInput {
         private static final long serialVersionUID = 0L;
         public Require2BitReferenceForBroadcast() {

src/main/java/org/broadinstitute/hellbender/tools/walkers/variantutils/UpdateVCFSequenceDictionary.java

@@ -13,6 +13,8 @@
 import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
 import org.broadinstitute.barclay.help.DocumentedFeature;
 import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
+import org.broadinstitute.hellbender.exceptions.UserException;
+import org.broadinstitute.hellbender.utils.SequenceDictionaryUtils;
 import picard.cmdline.programgroups.VariantManipulationProgramGroup;
 import org.broadinstitute.hellbender.engine.FeatureContext;
 import org.broadinstitute.hellbender.engine.ReadsContext;
@@ -79,7 +81,7 @@
 )
 @DocumentedFeature
 public final class UpdateVCFSequenceDictionary extends VariantWalker {
-    static final Logger logger = LogManager.getLogger(UpdateVCFSequenceDictionary.class);
+    private static final Logger logger = LogManager.getLogger(UpdateVCFSequenceDictionary.class);
 
     @Argument(fullName = StandardArgumentDefinitions.OUTPUT_LONG_NAME,
             shortName = StandardArgumentDefinitions.OUTPUT_SHORT_NAME,
@@ -183,14 +185,14 @@ public void closeTool() {
     @Override
     public SAMSequenceDictionary getBestAvailableSequenceDictionary() {
 
-        SAMSequenceDictionary resultDictionary;
+        final SAMSequenceDictionary resultDictionary;
 
         final SAMSequenceDictionary masterDictionary = getMasterSequenceDictionary();
         if (dictionarySource == null) {
             if (masterDictionary != null) {
                 // We'll accept the master dictionary if one was specified. Using the master dictionary
                 // arg will result in sequence dictionary validation.
-                logger.warn("Using the dictionary supplied via the \"%s\" argument", StandardArgumentDefinitions.SEQUENCE_DICTIONARY_NAME);
+                logger.warn("Using the dictionary supplied via the {} argument", StandardArgumentDefinitions.SEQUENCE_DICTIONARY_NAME);
                 resultDictionary = masterDictionary;
             }
             else if (hasReference()) {
@@ -209,11 +211,22 @@ else if (hasReference()) {
             if (resultDictionary == null || resultDictionary.getSequences().isEmpty()) {
                 throw new CommandLineException.BadArgumentValue(
                     String.format(
-                        "The specified dictionary source has an empty or invalid sequence dictionary",
+                        "The specified dictionary source has an empty or invalid sequence dictionary: %s",
                         dictionarySource)
                 );
             }
         }
+
+        if( seqValidationArguments.performSequenceDictionaryValidation()
+                && resultDictionary != null
+                && dictionaryHasMissingLengths(resultDictionary)) {
+            throw new UserException.SequenceDictionaryIsMissingContigLengths(dictionarySource, resultDictionary);
+        }
+
         return resultDictionary;
     }
+
+    private boolean dictionaryHasMissingLengths(final SAMSequenceDictionary resultDictionary) {
+        return resultDictionary.getSequences().stream().anyMatch(s -> s.getSequenceLength() == SAMSequenceRecord.UNKNOWN_SEQUENCE_LENGTH);
+    }
 }

src/test/java/org/broadinstitute/hellbender/tools/walkers/variantutils/UpdateVCFSequenceDictionaryIntegrationTest.java

@@ -8,6 +8,7 @@
 import org.broadinstitute.barclay.argparser.CommandLineException;
 import org.broadinstitute.hellbender.CommandLineProgramTest;
 import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
+import org.broadinstitute.hellbender.exceptions.UserException;
 import org.broadinstitute.hellbender.testutils.ArgumentsBuilder;
 
 import org.testng.Assert;
@@ -45,7 +46,7 @@ public Object[][] updateGoodSequenceDictionaryData() {
     }
 
     @Test(dataProvider="UpdateGoodSequenceDictionaryData")
-    private void testGoodUpdateSequenceDictionary(
+    public void testGoodUpdateSequenceDictionary(
             final File inputVariantsFile,
             final File inputSourceFile,
             final File inputReferenceFile,
@@ -88,7 +89,7 @@ public Object[][] updateBadSequenceDictionaryData() {
     }
 
     @Test(dataProvider="UpdateBadSequenceDictionaryData", expectedExceptions= CommandLineException.BadArgumentValue.class)
-    private void testBadUpdateSequenceDictionary(
+    public void testBadUpdateSequenceDictionary(
             final File inputVariantsFile,
             final File inputSourceFile,
             final File inputReferenceFile,
@@ -118,15 +119,15 @@ public void testUseMasterDictionary() {
     }
 
 
-    @Test(expectedExceptions= TribbleException.class)
+    @Test(expectedExceptions = UserException.SequenceDictionaryIsMissingContigLengths.class)
     public void testBadContigLengthWithValidation() {
         // throw an error if trying to force a replace with an invalid sequence dictionary without disabling sequence validation
         updateSequenceDictionary(
+                new File(testDir, "variantsWithDict.vcf"),
                 new File(testDir, "variantsWithDictBadContigLength.vcf"),
-                new File(testDir, "exampleFASTA.dict"),
                 null,
                 null,
-                false,
+                true,
                 false);
     }