Fix causal masks dtype (#443)

EricLBuehler · Jun 16, 2024 · 3a79137 · 3a79137
1 parent d355714
commit 3a79137
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 4 deletions.
diff --git a/mistralrs-core/src/models/quantized_phi2.rs b/mistralrs-core/src/models/quantized_phi2.rs
@@ -276,7 +276,7 @@ impl ModelWeights {
         let mask = CausalMasker.make_causal_mask_as_attn_bias(
             input_ids,
             &cache,
-            xs.dtype(),
+            DType::F32,
             self.layers[0].n_head,
         )?;
         for (i, layer) in self.layers.iter_mut().enumerate() {

diff --git a/mistralrs-core/src/models/quantized_phi3.rs b/mistralrs-core/src/models/quantized_phi3.rs
@@ -309,7 +309,7 @@ impl ModelWeights {
             input_ids,
             &cache,
             Some(self.max_seq_len),
-            xs.dtype(),
+            DType::F32,
             self.layers[0].n_head,
         )?;
         for (i, layer) in self.layers.iter_mut().enumerate() {

diff --git a/mistralrs-core/src/xlora_models/quantized_llama.rs b/mistralrs-core/src/xlora_models/quantized_llama.rs
@@ -761,7 +761,7 @@ impl ModelWeights {
         let mask = CausalMasker.make_causal_mask_as_attn_bias(
             x,
             &cache,
-            x.dtype(),
+            DType::F32,
             self.layers[0].n_head,
         )?;
         for (i, layer) in self.layers.iter_mut().enumerate() {

diff --git a/mistralrs-core/src/xlora_models/quantized_phi3.rs b/mistralrs-core/src/xlora_models/quantized_phi3.rs
@@ -395,7 +395,7 @@ impl ModelWeights {
             input_ids,
             &cache,
             Some(self.max_seq_len),
-            xs.dtype(),
+            DType::F32,
             self.layers[0].n_head,
         )?;
         for (i, layer) in self.layers.iter_mut().enumerate() {