Fix a panic when parsing unicode characters after a string delimiter

Cargo.toml

@@ -36,7 +36,7 @@ features = ["derive"]
 
 [dependencies.uwl]
 optional = true
-version = "0.5"
+version = "0.6.0"
 
 [dependencies.base64]
 optional = true

src/framework/standard/args.rs

@@ -1,5 +1,6 @@
 use uwl::Stream;
 
+use std::borrow::Cow;
 use std::cell::Cell;
 use std::error::Error as StdError;
 use std::marker::PhantomData;
@@ -53,6 +54,16 @@ pub enum Delimiter {
     Multiple(String),
 }
 
+impl Delimiter {
+    #[inline]
+    fn to_str(&self) -> Cow<'_, str> {
+        match self {
+            Delimiter::Single(c) => Cow::Owned(c.to_string()),
+            Delimiter::Multiple(s) => Cow::Borrowed(s),
+        }
+    }
+}
+
 impl From<char> for Delimiter {
     #[inline]
     fn from(c: char) -> Delimiter {
@@ -83,7 +94,6 @@ impl<'a> From<&'a str> for Delimiter {
 
 #[derive(Debug, Clone, Copy, PartialEq)]
 enum TokenKind {
-    Delimiter,
     Argument,
     QuotedArgument,
 }
@@ -101,41 +111,19 @@ impl Token {
     }
 }
 
-fn lex(stream: &mut Stream<'_>, delims: &[&Delimiter]) -> Option<Token> {
-    if stream.at_end() {
+fn lex(stream: &mut Stream<'_>, delims: &[Cow<'_, str>]) -> Option<Token> {
+    if stream.is_empty() {
         return None;
     }
 
-    for delim in delims {
-        match delim {
-            Delimiter::Single(c) => {
-                if stream.current()? == *c {
-                    let start = stream.offset();
-                    stream.next();
-                    return Some(Token::new(TokenKind::Delimiter, start, c.len_utf8()));
-                }
-            }
-            Delimiter::Multiple(s) => {
-                if *s == stream.peek_for(s.chars().count()) {
-                    let start = stream.offset();
-                    let end = start + s.len();
-
-                    // Move the offset pointer by `s.len()` bytes.
-                    stream.set(end);
-
-                    return Some(Token::new(TokenKind::Delimiter, start, end));
-                }
-            }
-        }
-    }
+    let start = stream.offset();
 
-    if stream.current()? == '"' {
-        let start = stream.offset();
+    if stream.current()? == b'"' {
         stream.next();
 
-        stream.take_until(|s| s == '"');
+        stream.take_until(|b| b == b'"');
 
-        let is_quote = stream.current().map_or(false, |s| s == '"');
+        let is_quote = stream.current().map_or(false, |b| b == b'"');
         stream.next();
 
         let end = stream.offset();
@@ -144,32 +132,26 @@ fn lex(stream: &mut Stream<'_>, delims: &[&Delimiter]) -> Option<Token> {
             Token::new(TokenKind::QuotedArgument, start, end)
         } else {
             // We're missing an end quote. View this as a normal argument.
-            Token::new(TokenKind::Argument, start, stream.source().len())
+            Token::new(TokenKind::Argument, start, stream.len())
         });
     }
 
-    let start = stream.offset();
+    let mut end = start;
 
-    'outer: while !stream.at_end() {
+    'outer: while !stream.is_empty() {
         for delim in delims {
-            match delim {
-                Delimiter::Single(c) => {
-                    if stream.current()? == *c {
-                        break 'outer;
-                    }
-                }
-                Delimiter::Multiple(s) => {
-                    if *s == stream.peek_for(s.chars().count()) {
-                        break 'outer;
-                    }
-                }
+            end = stream.offset();
+
+            if stream.eat(&delim) {
+                break 'outer;
             }
         }
 
-        stream.next();
+        stream.next_char();
+        end = stream.offset();
     }
 
-    Some(Token::new(TokenKind::Argument, start, stream.offset()))
+    Some(Token::new(TokenKind::Argument, start, end))
 }
 
 fn remove_quotes(s: &str) -> &str {
@@ -320,6 +302,7 @@ impl Args {
                 Delimiter::Single(c) => message.contains(*c),
                 Delimiter::Multiple(s) => message.contains(s),
             })
+            .map(|delim| delim.to_str())
             .collect::<Vec<_>>();
 
         let args = if delims.is_empty() && !message.is_empty() {
@@ -336,10 +319,6 @@ impl Args {
             let mut stream = Stream::new(message);
 
             while let Some(token) = lex(&mut stream, &delims) {
-                if token.kind == TokenKind::Delimiter {
-                    continue;
-                }
-
                 args.push(token);
             }
 

src/framework/standard/mod.rs

@@ -625,7 +625,7 @@ impl Framework for StandardFramework {
     fn dispatch(&mut self, mut ctx: Context, msg: Message, threadpool: &ThreadPool) {
         let mut stream = Stream::new(&msg.content);
 
-        stream.take_while(|s| s.is_whitespace());
+        stream.take_while_char(|c| c.is_whitespace());
 
         let prefix = parse::prefix(&mut ctx, &msg, &mut stream, &self.config);
 

src/framework/standard/parse/mod.rs

@@ -35,7 +35,7 @@ pub fn mention<'a>(stream: &mut Stream<'a>, config: &Configuration) -> Option<&'
     // Optional.
     stream.eat("!");
 
-    let id = stream.take_while(|s| s.is_numeric());
+    let id = stream.take_while(|b| b.is_ascii_digit());
 
     if !stream.eat(">") {
         // Backtrack to where we were.
@@ -60,7 +60,7 @@ fn find_prefix<'a>(
     stream: &Stream<'a>,
 ) -> Option<Cow<'a, str>> {
     let try_match = |prefix: &str| {
-        let peeked = stream.peek_for(prefix.chars().count());
+        let peeked = stream.peek_for_char(prefix.chars().count());
         let peeked = to_lowercase(config, peeked);
 
         if prefix == &peeked {
@@ -100,7 +100,7 @@ pub fn prefix<'a>(
     config: &Configuration,
 ) -> Option<Cow<'a, str>> {
     if let Some(id) = mention(stream, config) {
-        stream.take_while(|s| s.is_whitespace());
+        stream.take_while_char(|c| c.is_whitespace());
 
         return Some(Cow::Borrowed(id));
     }
@@ -112,7 +112,7 @@ pub fn prefix<'a>(
     }
 
     if config.with_whitespace.prefixes {
-        stream.take_while(|s| s.is_whitespace());
+        stream.take_while_char(|c| c.is_whitespace());
     }
 
     prefix
@@ -175,13 +175,13 @@ fn try_parse<M: ParseMap>(
     f: impl Fn(&str) -> String,
 ) -> (String, Option<M::Storage>) {
     if by_space {
-        let n = f(stream.peek_until(|s| s.is_whitespace()));
+        let n = f(stream.peek_until_char(|c| c.is_whitespace()));
 
         let o = map.get(&n);
 
         (n, o)
     } else {
-        let mut n = f(stream.peek_for(map.max_length()));
+        let mut n = f(stream.peek_for_char(map.max_length()));
         let mut o = None;
 
         for _ in 0..(map.max_length() - map.min_length()) {
@@ -217,7 +217,7 @@ fn parse_cmd(
         stream.increment(n.len());
 
         if config.with_whitespace.commands {
-            stream.take_while(|s| s.is_whitespace());
+            stream.take_while_char(|c| c.is_whitespace());
         }
 
         check_discrepancy(ctx, msg, config, &cmd.options)?;
@@ -248,7 +248,7 @@ fn parse_group(
         stream.increment(n.len());
 
         if config.with_whitespace.groups {
-            stream.take_while(|s| s.is_whitespace());
+            stream.take_while_char(|c| c.is_whitespace());
         }
 
         check_discrepancy(ctx, msg, config, &group.options)?;
@@ -329,12 +329,12 @@ pub fn command(
     // Precedence is taken over commands named as one of the help names.
     if let Some(names) = help_was_set {
         for name in names {
-            let n = to_lowercase(config, stream.peek_for(name.chars().count()));
+            let n = to_lowercase(config, stream.peek_for_char(name.chars().count()));
 
             if name == &n {
                 stream.increment(n.len());
 
-                stream.take_while(|s| s.is_whitespace());
+                stream.take_while_char(|c| c.is_whitespace());
 
                 return Ok(Invoke::Help(name));
             }