Skip to content

Commit

Permalink
Add parsing support for Unicode escape sequences
Browse files Browse the repository at this point in the history
While the lexer already supported Unicode escape sequences, I forgot to
add support for this in the parser. This commit fixes this, so you can
now use strings like this:

    foo\u{AC}bar

Changelog: added
  • Loading branch information
yorickpeterse committed Oct 6, 2022
1 parent 2609bbc commit 441b576
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 2 deletions.
40 changes: 38 additions & 2 deletions ast/src/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1617,7 +1617,7 @@ impl Parser {

return Ok(Expression::DoubleString(Box::new(string)));
}
TokenKind::StringText => {
TokenKind::StringText | TokenKind::UnicodeEscape => {
values.push(DoubleStringValue::Text(Box::new(
self.string_text(token),
)));
Expand Down Expand Up @@ -1651,7 +1651,10 @@ impl Parser {
let mut value = start.value;
let mut end_loc = start.location.clone();

while self.peek().kind == TokenKind::StringText {
while matches!(
self.peek().kind,
TokenKind::StringText | TokenKind::UnicodeEscape
) {
let token = self.next();

value += &token.value;
Expand Down Expand Up @@ -5915,6 +5918,39 @@ mod tests {
location: cols(1, 10)
}))
);

assert_eq!(
expr("\"foo\\u{AC}bar\""),
Expression::DoubleString(Box::new(DoubleStringLiteral {
values: vec![DoubleStringValue::Text(Box::new(StringText {
value: "foo\u{AC}bar".to_string(),
location: location(1..=1, 2..=13)
})),],
location: location(1..=1, 1..=14)
}))
);

assert_eq!(
expr("\"foo\\u{AC}\""),
Expression::DoubleString(Box::new(DoubleStringLiteral {
values: vec![DoubleStringValue::Text(Box::new(StringText {
value: "foo\u{AC}".to_string(),
location: location(1..=1, 2..=10)
})),],
location: location(1..=1, 1..=11)
}))
);

assert_eq!(
expr("\"\\u{AC}bar\""),
Expression::DoubleString(Box::new(DoubleStringLiteral {
values: vec![DoubleStringValue::Text(Box::new(StringText {
value: "\u{AC}bar".to_string(),
location: location(1..=1, 2..=10)
})),],
location: location(1..=1, 1..=11)
}))
);
}

#[test]
Expand Down
7 changes: 7 additions & 0 deletions docs/source/getting-started/syntax.md
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,13 @@ bar \
baz" # => "foo bar baz"
```

Double quoted strings support Unicode escape sequences using the syntax
`\u{XXXXX}`, such as this:

```inko
"foo\u{AC}bar"
```

### Integers

The syntax for integers is as follows:
Expand Down

0 comments on commit 441b576

Please sign in to comment.