Improve newlines support (#26)

Fixes #24

src/Parlot/Character.cs

@@ -24,7 +24,6 @@ public static bool IsWhiteSpace(char ch)
         {
             return (ch == 32) || // space
                    (ch == '\t') || // horizontal tab
-                   (ch == '\v') || // vertical tab
                    (ch == 0xC) || // form feed - new page
                    (ch == 0xA0) || // non-breaking space
                    (ch >= 0x1680 && (
@@ -38,7 +37,7 @@ public static bool IsWhiteSpace(char ch)
         }
 
         public static bool IsWhiteSpaceOrNewLine(char ch)
-            => (ch == '\n') || (ch == '\r') || IsWhiteSpace(ch);
+            => (ch == '\n') || (ch == '\r') || (ch == '\v') || IsWhiteSpace(ch);
 
         public static char ScanHexEscape(string text, int index, out int length)
         {

src/Parlot/Fluent/Between.cs

@@ -45,7 +45,7 @@ public override bool Parse(ParseContext context, ref ParseResult<T> result)
             {
                 if (_beforeSkipWhiteSpace)
                 {
-                    context.Scanner.SkipWhiteSpace();
+                    context.SkipWhiteSpace();
                 }
 
                 if (!context.Scanner.ReadChar(_beforeChar))
@@ -72,7 +72,7 @@ public override bool Parse(ParseContext context, ref ParseResult<T> result)
             {
                 if (_afterSkipWhiteSpace)
                 {
-                    context.Scanner.SkipWhiteSpace();
+                    context.SkipWhiteSpace();
                 }
 
                 if (!context.Scanner.ReadChar(_afterChar))

src/Parlot/Fluent/NonWhiteSpaceLiteral.cs

@@ -3,25 +3,36 @@
     public sealed class NonWhiteSpaceLiteral : Parser<TextSpan>
     {
         private readonly bool _skipWhiteSpace;
+        private readonly bool _includeNewLines;
 
-        public NonWhiteSpaceLiteral(bool skipWhiteSpace = true)
+        public NonWhiteSpaceLiteral(bool skipWhiteSpace = true, bool includeNewLines = false)
         {
             _skipWhiteSpace = skipWhiteSpace;
+            _includeNewLines = includeNewLines;
         }
 
         public override bool Parse(ParseContext context, ref ParseResult<TextSpan> result)
         {
             if (_skipWhiteSpace)
             {
-                context.Scanner.SkipWhiteSpace();
+                context.SkipWhiteSpace();
+            }
+
+            if (context.Scanner.Cursor.Eof)
+            {
+                return false;
             }
 
             var start = context.Scanner.Cursor.Offset;
 
-            while (!context.Scanner.Cursor.Eof && context.Scanner.ReadNonWhiteSpace())
+            if (_includeNewLines)
             {
-                context.Scanner.Cursor.Advance();
+                context.Scanner.ReadNonWhiteSpace();
             }
+            else
+            {
+                context.Scanner.ReadNonWhiteSpaceOrNewLine();
+            }            
 
             var end = context.Scanner.Cursor.Offset;
 

src/Parlot/Fluent/ParseContext.cs

@@ -6,14 +6,24 @@ public class ParseContext
     {
         private ParseResult<TextSpan> _whiteSpaceResult = new();
 
+        /// <summary>
+        /// Whether new lines are treated as normal chars or white spaces.
+        /// </summary>
+        /// <remarks>
+        /// When <c>false</c>, new lines will be skipped like any other white space.
+        /// Otherwise white spaces need to be read explicitely by a rule.
+        /// </remarks>
+        public bool UseNewLines { get; private set; }
+
         /// <summary>
         /// The scanner used for the parsing session.
         /// </summary>
         public readonly Scanner Scanner;
 
-        public ParseContext(Scanner scanner)
+        public ParseContext(Scanner scanner, bool useNewLines = false)
         {
             Scanner = scanner ?? throw new ArgumentNullException(nameof(scanner));
+            UseNewLines = useNewLines;
         }
 
         /// <summary>
@@ -35,7 +45,14 @@ public void SkipWhiteSpace()
             }
             else
             {
-                Scanner.SkipWhiteSpace();
+                if (UseNewLines)
+                {
+                    Scanner.SkipWhiteSpace();
+                }
+                else
+                {
+                    Scanner.SkipWhiteSpaceOrNewLine();
+                }
             }
         }
 

src/Parlot/Fluent/Parser.cs

@@ -49,9 +49,9 @@ public abstract class Parser<T>
 
     public static class ParserExtensions
     {
-        public static T Parse<T>(this Parser<T> parser, string text)
+        public static T Parse<T>(this Parser<T> parser, string text, ParseContext context = null)
         {
-            var context = new ParseContext(new Scanner(text));
+            context ??= new ParseContext(new Scanner(text));
 
             var localResult = new ParseResult<T>();
 
@@ -70,7 +70,7 @@ public static bool TryParse<TResult>(this Parser<TResult> parser, string text, o
             return parser.TryParse(text, out value, out _);
         }
 
-        public static bool TryParse<TResult>(this Parser<TResult> parser, string text, out TResult value, out ParseError error)
+        public static bool TryParse<TResult>(this Parser<TResult> parser, string text,out TResult value, out ParseError error)
         {
             return TryParse(parser, new ParseContext(new Scanner(text)), out value, out error);
         }

src/Parlot/Fluent/Parsers.cs

@@ -82,7 +82,7 @@ public class LiteralBuilder
         /// <summary>
         /// Builds a parser that matches anything until whitespaces.
         /// </summary>
-        public Parser<TextSpan> NonWhiteSpace() => new NonWhiteSpaceLiteral(skipWhiteSpace: false);
+        public Parser<TextSpan> NonWhiteSpace(bool includeNewLines = false) => new NonWhiteSpaceLiteral(skipWhiteSpace: false, includeNewLines: includeNewLines);
 
         /// <summary>
         /// Builds a parser that matches the specified text.
@@ -120,7 +120,7 @@ public class TermBuilder
         /// <summary>
         /// Builds a parser that matches anything until whitespaces.
         /// </summary>
-        public Parser<TextSpan> NonWhiteSpace() => new NonWhiteSpaceLiteral();
+        public Parser<TextSpan> NonWhiteSpace(bool includeNewLines = false) => new NonWhiteSpaceLiteral(includeNewLines: includeNewLines);
 
         /// <summary>
         /// Builds a parser that matches the specified text.

src/Parlot/Fluent/Separated.cs

@@ -66,7 +66,7 @@ public override bool Parse(ParseContext context, ref ParseResult<List<T>> result
                 {
                     if (_separatorWhiteSpace)
                     {
-                        context.Scanner.SkipWhiteSpace();
+                        context.SkipWhiteSpace();
                     }
 
                     if (!context.Scanner.ReadChar(_separatorChar))

src/Parlot/Scanner.cs

@@ -194,6 +194,11 @@ public bool ReadNonWhiteSpace(TokenResult result = null)
             return ReadWhile(static x => !Character.IsWhiteSpace(x), result);
         }
 
+        public bool ReadNonWhiteSpaceOrNewLine(TokenResult result = null)
+        {
+            return ReadWhile(static x => !Character.IsWhiteSpaceOrNewLine(x), result);
+        }
+
         /// <summary>
         /// Reads the specified text.
         /// </summary>

test/Parlot.Tests/FluentTests.cs

@@ -70,6 +70,12 @@ public void BetweenShouldParseBetweenTwoString()
             Assert.False(code.TryParse("[[123]", out _));
         }
 
+        [Fact]
+        public void ParseContextShouldUseNewLines()
+        {
+            Assert.Equal("a", Terms.NonWhiteSpace().Parse("\n\r\v a"));
+        }
+
         [Fact]
         public void LiteralsShouldNotSkipWhiteSpaceByDefault()
         {
@@ -281,5 +287,21 @@ public void OneOfShouldRestorePosition()
             Assert.Equal("abcd", choice.Parse("abcd"));
             Assert.Equal("abed", choice.Parse("abed"));
         }
+
+        [Fact]
+        public void NonWhiteSpaceShouldStopAtSpaceOrEof()
+        {
+            Assert.Equal("a", Terms.NonWhiteSpace().Parse(" a"));
+            Assert.Equal("a", Terms.NonWhiteSpace().Parse(" a "));
+            Assert.Equal("a", Terms.NonWhiteSpace().Parse(" a b"));
+            Assert.Equal("a", Terms.NonWhiteSpace().Parse("a b"));
+            Assert.Equal("abc", Terms.NonWhiteSpace().Parse("abc b"));
+            Assert.Equal("abc", Terms.NonWhiteSpace().Parse("abc\nb"));
+            Assert.Equal("abc\nb", Terms.NonWhiteSpace(true).Parse("abc\nb"));
+            Assert.Equal("abc", Terms.NonWhiteSpace().Parse("abc"));
+
+            Assert.False(Terms.NonWhiteSpace().TryParse("", out _));
+            Assert.False(Terms.NonWhiteSpace().TryParse(" ", out _));
+        }
     }
 }

test/Parlot.Tests/ScannerTests.cs

@@ -59,6 +59,7 @@ public void ShouldNotReadStringWithInvalidEscapes(string text)
         [Fact]
         public void SkipWhitespaceShouldSkipWhitespace()
         {
+            // New lines are not considered white spaces
             Scanner s = new("Lorem ipsum   \r\n   ");
 
             Assert.False(s.SkipWhiteSpace());