From 088cf994934a4c9244e302b9473284bd87b74634 Mon Sep 17 00:00:00 2001 From: Atif Aziz Date: Wed, 30 Oct 2024 00:02:29 +0100 Subject: [PATCH] Rename tokenizers that are actually parsers (#298) --- .../Parser/PythonParser.Args.cs | 14 +++---- .../Parser/PythonParser.Function.cs | 8 ++-- .../Parser/PythonParser.Parameters.cs | 14 +++---- .../Parser/PythonParser.TypeDef.cs | 4 +- src/CSnakes.Tests/TokenizerTests.cs | 38 +++++++++---------- src/CSnakes.Tests/TypeReflectionTests.cs | 2 +- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/CSnakes.SourceGeneration/Parser/PythonParser.Args.cs b/src/CSnakes.SourceGeneration/Parser/PythonParser.Args.cs index 7bf5579a..7cd941c4 100644 --- a/src/CSnakes.SourceGeneration/Parser/PythonParser.Args.cs +++ b/src/CSnakes.SourceGeneration/Parser/PythonParser.Args.cs @@ -5,26 +5,26 @@ namespace CSnakes.Parser; public static partial class PythonParser { - public static TokenListParser PythonStarArgTokenizer { get; } = + public static TokenListParser PythonStarArgParser { get; } = (from star in Token.EqualTo(PythonToken.Asterisk) from name in Token.EqualTo(PythonToken.Identifier).Optional() select new PythonParameterType(name.HasValue ? name.Value.ToStringValue() : "args", PythonFunctionParameterType.Star)) .Named("Star Arg"); - public static TokenListParser PythonDoubleStarArgTokenizer { get; } = + public static TokenListParser PythonDoubleStarArgParser { get; } = (from star in Token.EqualTo(PythonToken.DoubleAsterisk) from name in Token.EqualTo(PythonToken.Identifier) select new PythonParameterType(name.ToStringValue(), PythonFunctionParameterType.DoubleStar)) .Named("Double Star Arg"); - public static TokenListParser PythonNormalArgTokenizer { get; } = + public static TokenListParser PythonNormalArgParser { get; } = (from name in Token.EqualTo(PythonToken.Identifier) select new PythonParameterType(name.ToStringValue(), PythonFunctionParameterType.Normal)) .Named("Normal Arg"); - public static TokenListParser PythonArgTokenizer { get; } = - PythonStarArgTokenizer.AsNullable() - .Or(PythonDoubleStarArgTokenizer.AsNullable()) - .Or(PythonNormalArgTokenizer.AsNullable()) + public static TokenListParser PythonArgParser { get; } = + PythonStarArgParser.AsNullable() + .Or(PythonDoubleStarArgParser.AsNullable()) + .Or(PythonNormalArgParser.AsNullable()) .Named("Arg"); } diff --git a/src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs b/src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs index 486ee04c..b13a031a 100644 --- a/src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs +++ b/src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs @@ -9,11 +9,11 @@ namespace CSnakes.Parser; public static partial class PythonParser { - public static TokenListParser PythonFunctionDefinitionTokenizer { get; } = + public static TokenListParser PythonFunctionDefinitionParser { get; } = (from def in Token.EqualTo(PythonToken.Def) from name in Token.EqualTo(PythonToken.Identifier) - from parameters in PythonParameterListTokenizer.AssumeNotNull() - from arrow in Token.EqualTo(PythonToken.Arrow).Optional().Then(returnType => PythonTypeDefinitionTokenizer.AssumeNotNull().OptionalOrDefault()) + from parameters in PythonParameterListParser.AssumeNotNull() + from arrow in Token.EqualTo(PythonToken.Arrow).Optional().Then(returnType => PythonTypeDefinitionParser.AssumeNotNull().OptionalOrDefault()) from colon in Token.EqualTo(PythonToken.Colon) select new PythonFunctionDefinition(name.ToStringValue(), arrow, parameters)) .Named("Function Definition"); @@ -90,7 +90,7 @@ public static bool TryParseFunctionDefinitions(SourceText source, out PythonFunc foreach ((IEnumerable currentLines, ParsedTokens tokens) in functionLines) { TokenListParserResult functionDefinition = - PythonFunctionDefinitionTokenizer.TryParse(tokens); + PythonFunctionDefinitionParser.TryParse(tokens); if (functionDefinition.HasValue) { functionDefinitions.Add(functionDefinition.Value); diff --git a/src/CSnakes.SourceGeneration/Parser/PythonParser.Parameters.cs b/src/CSnakes.SourceGeneration/Parser/PythonParser.Parameters.cs index dce0cc28..6b6be7f2 100644 --- a/src/CSnakes.SourceGeneration/Parser/PythonParser.Parameters.cs +++ b/src/CSnakes.SourceGeneration/Parser/PythonParser.Parameters.cs @@ -10,10 +10,10 @@ public static partial class PythonParser .Select(d => new PythonFunctionParameter("/", null, null, PythonFunctionParameterType.Slash)) .Named("Positional Only Signal"); - public static TokenListParser PythonParameterTokenizer { get; } = - (from arg in PythonArgTokenizer + public static TokenListParser PythonParameterParser { get; } = + (from arg in PythonArgParser from type in Token.EqualTo(PythonToken.Colon).Optional().Then( - _ => PythonTypeDefinitionTokenizer.AssumeNotNull().OptionalOrDefault() + _ => PythonTypeDefinitionParser.AssumeNotNull().OptionalOrDefault() ) from defaultValue in Token.EqualTo(PythonToken.Equal).Optional().Then( _ => ConstantValueTokenizer.AssumeNotNull().OptionalOrDefault() @@ -21,14 +21,14 @@ from defaultValue in Token.EqualTo(PythonToken.Equal).Optional().Then( select new PythonFunctionParameter(arg.Name, type, defaultValue, arg.ParameterType)) .Named("Parameter"); - public static TokenListParser ParameterOrSlash { get; } = + public static TokenListParser ParameterOrSlashParser { get; } = PositionalOnlyParameterParser.AsNullable() - .Or(PythonParameterTokenizer.AsNullable()) + .Or(PythonParameterParser.AsNullable()) .Named("Parameter"); - public static TokenListParser PythonParameterListTokenizer { get; } = + public static TokenListParser PythonParameterListParser { get; } = (from openParen in Token.EqualTo(PythonToken.OpenParenthesis) - from parameters in ParameterOrSlash.ManyDelimitedBy(Token.EqualTo(PythonToken.Comma)) + from parameters in ParameterOrSlashParser.ManyDelimitedBy(Token.EqualTo(PythonToken.Comma)) from closeParen in Token.EqualTo(PythonToken.CloseParenthesis) select parameters) .Named("Parameter List"); diff --git a/src/CSnakes.SourceGeneration/Parser/PythonParser.TypeDef.cs b/src/CSnakes.SourceGeneration/Parser/PythonParser.TypeDef.cs index 286d71e8..4642ee57 100644 --- a/src/CSnakes.SourceGeneration/Parser/PythonParser.TypeDef.cs +++ b/src/CSnakes.SourceGeneration/Parser/PythonParser.TypeDef.cs @@ -13,12 +13,12 @@ public static partial class PythonParser .AtLeastOnceDelimitedBy(Character.EqualTo('.')) ); - public static TokenListParser PythonTypeDefinitionTokenizer { get; } = + public static TokenListParser PythonTypeDefinitionParser { get; } = (from name in Token.EqualTo(PythonToken.Identifier).Or(Token.EqualTo(PythonToken.None)).OptionalOrDefault() #pragma warning disable CS8620 from openBracket in Token.EqualTo(PythonToken.OpenBracket) .Then(_ => - PythonTypeDefinitionTokenizer + PythonTypeDefinitionParser .AssumeNotNull() .ManyDelimitedBy( Token.EqualTo(PythonToken.Comma), diff --git a/src/CSnakes.Tests/TokenizerTests.cs b/src/CSnakes.Tests/TokenizerTests.cs index 1aeca75a..d08c85f5 100644 --- a/src/CSnakes.Tests/TokenizerTests.cs +++ b/src/CSnakes.Tests/TokenizerTests.cs @@ -90,7 +90,7 @@ public void TokenizeWithDefaultValue() public void ParseFunctionParameter(string code, string expectedName, string expectedType) { var tokens = PythonTokenizer.Instance.Tokenize(code); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal(expectedName, result.Value.Name); Assert.Equal(expectedType, result.Value.Type.ToString()); @@ -100,7 +100,7 @@ public void ParseFunctionParameter(string code, string expectedName, string expe public void ParseFunctionParameterNoType() { var tokens = PythonTokenizer.Instance.Tokenize("a"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.False(result.Value.HasTypeAnnotation()); @@ -110,7 +110,7 @@ public void ParseFunctionParameterNoType() public void ParseFunctionParameterDefault() { var tokens = PythonTokenizer.Instance.Tokenize("a = 1"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal("1", result.Value.DefaultValue?.ToString()); @@ -125,7 +125,7 @@ public void ParseFunctionParameterDefault() public void ParseFunctionParameterDefaultValuesNoType(string value) { var tokens = PythonTokenizer.Instance.Tokenize($"a = {value}"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal(value, result.Value.DefaultValue?.ToString()); @@ -136,7 +136,7 @@ public void ParseFunctionParameterDefaultValuesNoType(string value) public void ParseFunctionParameterDefaultSingleQuotedString() { var tokens = PythonTokenizer.Instance.Tokenize($"a = 'hello'"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal("hello", result.Value.DefaultValue?.ToString()); @@ -147,7 +147,7 @@ public void ParseFunctionParameterDefaultSingleQuotedString() public void ParseFunctionParameterDefaultDoubleQuotedString() { var tokens = PythonTokenizer.Instance.Tokenize($"a = \"hello\""); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal("hello", result.Value.DefaultValue?.ToString()); @@ -158,7 +158,7 @@ public void ParseFunctionParameterDefaultDoubleQuotedString() public void ParseFunctionParameterDefaultDouble() { var tokens = PythonTokenizer.Instance.Tokenize($"a: float = -1.1"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal("-1.1", result.Value.DefaultValue?.ToString()); @@ -169,7 +169,7 @@ public void ParseFunctionParameterDefaultDouble() public void ParseFunctionParameterDefaultInt() { var tokens = PythonTokenizer.Instance.Tokenize($"a: int = 1234"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal("1234", result.Value.DefaultValue?.ToString()); @@ -180,7 +180,7 @@ public void ParseFunctionParameterDefaultInt() public void ParseFunctionParameterDefaultBoolTrue() { var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = True"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal("True", result.Value.DefaultValue?.ToString()); @@ -191,7 +191,7 @@ public void ParseFunctionParameterDefaultBoolTrue() public void ParseFunctionParameterDefaultBoolFalse() { var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = False"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal("False", result.Value.DefaultValue?.ToString()); @@ -202,7 +202,7 @@ public void ParseFunctionParameterDefaultBoolFalse() public void ParseFunctionParameterDefaultNone() { var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = None"); - var result = PythonParser.PythonParameterTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value.Name); Assert.Equal("None", result.Value.DefaultValue?.ToString()); @@ -214,7 +214,7 @@ public void ParseFunctionParameterListSingleGeneric() { var code = "(a: list[int])"; var tokens = PythonTokenizer.Instance.Tokenize(code); - var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterListParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value[0].Name); Assert.Equal("list[int]", result.Value[0].Type.ToString()); @@ -225,7 +225,7 @@ public void ParseFunctionParameterListDoubleGeneric() { var code = "(a: list[int], b)"; var tokens = PythonTokenizer.Instance.Tokenize(code); - var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterListParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value[0].Name); Assert.Equal("list[int]", result.Value[0].Type.ToString()); @@ -236,7 +236,7 @@ public void ParseFunctionParameterListQualifiedGenericType() { var code = "(a: typing.List[int], b)"; var tokens = PythonTokenizer.Instance.Tokenize(code); - var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterListParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value[0].Name); Assert.Equal("typing.List[int]", result.Value[0].Type.ToString()); @@ -247,7 +247,7 @@ public void ParseFunctionParameterListQualifiedBasicType() { var code = "(a: np.ndarray, b)"; var tokens = PythonTokenizer.Instance.Tokenize(code); - var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterListParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value[0].Name); Assert.Equal("np.ndarray", result.Value[0].Type.ToString()); @@ -258,7 +258,7 @@ public void ParseFunctionParameterListEasy() { var code = "(a: int, b: float, c: str)"; var tokens = PythonTokenizer.Instance.Tokenize(code); - var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterListParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value[0].Name); Assert.Equal("int", result.Value[0].Type.Name); @@ -273,7 +273,7 @@ public void ParseFunctionParameterListUntyped() { var code = "(a, b, c)"; var tokens = PythonTokenizer.Instance.Tokenize(code); - var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens); + var result = PythonParser.PythonParameterListParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("a", result.Value[0].Name); Assert.Equal("Any", result.Value[0].Type.Name); @@ -287,7 +287,7 @@ public void ParseFunctionParameterListUntyped() public void ParseFunctionDefinition() { var tokens = PythonTokenizer.Instance.Tokenize("def foo(a: int, b: str) -> None:"); - var result = PythonParser.PythonFunctionDefinitionTokenizer.TryParse(tokens); + var result = PythonParser.PythonFunctionDefinitionParser.TryParse(tokens); Assert.True(result.HasValue); Assert.Equal("foo", result.Value.Name); @@ -313,7 +313,7 @@ public void ParseFunctionDefinitionComplex(string code) { var tokens = PythonTokenizer.Instance.Tokenize(code); Assert.True(tokens.IsAtEnd == false, "Tokenize failed"); - var result = PythonParser.PythonFunctionDefinitionTokenizer.TryParse(tokens); + var result = PythonParser.PythonFunctionDefinitionParser.TryParse(tokens); Assert.True(result.HasValue, result.ToString()); } diff --git a/src/CSnakes.Tests/TypeReflectionTests.cs b/src/CSnakes.Tests/TypeReflectionTests.cs index d256e62d..e89631db 100644 --- a/src/CSnakes.Tests/TypeReflectionTests.cs +++ b/src/CSnakes.Tests/TypeReflectionTests.cs @@ -70,7 +70,7 @@ public void TupleParsingTest(string pythonType, string expectedType) => private static void ParsingTestInternal(string pythonType, string expectedType) { var tokens = PythonTokenizer.Instance.Tokenize(pythonType); - var result = PythonParser.PythonTypeDefinitionTokenizer.TryParse(tokens); + var result = PythonParser.PythonTypeDefinitionParser.TryParse(tokens); Assert.True(result.HasValue, result.ToString()); Assert.NotNull(result.Value); var reflectedType = TypeReflection.AsPredefinedType(result.Value, TypeReflection.ConversionDirection.FromPython);