diff --git a/csharp/Apache.Arrow.sln b/csharp/Apache.Arrow.sln index 8498c8a385b24..4e4175c9180db 100644 --- a/csharp/Apache.Arrow.sln +++ b/csharp/Apache.Arrow.sln @@ -17,6 +17,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Flight", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Flight.AspNetCore", "src\Apache.Arrow.Flight.AspNetCore\Apache.Arrow.Flight.AspNetCore.csproj", "{E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{1D1EF692-F180-40D2-AAD3-315EF1A4D1B9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.IntegrationTest", "test\Apache.Arrow.IntegrationTest\Apache.Arrow.IntegrationTest.csproj", "{E8264B7F-B680-4A55-939B-85DB628164BB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -51,6 +55,10 @@ Global {E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}.Debug|Any CPU.Build.0 = Debug|Any CPU {E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}.Release|Any CPU.ActiveCfg = Release|Any CPU {E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}.Release|Any CPU.Build.0 = Release|Any CPU + {E8264B7F-B680-4A55-939B-85DB628164BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E8264B7F-B680-4A55-939B-85DB628164BB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E8264B7F-B680-4A55-939B-85DB628164BB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E8264B7F-B680-4A55-939B-85DB628164BB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -58,4 +66,7 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FD0BB617-6031-4844-B99D-B331E335B572} EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {E8264B7F-B680-4A55-939B-85DB628164BB} = {1D1EF692-F180-40D2-AAD3-315EF1A4D1B9} + EndGlobalSection EndGlobal diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs index 7ef9813265a83..dcb56f05ec7a4 100644 --- a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs +++ b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs @@ -21,6 +21,7 @@ using System.IO; using System.Threading; using System.Threading.Tasks; +using Apache.Arrow.Arrays; using Apache.Arrow.Types; using FlatBuffers; @@ -46,6 +47,7 @@ internal class ArrowRecordBatchFlatBufferBuilder : IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, + IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, @@ -107,6 +109,12 @@ public void Visit(BinaryArray array) _buffers.Add(CreateBuffer(array.ValueBuffer)); } + public void Visit(FixedSizeBinaryArray array) + { + _buffers.Add(CreateBuffer(array.NullBitmapBuffer)); + _buffers.Add(CreateBuffer(array.ValueBuffer)); + } + public void Visit(Decimal128Array array) { _buffers.Add(CreateBuffer(array.NullBitmapBuffer)); diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs b/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs index 23f5b3f3d9c84..6b147a1faa9b3 100644 --- a/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs +++ b/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs @@ -64,7 +64,8 @@ class TypeVisitor : IArrowTypeVisitor, IArrowTypeVisitor, IArrowTypeVisitor, - IArrowTypeVisitor + IArrowTypeVisitor, + IArrowTypeVisitor { private FlatBufferBuilder Builder { get; } @@ -209,6 +210,14 @@ public void Visit(DictionaryType type) // type in the DictionaryEncoding metadata in the parent field type.ValueType.Accept(this); } + + public void Visit(FixedSizeBinaryType type) + { + Flatbuf.FixedSizeBinary.StartFixedSizeBinary(Builder); + Result = FieldType.Build( + Flatbuf.Type.FixedSizeBinary, + Flatbuf.FixedSizeBinary.EndFixedSizeBinary(Builder)); + } public void Visit(IArrowType type) { diff --git a/csharp/test/Apache.Arrow.IntegrationTest/Apache.Arrow.IntegrationTest.csproj b/csharp/test/Apache.Arrow.IntegrationTest/Apache.Arrow.IntegrationTest.csproj new file mode 100644 index 0000000000000..85bbb7e98e1a2 --- /dev/null +++ b/csharp/test/Apache.Arrow.IntegrationTest/Apache.Arrow.IntegrationTest.csproj @@ -0,0 +1,15 @@ + + + + + Exe + netcoreapp3.1 + + + + + + + + + \ No newline at end of file diff --git a/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs b/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs new file mode 100644 index 0000000000000..a6073f29da852 --- /dev/null +++ b/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs @@ -0,0 +1,425 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Apache.Arrow.Arrays; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; + +namespace Apache.Arrow.IntegrationTest +{ + public class IntegrationCommand + { + public string Mode { get; set; } + public FileInfo JsonFileInfo { get; set; } + public FileInfo ArrowFileInfo { get; set; } + + public IntegrationCommand(string mode, FileInfo jsonFileInfo, FileInfo arrowFileInfo) + { + Mode = mode; + JsonFileInfo = jsonFileInfo; + ArrowFileInfo = arrowFileInfo; + } + + public async Task Execute() + { + Func commandDelegate = Mode switch + { + "validate" => Validate, + "json-to-arrow" => JsonToArrow, + "stream-to-file" => StreamToFile, + "file-to-stream" => FileToStream, + _ => () => + { + Console.WriteLine($"Mode '{Mode}' is not supported."); + return Task.CompletedTask; + } + }; + await commandDelegate(); + } + + private Task Validate() + { + return Task.CompletedTask; + } + + private async Task JsonToArrow() + { + JsonFile jsonFile = await ParseJsonFile(); + Schema schema = CreateSchema(jsonFile.Schema); + + using (FileStream fs = ArrowFileInfo.Create()) + { + ArrowFileWriter writer = new ArrowFileWriter(fs, schema); + foreach (var jsonRecordBatch in jsonFile.Batches) + { + RecordBatch batch = CreateRecordBatch(schema, jsonRecordBatch); + await writer.WriteRecordBatchAsync(batch); + } + await writer.WriteEndAsync(); + } + } + + private RecordBatch CreateRecordBatch(Schema schema, JsonRecordBatch jsonRecordBatch) + { + if (schema.Fields.Count != jsonRecordBatch.Columns.Count) + { + throw new NotSupportedException($"jsonRecordBatch.Columns.Count '{jsonRecordBatch.Columns.Count}' doesn't match schema field count '{schema.Fields.Count}'"); + } + + List arrays = new List(jsonRecordBatch.Columns.Count); + for (int i = 0; i < jsonRecordBatch.Columns.Count; i++) + { + JsonFieldData data = jsonRecordBatch.Columns[i]; + Field field = schema.GetFieldByName(data.Name); + ArrayCreator creator = new ArrayCreator(data); + field.DataType.Accept(creator); + arrays.Add(creator.Array); + } + + return new RecordBatch(schema, arrays, jsonRecordBatch.Count); + } + + private static Schema CreateSchema(JsonSchema jsonSchema) + { + Schema.Builder builder = new Schema.Builder(); + for (int i = 0; i < jsonSchema.Fields.Count; i++) + { + builder.Field(f => CreateField(f, jsonSchema.Fields[i])); + } + return builder.Build(); + } + + private static void CreateField(Field.Builder builder, JsonField jsonField) + { + builder.Name(jsonField.Name) + .DataType(ToArrowType(jsonField.Type)) + .Nullable(jsonField.Nullable); + + if (jsonField.Metadata != null) + { + builder.Metadata(jsonField.Metadata); + } + } + + private static IArrowType ToArrowType(JsonArrowType type) + { + return type.Name switch + { + "bool" => BooleanType.Default, + "int" => ToIntArrowType(type), + "floatingpoint" => ToFloatingPointArrowType(type), + "binary" => BinaryType.Default, + "utf8" => StringType.Default, + "fixedsizebinary" => new FixedSizeBinaryType(type.ByteWidth), + _ => throw new NotSupportedException($"JsonArrowType not supported: {type.Name}") + }; + } + + private static IArrowType ToIntArrowType(JsonArrowType type) + { + return (type.BitWidth, type.IsSigned) switch + { + (8, true) => Int8Type.Default, + (8, false) => UInt8Type.Default, + (16, true) => Int16Type.Default, + (16, false) => UInt16Type.Default, + (32, true) => Int32Type.Default, + (32, false) => UInt32Type.Default, + (64, true) => Int64Type.Default, + (64, false) => UInt64Type.Default, + _ => throw new NotSupportedException($"Int type not supported: {type.BitWidth}, {type.IsSigned}") + }; + } + + private static IArrowType ToFloatingPointArrowType(JsonArrowType type) + { + return type.Precision switch + { + "SINGLE" => FloatType.Default, + "DOUBLE" => DoubleType.Default, + _ => throw new NotSupportedException($"FloatingPoint type not supported: {type.Precision}") + }; + } + + private class ArrayCreator : + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor + { + private JsonFieldData JsonFieldData { get; } + public IArrowArray Array { get; private set; } + + public ArrayCreator(JsonFieldData jsonFieldData) + { + JsonFieldData = jsonFieldData; + } + + public void Visit(BooleanType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + ArrowBuffer.BitmapBuilder valueBuilder = new ArrowBuffer.BitmapBuilder(validityBuffer.Length); + + var json = JsonFieldData.Data.GetRawText(); + bool[] values = JsonSerializer.Deserialize(json); + + foreach (bool value in values) + { + valueBuilder.Append(value); + } + ArrowBuffer valueBuffer = valueBuilder.Build(); + + Array = new BooleanArray( + valueBuffer, validityBuffer, + JsonFieldData.Count, nullCount, 0); + } + + public void Visit(Int8Type type) => GenerateArray((v, n, c, nc, o) => new Int8Array(v, n, c, nc, o)); + public void Visit(Int16Type type) => GenerateArray((v, n, c, nc, o) => new Int16Array(v, n, c, nc, o)); + public void Visit(Int32Type type) => GenerateArray((v, n, c, nc, o) => new Int32Array(v, n, c, nc, o)); + public void Visit(Int64Type type) => GenerateLongArray((v, n, c, nc, o) => new Int64Array(v, n, c, nc, o), s => long.Parse(s)); + public void Visit(UInt8Type type) => GenerateArray((v, n, c, nc, o) => new UInt8Array(v, n, c, nc, o)); + public void Visit(UInt16Type type) => GenerateArray((v, n, c, nc, o) => new UInt16Array(v, n, c, nc, o)); + public void Visit(UInt32Type type) => GenerateArray((v, n, c, nc, o) => new UInt32Array(v, n, c, nc, o)); + public void Visit(UInt64Type type) => GenerateLongArray((v, n, c, nc, o) => new UInt64Array(v, n, c, nc, o), s => ulong.Parse(s)); + public void Visit(FloatType type) => GenerateArray((v, n, c, nc, o) => new FloatArray(v, n, c, nc, o)); + public void Visit(DoubleType type) => GenerateArray((v, n, c, nc, o) => new DoubleArray(v, n, c, nc, o)); + + public void Visit(Decimal128Type type) + { + throw new NotImplementedException(); + } + + public void Visit(Decimal256Type type) + { + throw new NotImplementedException(); + } + + public void Visit(Date32Type type) + { + throw new NotImplementedException(); + } + + public void Visit(Date64Type type) + { + throw new NotImplementedException(); + } + + public void Visit(TimestampType type) + { + throw new NotImplementedException(); + } + + public void Visit(StringType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + ArrowBuffer offsetBuffer = GetOffsetBuffer(); + + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize(json, s_options); + + ArrowBuffer.Builder valueBuilder = new ArrowBuffer.Builder(); + foreach (string value in values) + { + valueBuilder.Append(Encoding.UTF8.GetBytes(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(default); + + Array = new StringArray(JsonFieldData.Count, offsetBuffer, valueBuffer, validityBuffer, nullCount); + } + + public void Visit(BinaryType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + ArrowBuffer offsetBuffer = GetOffsetBuffer(); + + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize(json, s_options); + + ArrowBuffer.Builder valueBuilder = new ArrowBuffer.Builder(); + foreach (string value in values) + { + valueBuilder.Append(ConvertHexStringToByteArray(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(default); + + ArrayData arrayData = new ArrayData(type, JsonFieldData.Count, nullCount, 0, new[] { validityBuffer, offsetBuffer, valueBuffer }); + Array = new BinaryArray(arrayData); + } + + public void Visit(FixedSizeBinaryType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize(json, s_options); + + ArrowBuffer.Builder valueBuilder = new ArrowBuffer.Builder(); + foreach (string value in values) + { + valueBuilder.Append(ConvertHexStringToByteArray(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(default); + + ArrayData arrayData = new ArrayData(type, JsonFieldData.Count, nullCount, 0, new[] { validityBuffer, valueBuffer }); + Array = new FixedSizeBinaryArray(arrayData); + } + + public void Visit(ListType type) + { + throw new NotImplementedException(); + } + + public void Visit(StructType type) + { + throw new NotImplementedException(); + } + + private static byte[] ConvertHexStringToByteArray(string hexString) + { + byte[] data = new byte[hexString.Length / 2]; + for (int index = 0; index < data.Length; index++) + { + data[index] = byte.Parse(hexString.AsSpan(index * 2, 2) , NumberStyles.HexNumber, CultureInfo.InvariantCulture); + } + + return data; + } + + private static readonly JsonSerializerOptions s_options = new JsonSerializerOptions() + { + Converters = + { + new ByteArrayConverter() + } + }; + + private void GenerateArray(Func createArray) + where TArray : PrimitiveArray + where T : struct + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + ArrowBuffer.Builder valueBuilder = new ArrowBuffer.Builder(JsonFieldData.Count); + var json = JsonFieldData.Data.GetRawText(); + T[] values = JsonSerializer.Deserialize(json, s_options); + + foreach (T value in values) + { + valueBuilder.Append(value); + } + ArrowBuffer valueBuffer = valueBuilder.Build(); + + Array = createArray( + valueBuffer, validityBuffer, + JsonFieldData.Count, nullCount, 0); + } + + private void GenerateLongArray(Func createArray, Func parse) + where TArray : PrimitiveArray + where T : struct + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + ArrowBuffer.Builder valueBuilder = new ArrowBuffer.Builder(JsonFieldData.Count); + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize(json); + + foreach (string value in values) + { + valueBuilder.Append(parse(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(); + + Array = createArray( + valueBuffer, validityBuffer, + JsonFieldData.Count, nullCount, 0); + } + + private ArrowBuffer GetOffsetBuffer() + { + ArrowBuffer.Builder valueOffsets = new ArrowBuffer.Builder(JsonFieldData.Offset.Length); + valueOffsets.AppendRange(JsonFieldData.Offset); + return valueOffsets.Build(default); + } + + private ArrowBuffer GetValidityBuffer(out int nullCount) + { + if (JsonFieldData.Validity == null) + { + nullCount = 0; + return ArrowBuffer.Empty; + } + + ArrowBuffer.BitmapBuilder validityBuilder = new ArrowBuffer.BitmapBuilder(JsonFieldData.Validity.Length); + validityBuilder.AppendRange(JsonFieldData.Validity); + + nullCount = validityBuilder.UnsetBitCount; + return validityBuilder.Build(); + } + + public void Visit(IArrowType type) + { + throw new NotImplementedException($"{type.Name} not implemented"); + } + } + + private Task StreamToFile() + { + return Task.CompletedTask; + } + + private Task FileToStream() + { + return Task.CompletedTask; + } + + private async ValueTask ParseJsonFile() + { + using var fileStream = JsonFileInfo.OpenRead(); + JsonSerializerOptions options = new JsonSerializerOptions() + { +  PropertyNamingPolicy = JsonFileNamingPolicy.Instance, + }; + options.Converters.Add(new ValidityConverter()); + + return await JsonSerializer.DeserializeAsync(fileStream, options); + } + } +} diff --git a/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs b/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs new file mode 100644 index 0000000000000..c86150afd31d0 --- /dev/null +++ b/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Apache.Arrow.IntegrationTest +{ + public class JsonFile + { + public JsonSchema Schema { get; set; } + public List Batches { get; set; } + //public List Dictionaries {get;set;} + } + + public class JsonSchema + { + public List Fields { get; set; } + public JsonMetadata Metadata { get; set; } + } + + public class JsonField + { + public string Name { get; set; } + public bool Nullable { get; set; } + public JsonArrowType Type { get; set; } + public List Children { get; set; } + public JsonDictionaryIndex Dictionary { get; set; } + public JsonMetadata Metadata { get; set; } + } + + public class JsonArrowType + { + public string Name { get; set; } + + // int fields + public int BitWidth { get; set; } + public bool IsSigned { get; set; } + + // floating point fields + public string Precision { get; set; } + + // date and time fields + public string Unit { get; set; } + // timestamp fields + public string Timezone { get; set; } + + // FixedSizeBinary fields + public int ByteWidth { get; set; } + } + + public class JsonDictionaryIndex + { + public int Id { get; set; } + public JsonArrowType Type { get; set; } + public bool IsOrdered { get; set; } + } + + public class JsonMetadata : List> + { + } + + public class JsonRecordBatch + { + public int Count { get; set; } + public List Columns { get; set; } + } + + public class JsonFieldData + { + public string Name { get; set; } + public int Count { get; set; } + public bool[] Validity { get; set; } + public int[] Offset { get; set; } + public int[] TypeId { get; set; } + public JsonElement Data { get; set; } + public List Children { get; set; } + } + + internal sealed class ValidityConverter : JsonConverter + { + public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.True) return true; + if (reader.TokenType == JsonTokenType.False) return false; + + if (typeToConvert != typeof(bool) || reader.TokenType != JsonTokenType.Number) + { + throw new InvalidOperationException($"Unexpected bool data: {reader.TokenType}"); + } + + int value = reader.GetInt32(); + if (value == 0) return false; + if (value == 1) return true; + + throw new InvalidOperationException($"Unexpected bool value: {value}"); + } + + public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options) => throw new NotImplementedException(); + } + + internal sealed class ByteArrayConverter : JsonConverter + { + public override byte[] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartArray) + { + throw new InvalidOperationException($"Unexpected byte[] token: {reader.TokenType}"); + } + + List values = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + { + return values.ToArray(); + } + + if (reader.TokenType != JsonTokenType.Number) + { + throw new InvalidOperationException($"Unexpected byte token: {reader.TokenType}"); + } + + values.Add(reader.GetByte()); + } + + throw new InvalidOperationException("Unexpectedly reached the end of the reader"); + } + + public override void Write(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options) => throw new NotImplementedException(); + } + + internal sealed class JsonFileNamingPolicy : JsonNamingPolicy + { + public static JsonFileNamingPolicy Instance { get; } = new JsonFileNamingPolicy(); + + public override string ConvertName(string name) + { + if (name == "Validity") + { + return "VALIDITY"; + } + else if (name == "Offset") + { + return "OFFSET"; + } + else if (name == "TypeId") + { + return "TYPE_ID"; + } + else if (name == "Data") + { + return "DATA"; + } + else + { + return CamelCase.ConvertName(name); + } + } + } +} \ No newline at end of file diff --git a/csharp/test/Apache.Arrow.IntegrationTest/Program.cs b/csharp/test/Apache.Arrow.IntegrationTest/Program.cs new file mode 100644 index 0000000000000..243269386593c --- /dev/null +++ b/csharp/test/Apache.Arrow.IntegrationTest/Program.cs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Apache.Arrow.IntegrationTest +{ + public class Program + { + public static async Task Main(string[] args) + { + var integrationTestCommand = new RootCommand + { + new Option( + "--mode", + description: "Which command to run"), + new Option( + new[] { "--json-file", "-j" }, + "The JSON file to interact with"), + new Option( + new[] { "--arrow-file", "-a" }, + "The arrow file to interact with") + }; + + integrationTestCommand.Description = "Integration test app for Apache.Arrow .NET Library."; + + integrationTestCommand.Handler = CommandHandler.Create(async (mode, j, a) => + { + var integrationCommand = new IntegrationCommand(mode, j, a); + await integrationCommand.Execute(); + }); + return await integrationTestCommand.InvokeAsync(args); + } + } +} diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 582a428849216..a99903373d2e8 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -692,6 +692,8 @@ def _set_default(opt, default): help="Seed for PRNG when generating test data") @click.option('--with-cpp', type=bool, default=False, help='Include C++ in integration tests') +@click.option('--with-csharp', type=bool, default=False, + help='Include CSharp in integration tests') @click.option('--with-java', type=bool, default=False, help='Include Java in integration tests') @click.option('--with-js', type=bool, default=False, @@ -732,7 +734,7 @@ def integration(with_all=False, random_seed=12345, **args): gen_path = args['write_generated_json'] - languages = ['cpp', 'java', 'js', 'go', 'rust'] + languages = ['cpp', 'csharp', 'java', 'js', 'go', 'rust'] enabled_languages = 0 for lang in languages: diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py index 6f4c1385abf89..bfd9cd0b876c0 100644 --- a/dev/archery/archery/integration/runner.py +++ b/dev/archery/archery/integration/runner.py @@ -32,6 +32,7 @@ from .tester_rust import RustTester from .tester_java import JavaTester from .tester_js import JSTester +from .tester_csharp import CSharpTester from .util import (ARROW_ROOT_DEFAULT, guid, SKIP_ARROW, SKIP_FLIGHT, printer) from . import datagen @@ -330,8 +331,8 @@ def get_static_json_files(): def run_all_tests(with_cpp=True, with_java=True, with_js=True, - with_go=True, with_rust=False, run_flight=False, - tempdir=None, **kwargs): + with_csharp=True, with_go=True, with_rust=False, + run_flight=False, tempdir=None, **kwargs): tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-') testers = [] @@ -345,6 +346,9 @@ def run_all_tests(with_cpp=True, with_java=True, with_js=True, if with_js: testers.append(JSTester(**kwargs)) + if with_csharp: + testers.append(CSharpTester(**kwargs)) + if with_go: testers.append(GoTester(**kwargs)) diff --git a/dev/archery/archery/integration/tester_csharp.py b/dev/archery/archery/integration/tester_csharp.py new file mode 100644 index 0000000000000..49dc24b8ce1c8 --- /dev/null +++ b/dev/archery/archery/integration/tester_csharp.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +from .tester import Tester +from .util import run_cmd, ARROW_ROOT_DEFAULT, log + + +class CSharpTester(Tester): + PRODUCER = True + CONSUMER = True + + EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'csharp/artifacts/Apache.Arrow.IntegrationTest/Debug/netcoreapp3.1/Apache.Arrow.IntegrationTester.exe') + + name = 'csharp' + + def _run(self, json_path=None, arrow_path=None, command='validate'): + cmd = [EXE_PATH] + + cmd.extend(['--mode', command]) + + if json_path is not None: + cmd.extend(['-j', json_path]) + + if arrow_path is not None: + cmd.extend(['-a', arrow_path]) + + if self.debug: + log(' '.join(cmd)) + + run_cmd(cmd) + + def validate(self, json_path, arrow_path): + return self._run('validate', json_path, arrow_path) + + def json_to_file(self, json_path, arrow_path): + return self._run('json-to-arrow', json_path, arrow_path) + + def stream_to_file(self, stream_path, file_path): + cmd = [EXE_PATH] + cmd.extend(['--mode', 'stream-to-file']) + cmd.extend(['<', stream_path, '>', file_path]) + self.run_shell_command(cmd) + + def file_to_stream(self, file_path, stream_path): + cmd = [EXE_PATH] + cmd.extend(['--mode', 'file-to-stream']) + cmd.extend(['-f', file_path, '>', stream_path]) + self.run_shell_command(cmd)