Skip to content

Commit

Permalink
feat(build): attempt to work with tinygo
Browse files Browse the repository at this point in the history
remove parquet import in hashing

cleanup type check

fix uintptr usage

exclude goccy/go-json for tinygo builds

last uintptr problem
  • Loading branch information
Adam Gaynor authored and Chris Casola committed May 30, 2023
1 parent e628ca5 commit db145f4
Show file tree
Hide file tree
Showing 62 changed files with 309 additions and 479 deletions.
3 changes: 2 additions & 1 deletion go/arrow/_tools/tmpl/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package main

import (
"bytes"
"encoding/json"
"flag"
"fmt"
"go/format"
Expand All @@ -28,6 +27,8 @@ import (
"path/filepath"
"strings"
"text/template"

"github.com/apache/arrow/go/v13/internal/json"
)

const Ext = ".tmpl"
Expand Down
18 changes: 9 additions & 9 deletions go/arrow/array.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@
package arrow

import (
"encoding/json"
"fmt"

"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/apache/arrow/go/v13/internal/json"
)

// ArrayData is the underlying memory and metadata of an Arrow array, corresponding
Expand All @@ -32,21 +32,21 @@ import (
// which allows for manipulating the internal data and casting. For example,
// one could cast the raw bytes from int64 to float64 like so:
//
// arrdata := GetMyInt64Data().Data()
// newdata := array.NewData(arrow.PrimitiveTypes.Float64, arrdata.Len(),
// arrdata.Buffers(), nil, arrdata.NullN(), arrdata.Offset())
// defer newdata.Release()
// float64arr := array.NewFloat64Data(newdata)
// defer float64arr.Release()
// arrdata := GetMyInt64Data().Data()
// newdata := array.NewData(arrow.PrimitiveTypes.Float64, arrdata.Len(),
// arrdata.Buffers(), nil, arrdata.NullN(), arrdata.Offset())
// defer newdata.Release()
// float64arr := array.NewFloat64Data(newdata)
// defer float64arr.Release()
//
// This is also useful in an analytics setting where memory may be reused. For
// example, if we had a group of operations all returning float64 such as:
//
// Log(Sqrt(Expr(arr)))
// Log(Sqrt(Expr(arr)))
//
// The low-level implementations could have signatures such as:
//
// func Log(values arrow.ArrayData) arrow.ArrayData
// func Log(values arrow.ArrayData) arrow.ArrayData
//
// Another example would be a function that consumes one or more memory buffers
// in an input array and replaces them with newly-allocated data, changing the
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/binary.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
"unsafe"

"github.com/apache/arrow/go/v13/arrow"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

type BinaryLike interface {
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/binarybuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// A BinaryBuilder is used to build a Binary array using the Append methods.
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/boolean.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/bitutil"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// A type which represents an immutable sequence of boolean values.
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/booleanbuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
"github.com/apache/arrow/go/v13/arrow/bitutil"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

type BooleanBuilder struct {
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/bitutil"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

const (
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/decimal128.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
"github.com/apache/arrow/go/v13/arrow/decimal128"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// A type which represents an immutable sequence of 128-bit decimal values.
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/decimal256.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
"github.com/apache/arrow/go/v13/arrow/decimal256"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// Decimal256 is a type that represents an immutable sequence of 256-bit decimal values.
Expand Down
8 changes: 4 additions & 4 deletions go/arrow/array/dictionary.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ import (
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/apache/arrow/go/v13/internal/hashing"
"github.com/apache/arrow/go/v13/internal/json"
"github.com/apache/arrow/go/v13/internal/utils"
"github.com/goccy/go-json"
)

// Dictionary represents the type for dictionary-encoded data with a data
Expand All @@ -45,12 +45,12 @@ import (
//
// For example, the array:
//
// ["foo", "bar", "foo", "bar", "foo", "bar"]
// ["foo", "bar", "foo", "bar", "foo", "bar"]
//
// with dictionary ["bar", "foo"], would have the representation of:
//
// indices: [1, 0, 1, 0, 1, 0]
// dictionary: ["bar", "foo"]
// indices: [1, 0, 1, 0, 1, 0]
// dictionary: ["bar", "foo"]
//
// The indices in principle may be any integer type.
type Dictionary struct {
Expand Down
6 changes: 3 additions & 3 deletions go/arrow/array/diff_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package array_test

import (
"encoding/json"
"fmt"
"math/rand"
"reflect"
Expand All @@ -27,6 +26,7 @@ import (
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/array"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/apache/arrow/go/v13/internal/json"
"github.com/apache/arrow/go/v13/internal/types"
)

Expand Down Expand Up @@ -640,13 +640,13 @@ func TestEdits_UnifiedDiff(t *testing.T) {
dataType: arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int32),
baseJSON: `[
[{"key": "foo", "value": 2}, {"key": "bar", "value": 3}, {"key": "baz", "value": 1}],
[{"key": "quux", "value": 13}]
[{"key": "quux", "value": 13}],
[]
]`,
targetJSON: `[
[{"key": "foo", "value": 2}, {"key": "bar", "value": 3}, {"key": "baz", "value": 1}],
[{"key": "ytho", "value": 11}],
[{"key": "quux", "value": 13}]
[{"key": "quux", "value": 13}],
[]
]`,
want: `@@ -1, +1 @@
Expand Down
30 changes: 17 additions & 13 deletions go/arrow/array/encoded.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ import (
"github.com/apache/arrow/go/v13/arrow/encoded"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/apache/arrow/go/v13/internal/json"
"github.com/apache/arrow/go/v13/internal/utils"
"github.com/goccy/go-json"
)

// RunEndEncoded represents an array containing two children:
Expand Down Expand Up @@ -74,22 +74,24 @@ func (r *RunEndEncoded) Release() {
// run, only over the range of run values inside the logical offset/length
// range of the parent array.
//
// Example
// # Example
//
// For this array:
// RunEndEncoded: { Offset: 150, Length: 1500 }
// RunEnds: [ 1, 2, 4, 6, 10, 1000, 1750, 2000 ]
// Values: [ "a", "b", "c", "d", "e", "f", "g", "h" ]
//
// RunEndEncoded: { Offset: 150, Length: 1500 }
// RunEnds: [ 1, 2, 4, 6, 10, 1000, 1750, 2000 ]
// Values: [ "a", "b", "c", "d", "e", "f", "g", "h" ]
//
// LogicalValuesArray will return the following array:
// [ "f", "g" ]
//
// [ "f", "g" ]
//
// This is because the offset of 150 tells it to skip the values until
// "f" which corresponds with the logical offset (the run from 10 - 1000),
// and stops after "g" because the length + offset goes to 1650 which is
// within the run from 1000 - 1750, corresponding to the "g" value.
//
// Note
// # Note
//
// The return from this needs to be Released.
func (r *RunEndEncoded) LogicalValuesArray() arrow.Array {
Expand All @@ -109,23 +111,25 @@ func (r *RunEndEncoded) LogicalValuesArray() arrow.Array {
// that are adjusted so the new array can have an offset of 0. As a result
// this method can be expensive to call for an array with a non-zero offset.
//
// Example
// # Example
//
// For this array:
// RunEndEncoded: { Offset: 150, Length: 1500 }
// RunEnds: [ 1, 2, 4, 6, 10, 1000, 1750, 2000 ]
// Values: [ "a", "b", "c", "d", "e", "f", "g", "h" ]
//
// RunEndEncoded: { Offset: 150, Length: 1500 }
// RunEnds: [ 1, 2, 4, 6, 10, 1000, 1750, 2000 ]
// Values: [ "a", "b", "c", "d", "e", "f", "g", "h" ]
//
// LogicalRunEndsArray will return the following array:
// [ 850, 1500 ]
//
// [ 850, 1500 ]
//
// This is because the offset of 150 tells us to skip all run-ends less
// than 150 (by finding the physical offset), and we adjust the run-ends
// accordingly (1000 - 150 = 850). The logical length of the array is 1500,
// so we know we don't want to go past the 1750 run end. Thus the last
// run-end is determined by doing: min(1750 - 150, 1500) = 1500.
//
// Note
// # Note
//
// The return from this needs to be Released
func (r *RunEndEncoded) LogicalRunEndsArray(mem memory.Allocator) arrow.Array {
Expand Down
5 changes: 3 additions & 2 deletions go/arrow/array/encoded_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@
package array_test

import (
"encoding/json"
"strings"
"testing"

"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/array"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/apache/arrow/go/v13/internal/json"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -435,7 +436,7 @@ func TestRunEndEncodedUnmarshalNestedJSON(t *testing.T) {
defer bldr.Release()

const testJSON = `
[null, [1, 2, 3], [1, 2, 3], [1, 2, 3], [1, null, 3], [4, 5, null], null, null,
[null, [1, 2, 3], [1, 2, 3], [1, 2, 3], [1, null, 3], [4, 5, null], null, null,
[4, 5, null], [4, 5, null], [4, 5, null]]
`

Expand Down
25 changes: 12 additions & 13 deletions go/arrow/array/extension.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (

"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// ExtensionArray is the interface that needs to be implemented to handle
Expand Down Expand Up @@ -120,10 +120,9 @@ func NewExtensionData(data arrow.ArrayData) ExtensionArray {
// ExtensionArrayBase is the base struct for user-defined Extension Array types
// and must be embedded in any user-defined extension arrays like so:
//
// type UserDefinedArray struct {
// array.ExtensionArrayBase
// }
//
// type UserDefinedArray struct {
// array.ExtensionArrayBase
// }
type ExtensionArrayBase struct {
array
storage arrow.Array
Expand Down Expand Up @@ -212,18 +211,18 @@ type ExtensionBuilder struct {
// the appropriate extension array type and set the storage correctly, resetting the builder for
// reuse.
//
// Example
// # Example
//
// Simple example assuming an extension type of a UUID defined as a FixedSizeBinary(16) was registered
// using the type name "uuid":
//
// uuidType := arrow.GetExtensionType("uuid")
// bldr := array.NewExtensionBuilder(memory.DefaultAllocator, uuidType)
// defer bldr.Release()
// uuidBldr := bldr.StorageBuilder().(*array.FixedSizeBinaryBuilder)
// /* build up the fixed size binary array as usual via Append/AppendValues */
// uuidArr := bldr.NewExtensionArray()
// defer uuidArr.Release()
// uuidType := arrow.GetExtensionType("uuid")
// bldr := array.NewExtensionBuilder(memory.DefaultAllocator, uuidType)
// defer bldr.Release()
// uuidBldr := bldr.StorageBuilder().(*array.FixedSizeBinaryBuilder)
// /* build up the fixed size binary array as usual via Append/AppendValues */
// uuidArr := bldr.NewExtensionArray()
// defer uuidArr.Release()
//
// Because the storage builder is embedded in the Extension builder it also means
// that any of the functions available on the Builder interface can be called on
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/fixed_size_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"github.com/apache/arrow/go/v13/arrow/bitutil"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// FixedSizeList represents an immutable sequence of N array values.
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/fixedsize_binary.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"strings"

"github.com/apache/arrow/go/v13/arrow"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// A type which represents an immutable sequence of fixed-length binary strings.
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/fixedsize_binarybuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods.
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/float16.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (

"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/float16"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

// A type which represents an immutable sequence of Float16 values.
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/float16_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
"github.com/apache/arrow/go/v13/arrow/float16"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

type Float16Builder struct {
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/interval.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
"github.com/apache/arrow/go/v13/arrow/bitutil"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

func NewIntervalData(data arrow.ArrayData) arrow.Array {
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/json_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

type Option func(config)
Expand Down
2 changes: 1 addition & 1 deletion go/arrow/array/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"github.com/apache/arrow/go/v13/arrow/bitutil"
"github.com/apache/arrow/go/v13/arrow/internal/debug"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/goccy/go-json"
"github.com/apache/arrow/go/v13/internal/json"
)

type ListLike interface {
Expand Down
Loading

0 comments on commit db145f4

Please sign in to comment.