Skip to content

Commit

Permalink
feat: Add JSON schema for specs (#343)
Browse files Browse the repository at this point in the history
Required for file-based plugins (file, s3, etc).
  • Loading branch information
candiduslynx authored Oct 23, 2023
1 parent 5b0754f commit 77cacac
Show file tree
Hide file tree
Showing 15 changed files with 542 additions and 14 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/unittest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,11 @@ jobs:
go-version-file: go.mod
- run: go mod download
- run: go build ./...
- name: gen
if: github.event_name == 'pull_request'
run: make gen
- name: Fail if generation updated files
if: github.event_name == 'pull_request'
run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
- name: Run tests
run: go test -race ./...
8 changes: 8 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,11 @@ test:
.PHONY: lint
lint:
golangci-lint run

.PHONY: gen-spec-schema
gen-spec-schema:
go run schemagen/main.go

# All gen targets
.PHONY: gen
gen: gen-spec-schema
2 changes: 1 addition & 1 deletion client.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func NewClient(spec *FileSpec) (*Client, error) {
}

if err != nil {
return &Client{}, err
return nil, err
}

return &Client{spec: spec, filetype: client}, nil
Expand Down
27 changes: 26 additions & 1 deletion csv/spec.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,37 @@
package csv

import "fmt"
import (
"fmt"

"github.com/invopop/jsonschema"
)

type Spec struct {
SkipHeader bool `json:"skip_header,omitempty"`
Delimiter string `json:"delimiter,omitempty"`
}

func (Spec) JSONSchema() *jsonschema.Schema {
properties := jsonschema.NewProperties()
properties.Set("skip_header", &jsonschema.Schema{
Type: "boolean",
Description: "Specifies if the first line of a file should be the header.",
Default: false,
})
properties.Set("delimiter", &jsonschema.Schema{
Type: "string",
Description: "Character that will be used as the delimiter.",
Pattern: `^.$`, // a single character
Default: ",",
})
return &jsonschema.Schema{
Description: "CloudQuery CSV file output spec.",
Properties: properties,
Type: "object",
AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false
}
}

func (s *Spec) SetDefaults() {
if s.Delimiter == "" {
s.Delimiter = ","
Expand Down
71 changes: 71 additions & 0 deletions csv/spec_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package csv

import (
"testing"

"github.com/cloudquery/codegen/jsonschema"
"github.com/stretchr/testify/require"
)

func TestSpec_JSONSchema(t *testing.T) {
schema, err := jsonschema.Generate(Spec{})
require.NoError(t, err)

jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
{
Name: "empty",
Spec: `{}`,
},
{
Name: "extra keyword",
Err: true,
Spec: `{"extra":true}`,
},
{
Name: "skip_header:true",
Spec: `{"skip_header":true}`,
},
{
Name: "skip_header:false",
Spec: `{"skip_header":false}`,
},
{
Name: "null skip_header",
Err: true,
Spec: `{"skip_header":null}`,
},
{
Name: "bad skip_header",
Err: true,
Spec: `{"skip_header":123}`,
},
{
Name: "empty delimiter",
Err: true,
Spec: `{"delimiter":""}`,
},
{
Name: "null delimiter",
Err: true,
Spec: `{"delimiter":null}`,
},
{
Name: "bad delimiter",
Err: true,
Spec: `{"delimiter":123}`,
},
{
Name: "delimiter:\",,\"",
Err: true,
Spec: `{"delimiter":",,"}`,
},
{
Name: "tab delimiter",
Spec: `{"delimiter":"\t"}`,
},
{
Name: "space delimiter",
Spec: `{"delimiter":" "}`,
},
})
}
24 changes: 15 additions & 9 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,49 +5,55 @@ go 1.21.1
require (
github.com/apache/arrow/go/v14 v14.0.0-20230929201650-00efb06dc0de
github.com/bradleyjkemp/cupaloy/v2 v2.8.0
github.com/cloudquery/codegen v0.3.10
github.com/cloudquery/plugin-sdk/v4 v4.16.1
github.com/goccy/go-json v0.10.2
github.com/google/uuid v1.3.1
github.com/invopop/jsonschema v0.11.0
github.com/stretchr/testify v1.8.4
github.com/wk8/go-ordered-map/v2 v2.1.8
)

require (
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
)

// TODO: remove once all updates are merged
replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561

require (
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/apache/thrift v0.19.0 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v23.5.26+incompatible // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/zerolog v1.31.0 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
github.com/thoas/go-funk v0.9.3 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect; indirect // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.14.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
google.golang.org/grpc v1.58.2 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

// TODO: remove once all updates are merged
replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561

// github.com/cloudquery/jsonschema @ cqmain
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f
13 changes: 13 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,18 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk=
github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561 h1:sXA8imGI4P8EPdycL1w7mzigIHx2KqyntgAqGuTgTi8=
github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY=
github.com/cloudquery/codegen v0.3.10 h1:xd6g8UjxnMHzpY/TcM4r1YCcQ75tyc2Ehyu8d+yCpm4=
github.com/cloudquery/codegen v0.3.10/go.mod h1:ZqCxX9AGSMUnafE3gKmcT+RJCTgFg7izDxJwFSLGRcg=
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8=
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
github.com/cloudquery/plugin-sdk/v4 v4.16.1 h1:Ir2fkqsu5htnnI4wTGTmA6wp1LkixZmmbyxRSIsGoxM=
github.com/cloudquery/plugin-sdk/v4 v4.16.1/go.mod h1:ujSFEUAp8BmozOee0ljjsPHQfddXJCUTAzCD6sVKsu8=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
Expand All @@ -29,6 +37,7 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
Expand All @@ -39,6 +48,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
Expand Down Expand Up @@ -70,6 +81,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
Expand Down
10 changes: 10 additions & 0 deletions json/spec.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
package json

import "github.com/invopop/jsonschema"

type Spec struct{}

func (Spec) JSONSchema() *jsonschema.Schema {
return &jsonschema.Schema{
Description: "CloudQuery JSON file output spec.",
Type: "object",
AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false
}
}

func (*Spec) SetDefaults() {}

func (*Spec) Validate() error {
Expand Down
25 changes: 25 additions & 0 deletions json/spec_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package json

import (
"testing"

"github.com/cloudquery/codegen/jsonschema"
"github.com/stretchr/testify/require"
)

func TestSpec_JSONSchema(t *testing.T) {
schema, err := jsonschema.Generate(Spec{})
require.NoError(t, err)

jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
{
Name: "empty",
Spec: `{}`,
},
{
Name: "extra keyword",
Err: true,
Spec: `{"extra":true}`,
},
})
}
10 changes: 10 additions & 0 deletions parquet/spec.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
package parquet

import "github.com/invopop/jsonschema"

type Spec struct{}

func (Spec) JSONSchema() *jsonschema.Schema {
return &jsonschema.Schema{
Description: "CloudQuery Parquet file output spec.",
Type: "object",
AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false
}
}

func (*Spec) SetDefaults() {
}

Expand Down
25 changes: 25 additions & 0 deletions parquet/spec_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package parquet

import (
"testing"

"github.com/cloudquery/codegen/jsonschema"
"github.com/stretchr/testify/require"
)

func TestSpec_JSONSchema(t *testing.T) {
schema, err := jsonschema.Generate(Spec{})
require.NoError(t, err)

jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
{
Name: "empty",
Spec: `{}`,
},
{
Name: "extra keyword",
Err: true,
Spec: `{"extra":true}`,
},
})
}
Loading

0 comments on commit 77cacac

Please sign in to comment.