Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Rename spec structs for better representation in JSON schema #424

Merged
merged 1 commit into from
Jan 31, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ lint:

.PHONY: gen-spec-schema
gen-spec-schema:
# vendor for gen
go mod vendor
go run schemagen/main.go

# All gen targets
Expand Down
9 changes: 5 additions & 4 deletions csv/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ import (
"github.com/invopop/jsonschema"
)

type Spec struct {
// nolint:revive
type CSVSpec struct {
SkipHeader bool `json:"skip_header,omitempty"`
Delimiter string `json:"delimiter,omitempty"`
}

func (Spec) JSONSchema() *jsonschema.Schema {
func (CSVSpec) JSONSchema() *jsonschema.Schema {
properties := jsonschema.NewProperties()
properties.Set("skip_header", &jsonschema.Schema{
Type: "boolean",
Expand All @@ -32,13 +33,13 @@ func (Spec) JSONSchema() *jsonschema.Schema {
}
}

func (s *Spec) SetDefaults() {
func (s *CSVSpec) SetDefaults() {
if s.Delimiter == "" {
s.Delimiter = ","
}
}

func (s *Spec) Validate() error {
func (s *CSVSpec) Validate() error {
if len(s.Delimiter) != 1 {
return fmt.Errorf("delimiter must be a single character")
}
Expand Down
2 changes: 1 addition & 1 deletion csv/spec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
)

func TestSpec_JSONSchema(t *testing.T) {
schema, err := jsonschema.Generate(Spec{})
schema, err := jsonschema.Generate(CSVSpec{})
require.NoError(t, err)

jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
Expand Down
9 changes: 5 additions & 4 deletions json/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,19 @@ package json

import "github.com/invopop/jsonschema"

type Spec struct{}
// nolint:revive
type JSONSpec struct{}

func (Spec) JSONSchema() *jsonschema.Schema {
func (JSONSpec) JSONSchema() *jsonschema.Schema {
return &jsonschema.Schema{
Description: "CloudQuery JSON file output spec.",
Type: "object",
AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false
}
}

func (*Spec) SetDefaults() {}
func (*JSONSpec) SetDefaults() {}

func (*Spec) Validate() error {
func (*JSONSpec) Validate() error {
return nil
}
2 changes: 1 addition & 1 deletion json/spec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
)

func TestSpec_JSONSchema(t *testing.T) {
schema, err := jsonschema.Generate(Spec{})
schema, err := jsonschema.Generate(JSONSpec{})
require.NoError(t, err)

jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
Expand Down
4 changes: 2 additions & 2 deletions parquet/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ type Options func(*Client)

// Client is a parquet client.
type Client struct {
spec Spec
spec ParquetSpec
}

func NewClient(options ...Options) (*Client, error) {
Expand All @@ -16,7 +16,7 @@ func NewClient(options ...Options) (*Client, error) {
return c, nil
}

func WithSpec(spec Spec) Options {
func WithSpec(spec ParquetSpec) Options {
return func(c *Client) {
c.spec = spec
}
Expand Down
9 changes: 5 additions & 4 deletions parquet/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,20 @@ package parquet

import "github.com/invopop/jsonschema"

type Spec struct{}
// nolint:revive
type ParquetSpec struct{}

func (Spec) JSONSchema() *jsonschema.Schema {
func (ParquetSpec) JSONSchema() *jsonschema.Schema {
return &jsonschema.Schema{
Description: "CloudQuery Parquet file output spec.",
Type: "object",
AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false
}
}

func (*Spec) SetDefaults() {
func (*ParquetSpec) SetDefaults() {
}

func (*Spec) Validate() error {
func (*ParquetSpec) Validate() error {
return nil
}
2 changes: 1 addition & 1 deletion parquet/spec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
)

func TestSpec_JSONSchema(t *testing.T) {
schema, err := jsonschema.Generate(Spec{})
schema, err := jsonschema.Generate(ParquetSpec{})
require.NoError(t, err)

jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
Expand Down
6 changes: 3 additions & 3 deletions schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ func (FileSpec) JSONSchemaOptions() []cqjsonschema.Option {
return nil
}
return reflect.VisibleFields(reflect.TypeOf(struct {
CSVSpec csv.Spec
JSONSpec jsonfile.Spec
ParquetSpec parquet.Spec
CSVSpec csv.CSVSpec
JSONSpec jsonfile.JSONSpec
ParquetSpec parquet.ParquetSpec
}{}))
}
if r.AdditionalFields == nil {
Expand Down
52 changes: 26 additions & 26 deletions schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,24 @@
"$id": "https://github.com/cloudquery/filetypes/v4/file-spec",
"$ref": "#/$defs/FileSpec",
"$defs": {
"CSVSpec": {
"properties": {
"skip_header": {
"type": "boolean",
"description": "Specifies if the first line of a file should be the header.",
"default": false
},
"delimiter": {
"type": "string",
"pattern": "^.$",
"description": "Character that will be used as the delimiter.",
"default": ","
}
},
"additionalProperties": false,
"type": "object",
"description": "CloudQuery CSV file output spec."
},
"FileSpec": {
"oneOf": [
{
Expand All @@ -14,7 +32,7 @@
"format_spec": {
"oneOf": [
{
"$ref": "#/$defs/Spec"
"$ref": "#/$defs/CSVSpec"
},
{
"type": "null"
Expand All @@ -32,7 +50,7 @@
"format_spec": {
"oneOf": [
{
"$ref": "#/$defs/Spec-1"
"$ref": "#/$defs/JSONSpec"
},
{
"type": "null"
Expand All @@ -50,7 +68,7 @@
"format_spec": {
"oneOf": [
{
"$ref": "#/$defs/Spec-2"
"$ref": "#/$defs/ParquetSpec"
},
{
"type": "null"
Expand All @@ -75,13 +93,13 @@
{
"anyOf": [
{
"$ref": "#/$defs/Spec"
"$ref": "#/$defs/CSVSpec"
},
{
"$ref": "#/$defs/Spec-1"
"$ref": "#/$defs/JSONSpec"
},
{
"$ref": "#/$defs/Spec-2"
"$ref": "#/$defs/ParquetSpec"
}
]
},
Expand All @@ -105,30 +123,12 @@
"format"
]
},
"Spec": {
"properties": {
"skip_header": {
"type": "boolean",
"description": "Specifies if the first line of a file should be the header.",
"default": false
},
"delimiter": {
"type": "string",
"pattern": "^.$",
"description": "Character that will be used as the delimiter.",
"default": ","
}
},
"additionalProperties": false,
"type": "object",
"description": "CloudQuery CSV file output spec."
},
"Spec-1": {
"JSONSpec": {
"additionalProperties": false,
"type": "object",
"description": "CloudQuery JSON file output spec."
},
"Spec-2": {
"ParquetSpec": {
"additionalProperties": false,
"type": "object",
"description": "CloudQuery Parquet file output spec."
Expand Down
12 changes: 6 additions & 6 deletions spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ type FileSpec struct {
// Empty or missing stands for no compression.
Compression CompressionType `json:"compression,omitempty" jsonschema:"enum=,enum=gzip"`

csvSpec *csv.Spec
jsonSpec *jsonfile.Spec
parquetSpec *parquet.Spec
csvSpec *csv.CSVSpec
jsonSpec *jsonfile.JSONSpec
parquetSpec *parquet.ParquetSpec
}

func (s *FileSpec) SetDefaults() {
Expand Down Expand Up @@ -87,13 +87,13 @@ func (s *FileSpec) UnmarshalSpec() error {

switch s.Format {
case FormatTypeCSV:
s.csvSpec = &csv.Spec{}
s.csvSpec = &csv.CSVSpec{}
return dec.Decode(s.csvSpec)
case FormatTypeJSON:
s.jsonSpec = &jsonfile.Spec{}
s.jsonSpec = &jsonfile.JSONSpec{}
return dec.Decode(s.jsonSpec)
case FormatTypeParquet:
s.parquetSpec = &parquet.Spec{}
s.parquetSpec = &parquet.ParquetSpec{}
return dec.Decode(s.parquetSpec)
default:
return fmt.Errorf("unknown format %s", s.Format)
Expand Down
24 changes: 12 additions & 12 deletions spec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,19 @@ import (
func TestSpecMethods(t *testing.T) {
testCases := []struct {
FileSpec *FileSpec
preDefaultsCSV *csv.Spec
preDefaultsJSON *json.Spec
postDefaultsCSV *csv.Spec
postDefaultsJSON *json.Spec
preDefaultsCSV *csv.CSVSpec
preDefaultsJSON *json.JSONSpec
postDefaultsCSV *csv.CSVSpec
postDefaultsJSON *json.JSONSpec
expectError bool
}{
{
FileSpec: &FileSpec{
Format: FormatTypeCSV,
FormatSpec: map[string]any{},
},
preDefaultsCSV: &csv.Spec{},
postDefaultsCSV: &csv.Spec{
preDefaultsCSV: &csv.CSVSpec{},
postDefaultsCSV: &csv.CSVSpec{
SkipHeader: false,
Delimiter: ",",
},
Expand All @@ -36,11 +36,11 @@ func TestSpecMethods(t *testing.T) {
"skip_header": true,
},
},
preDefaultsCSV: &csv.Spec{
preDefaultsCSV: &csv.CSVSpec{
SkipHeader: true,
Delimiter: ",",
},
postDefaultsCSV: &csv.Spec{
postDefaultsCSV: &csv.CSVSpec{
SkipHeader: true,
Delimiter: ",",
},
Expand All @@ -50,11 +50,11 @@ func TestSpecMethods(t *testing.T) {
Format: FormatTypeCSV,
FormatSpec: map[string]any{},
},
preDefaultsCSV: &csv.Spec{
preDefaultsCSV: &csv.CSVSpec{
SkipHeader: false,
Delimiter: "",
},
postDefaultsCSV: &csv.Spec{
postDefaultsCSV: &csv.CSVSpec{
SkipHeader: false,
Delimiter: ",",
},
Expand All @@ -63,8 +63,8 @@ func TestSpecMethods(t *testing.T) {
FileSpec: &FileSpec{
Format: FormatTypeJSON,
},
preDefaultsJSON: &json.Spec{},
postDefaultsJSON: &json.Spec{},
preDefaultsJSON: &json.JSONSpec{},
postDefaultsJSON: &json.JSONSpec{},
},

{
Expand Down
Loading