feat: First commit

.github/add_tags.yml

@@ -0,0 +1,15 @@
+# Warning, do not check out untrusted code with
+# the pull_request_target event.
+# the current workflow IS safe as long as you dont checkout untrusted code
+# https://nathandavison.com/blog/github-actions-and-the-threat-of-malicious-pull-requests
+on:
+  pull_request_target:
+    types: [opened, edited]
+name: conventional-release-labels
+jobs:
+  label:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bcoe/conventional-release-labels@v1
+        with:
+          type_labels: '{ "feat": "feat", "fix": "fix", "chore": "chore", "refactor": "refactor", "test": "test", "breaking": "breaking" }'

.github/lint_golang.yml

@@ -0,0 +1,23 @@
+name: Lint
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  golangci:
+    name: Lint with GolangCI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.50.1
+          args: --verbose

.github/pr_title.yml

@@ -0,0 +1,52 @@
+name: "Validate PR title"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  main:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      # Please look up the latest version from
+      # https://github.com/amannn/action-semantic-pull-request/releases
+      - uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # Configure which types are allowed.
+          # Default: https://github.com/commitizen/conventional-commit-types
+          types: |
+            fix
+            feat
+            chore
+            refactor
+            test
+          # Configure that a scope must always be provided.
+          requireScope: false
+          # Configure additional validation for the subject based on a regex.
+          # This example ensures the subject starts with an uppercase character.
+          subjectPattern: ^[A-Z].+$
+          # If `subjectPattern` is configured, you can use this property to override
+          # the default error message that is shown when the pattern doesn't match.
+          # The variables `subject` and `title` can be used within the message.
+          subjectPatternError: |
+            The subject "{subject}" found in the pull request title "{title}"
+            didn't match the configured pattern. Please ensure that the subject
+            starts with an uppercase character.
+          # For work-in-progress PRs you can typically use draft pull requests
+          # from Github. However, private repositories on the free plan don't have
+          # this option and therefore this action allows you to opt-in to using the
+          # special "[WIP]" prefix to indicate this state. This will avoid the
+          # validation of the PR title and the pull request checks remain pending.
+          # Note that a second check will be reported if this is enabled.
+          wip: true
+          # When using "Squash and merge" on a PR with only one commit, GitHub
+          # will suggest using that commit message instead of the PR title for the
+          # merge commit, and it's easy to commit this by mistake. Enable this option
+          # to also validate the commit message for one commit PRs.
+          validateSingleCommit: false

.github/release_pr.yml

@@ -0,0 +1,45 @@
+name: filetypes
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: google-github-actions/release-please-action@v3
+        id: release
+        with:
+          release-type: go
+          package-name: filetypes
+          token: ${{ secrets.GH_CQ_BOT }}
+          pull-request-title-pattern: "chore${scope}: Release${component} v${version}"
+          bump-minor-pre-major: true
+          bump-patch-for-minor-pre-major: true
+      - name: Parse semver string
+        if: steps.release.outputs.release_created
+        id: semver_parser
+        uses: booxmedialtd/ws-action-parse-semver@e81ad80123156d7ddd4f6c8383e63f497f857deb
+        with:
+          input_string: ${{ steps.release.outputs.tag_name }}
+      - name: Mark as pre-release
+        if: steps.semver_parser.outputs.prerelease != ''
+        uses: tubone24/update_release@1cfa79ccf7422570d9a9c6b6de749ff4b280e48b
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG_NAME: ${{ steps.release.outputs.tag_name }}
+        with:
+          prerelease: true
+      - name: Trigger Renovate
+        uses: actions/github-script@v6
+        if: steps.release.outputs.release_created && steps.semver_parser.outputs.prerelease == ''
+        with:
+          github-token: ${{ secrets.GH_CQ_BOT }}
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: 'cloudquery',
+              repo: '.github',
+              workflow_id: 'renovate.yml',
+              ref: 'main',
+            })

.github/unittest.yml

@@ -0,0 +1,31 @@
+name: "Unit tests"
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  unitests:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: "go.mod"
+          cache: true
+      - run: go mod download
+      - run: go build ./...
+      - name: Run tests
+        if: matrix.os != 'ubuntu-latest'
+        run: go test -race ./...
+      - name: Run tests
+        if: matrix.os == 'ubuntu-latest'
+        run: go test -race ./...

csv/read.go

@@ -0,0 +1,37 @@
+package csv
+
+import (
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/cloudquery/plugin-sdk/schema"
+)
+
+
+func Read(f io.Reader, table *schema.Table, sourceName string, res chan<- []any) error {
+	reader := csv.NewReader(f)
+	sourceNameIndex := table.Columns.Index(schema.CqSourceNameColumn.Name)
+	if sourceNameIndex == -1 {
+		return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name)
+	}
+	for {
+		record, err := reader.Read()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return err
+		}
+		if record[sourceNameIndex] != sourceName {
+			continue
+		}
+		values := make([]any, len(record))
+		for i, v := range record {
+			values[i] = v
+		}
+		res <- values
+	}
+	return nil
+}

csv/transformer.go

@@ -0,0 +1,84 @@
+package csv
+
+import (
+	"github.com/cloudquery/plugin-sdk/plugins/destination"
+	"github.com/cloudquery/plugin-sdk/schema"
+)
+
+type ReverseTransformer struct{
+	defaultTransformer destination.DefaultReverseTransformer
+}
+
+func (t *ReverseTransformer) ReverseTransformValues(table *schema.Table, values []any) (schema.CQTypes, error) {
+	return t.defaultTransformer.ReverseTransformValues(table, values) 
+}
+
+type Transformer struct{}
+
+func (*Transformer) TransformBool(v *schema.Bool) any {
+	return v.String()
+}
+
+func (*Transformer) TransformBytea(v *schema.Bytea) any {
+	return v.String()
+}
+
+func (*Transformer) TransformFloat8(v *schema.Float8) any {
+	return v.String()
+}
+
+func (*Transformer) TransformInt8(v *schema.Int8) any {
+	return v.String()
+}
+
+func (*Transformer) TransformInt8Array(v *schema.Int8Array) any {
+	return v.String()
+}
+
+func (*Transformer) TransformJSON(v *schema.JSON) any {
+	return v.String()
+}
+
+func (*Transformer) TransformText(v *schema.Text) any {
+	return v.Str
+}
+
+func (*Transformer) TransformTextArray(v *schema.TextArray) any {
+	return v.String()
+}
+
+func (*Transformer) TransformTimestamptz(v *schema.Timestamptz) any {
+	return v.String()
+}
+
+func (*Transformer) TransformUUID(v *schema.UUID) any {
+	return v.String()
+}
+
+func (*Transformer) TransformUUIDArray(v *schema.UUIDArray) any {
+	return v.String()
+}
+
+func (*Transformer) TransformCIDR(v *schema.CIDR) any {
+	return v.String()
+}
+
+func (*Transformer) TransformCIDRArray(v *schema.CIDRArray) any {
+	return v.String()
+}
+
+func (*Transformer) TransformInet(v *schema.Inet) any {
+	return v.String()
+}
+
+func (*Transformer) TransformInetArray(v *schema.InetArray) any {
+	return v.String()
+}
+
+func (*Transformer) TransformMacaddr(v *schema.Macaddr) any {
+	return v.String()
+}
+
+func (*Transformer) TransformMacaddrArray(v *schema.MacaddrArray) any {
+	return v.String()
+}

csv/write.go

@@ -0,0 +1,26 @@
+package csv
+
+import (
+	"encoding/csv"
+	"io"
+
+	"github.com/cloudquery/plugin-sdk/schema"
+)
+
+
+func WriteTableBatch(w io.Writer, table *schema.Table, resources [][]any) error {
+	writer := csv.NewWriter(w)	
+	for _, resource := range resources {
+		record := make([]string, len(resource))
+		for i, v := range resource {
+			record[i] = v.(string)
+		}
+		if err := writer.Write(record); err != nil {
+			return err
+		}
+	}
+	writer.Flush()
+	return nil
+}
+
+

csv/write_read_test.go

@@ -0,0 +1,54 @@
+package csv
+
+import (
+	"bufio"
+	"bytes"
+	"testing"
+
+	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/cloudquery/plugin-sdk/testdata"
+)
+
+
+
+func TestWriteRead(t *testing.T) {
+	var b bytes.Buffer
+	table := testdata.TestTable("test")
+	cqtypes := testdata.GenTestData(table)
+	if err := cqtypes[0].Set("test-source"); err != nil {
+		t.Fatal(err)
+	}
+	writer := bufio.NewWriter(&b)
+	transformer := &Transformer{}
+	transformedValues := schema.TransformWithTransformer(transformer, cqtypes)
+	// schema.TransformWithTransformer(tra)
+	if err := WriteTableBatch(writer, table, [][]any{transformedValues}); err != nil {
+		t.Fatal(err)
+	}
+	writer.Flush()
+	reader := bufio.NewReader(&b)
+	ch := make(chan []any)
+	var readErr error
+	go func() {
+		readErr = Read(reader, table, "test-source", ch)
+		close(ch)
+	}()
+	totalCount := 0
+	reverseTransformer := &ReverseTransformer{}
+	for row := range ch {
+		gotCqtypes, err := reverseTransformer.ReverseTransformValues(table, row)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if diff := cqtypes.Diff(gotCqtypes); diff != "" {
+			t.Fatalf("got diff: %s", diff)
+		}
+		totalCount++
+	}
+	if readErr != nil {
+		t.Fatal(readErr)
+	}
+	if totalCount != 1 {
+		t.Fatalf("expected 1 row, got %d", totalCount)
+	}
+}