Merge pull request #34 from wesen/misc/update-for-new-api

Update embeddings config and add RAG query example

.github/workflows/lint.yml

@@ -18,15 +18,15 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: git fetch --force --tags
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: '>=1.19.5'
           cache: true
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.1.0
+        uses: golangci/golangci-lint-action@v6
         with:
           args: --timeout=5m

.github/workflows/push.yml

@@ -10,11 +10,11 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: git fetch --force --tags
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: '>=1.19.5'
           cache: true

.github/workflows/release.yml

@@ -14,11 +14,11 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: git fetch --force --tags
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: '>=1.19.5'
           cache: true

cmd/escuse-me/cmds/documents/delete-by-query.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/documents/delete.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/documents/get.go

@@ -6,8 +6,8 @@ import (
 	"io"
 
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/documents/index.go

@@ -7,8 +7,8 @@ import (
 	"io"
 
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/documents/multi-get.go

@@ -7,8 +7,8 @@ import (
 	"io"
 
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/documents/search.go

@@ -5,8 +5,8 @@ import (
 	"context"
 	"encoding/json"
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/documents/update.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/indices/clone.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/indices/close.go

@@ -3,18 +3,19 @@ package indices
 import (
 	"context"
 	"encoding/json"
+	"io"
+	"strings"
+
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"
 	"github.com/go-go-golems/glazed/pkg/middlewares"
 	"github.com/go-go-golems/glazed/pkg/settings"
 	"github.com/go-go-golems/glazed/pkg/types"
 	"github.com/pkg/errors"
-	"io"
-	"strings"
 )
 
 type CloseIndexCommand struct {

cmd/escuse-me/cmds/indices/create-index.go

@@ -6,8 +6,8 @@ import (
 	"encoding/json"
 	"io"
 
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/cmds/indices/delete-index.go

@@ -3,17 +3,18 @@ package indices
 import (
 	"context"
 	"encoding/json"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
+	"io"
+	"strings"
+
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"
 	"github.com/go-go-golems/glazed/pkg/middlewares"
 	"github.com/go-go-golems/glazed/pkg/settings"
 	"github.com/go-go-golems/glazed/pkg/types"
 	"github.com/pkg/errors"
-	"io"
-	"strings"
 )
 
 type DeleteIndexCommand struct {

cmd/escuse-me/cmds/indices/flush.go

@@ -3,17 +3,18 @@ package indices
 import (
 	"context"
 	"encoding/json"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
+	"io"
+	"strings"
+
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"
 	"github.com/go-go-golems/glazed/pkg/middlewares"
 	"github.com/go-go-golems/glazed/pkg/settings"
 	"github.com/go-go-golems/glazed/pkg/types"
 	"github.com/pkg/errors"
-	"io"
-	"strings"
 )
 
 type FlushCommand struct {

cmd/escuse-me/cmds/indices/update-mapping.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/go-go-golems/escuse-me/cmd/escuse-me/pkg/helpers"
 	es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers"
+	"github.com/go-go-golems/escuse-me/pkg/helpers"
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"

cmd/escuse-me/main.go

@@ -258,6 +258,20 @@ func initAllCommands(helpSystem *help.HelpSystem) error {
 		ls_commands.WithCommandDescriptionOptions(
 			glazed_cmds.WithShort("Commands related to sqleton queries"),
 		),
+		ls_commands.WithAddCommandToRowFunc(func(
+			command glazed_cmds.Command,
+			row types.Row,
+			parsedLayers *glazed_layers.ParsedLayers,
+		) ([]types.Row, error) {
+			ret := []types.Row{row}
+			switch command.(type) {
+			case *es_cmds.ElasticSearchCommand:
+				row.Set("type", "escuse-me")
+			default:
+			}
+
+			return ret, nil
+		}),
 	)
 
 	if err != nil {

cmd/escuse-me/queries/examples/rag-query.yaml

@@ -0,0 +1,87 @@
+name: "rag-query"
+short: "Execute RAG queries across indexes"
+long: "Execute RAG (Retrieval Augmented Generation) queries across multiple indexes with support for metadata filtering and additional fields"
+
+flags:
+  - name: query_text
+    type: string
+    help: "The main query text to search for"
+    required: true
+  - name: index_names
+    type: stringList
+    help: "Optional list of indexes to query (uses default indexes if not specified)"
+  - name: slack_user_id
+    type: int
+    help: "Optional Slack user ID for filtering"
+  - name: slack_app_installation_id
+    type: int
+    help: "Optional Slack app installation ID for filtering"
+  - name: external_uuid
+    type: string
+    help: "Optional external UUID for tracking"
+  - name: show_vectors
+    type: bool
+    default: false
+    help: "Show vector fields in the output"
+  - name: show_raw_query
+    type: bool
+    default: false
+    help: "Show the raw Elasticsearch query"
+  - name: size
+    type: int
+    default: 10
+    help: "Number of results to return per index"
+  - name: fuzziness
+    type: string
+    default: "AUTO"
+    help: "Fuzziness level for text matching (AUTO, 0, 1, 2)"
+
+query:
+  _source: !Concat
+    - ["id", "content", "metadata", "created_at", "updated_at", "title", "name", "real_name", "subject", "insight_type"]
+    - !If
+        test: !Var show_vectors
+        then: ["content_vector", "metadata_vector"]
+        else: !Void
+  size: !Var size
+  aggs:
+    insight_types:
+      terms:
+        field: "insight_type"
+        size: 20
+  query:
+    bool:
+      must: !Concat
+        - !If
+            test: !Exists query_text
+            then:
+              - multi_match:
+                  query: !Var query_text
+                  fields: [
+                    "title^3",
+                    "name",
+                    "real_name",
+                    "subject",
+                    "content^2",
+                    "metadata.*"
+                  ]
+                  fuzziness: !Var fuzziness
+            else: !Void
+        - !If
+            test: !Exists slack_user_id
+            then:
+              - term:
+                  slack_user_id: !Var slack_user_id
+            else: !Void
+        - !If
+            test: !Exists slack_app_installation_id
+            then:
+              - term:
+                  slack_app_installation_id: !Var slack_app_installation_id
+            else: !Void
+        - !If
+            test: !Exists external_uuid
+            then:
+              - term:
+                  external_uuid: !Var external_uuid
+            else: !Void 

go.mod

@@ -6,11 +6,11 @@ toolchain go1.23.3
 
 require (
 	github.com/elastic/go-elasticsearch/v8 v8.17.0
-	github.com/go-go-golems/clay v0.1.27
-	github.com/go-go-golems/geppetto v0.4.34
-	github.com/go-go-golems/glazed v0.5.29
+	github.com/go-go-golems/clay v0.1.31
+	github.com/go-go-golems/geppetto v0.4.36
+	github.com/go-go-golems/glazed v0.5.34
 	github.com/go-go-golems/go-emrichen v0.0.4
-	github.com/go-go-golems/parka v0.5.18
+	github.com/go-go-golems/parka v0.5.20
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.33.0
 	github.com/spf13/cobra v1.8.1
@@ -58,7 +58,6 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/errors v0.22.0 // indirect
 	github.com/go-openapi/strfmt v0.23.0 // indirect
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -73,7 +72,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kopoli/go-terminal-size v0.0.0-20170219200355-5c97524c8b54 // indirect
 	github.com/kucherenkovova/safegroup v1.0.2 // indirect
-	github.com/labstack/echo/v4 v4.12.0 // indirect
+	github.com/labstack/echo/v4 v4.13.3 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
@@ -125,7 +124,7 @@ require (
 	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/time v0.8.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect