From 56ad9f1c52e06e3be3b8f50450f82ae30477c928 Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Fri, 31 Jan 2025 11:18:20 -0500 Subject: [PATCH 1/4] :sparkles: Add embeddings funcmap --- .vscode/launch.json | 18 +++++++ .../examples/search-summaries-embeddings.yaml | 24 +++++++++ pkg/cmds/cmd.go | 53 +++++++++++++++---- pkg/cmds/cobra.go | 4 +- 4 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index 29ebeda..faf2b20 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,6 +20,24 @@ "program": "${workspaceFolder}/cmd/escuse-me", "args": ["mento", "index-stats", "--output", "yaml"], "envFile": "${workspaceFolder}/.envrc" + }, + { + "name": "Search Summaries Embeddings", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/escuse-me", + "args": ["examples", "search-summaries-embeddings", "--query", "test", "--print-query"], + "envFile": "${workspaceFolder}/.envrc" + }, + { + "name": "Process test-data/concat.yml", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/../go-emrichen/cmd/emrichen", + "cwd": "${workspaceFolder}/../go-emrichen", + "args": ["process", "test-data/defaults-var-format.yml"] } ] } \ No newline at end of file diff --git a/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml b/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml new file mode 100644 index 0000000..75851a3 --- /dev/null +++ b/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml @@ -0,0 +1,24 @@ +name: "search-summaries-embeddings" +short: "Search through summaries using embeddings" +long: "Search through the summaries using semantic similarity via embeddings" + +flags: + - name: query + type: string + help: "Search text to find similar content" + +default-index: local-testing-multi-document-summarization + +query: + embeddings: !Embeddings + query: {{query}} + # _source: ["content"] + # size: 5 + # query: + # knn: + # field: content_vector + # query_vector: !Embeddings + # query: {{query}} + # # query_vector: !Embeddings + # # - !Var query + # k: 5 diff --git a/pkg/cmds/cmd.go b/pkg/cmds/cmd.go index b23c95c..a1be227 100644 --- a/pkg/cmds/cmd.go +++ b/pkg/cmds/cmd.go @@ -12,6 +12,7 @@ import ( "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/go-elasticsearch/v8/esapi" es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers" + "github.com/go-go-golems/geppetto/pkg/embeddings" "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/layers" "github.com/go-go-golems/glazed/pkg/cmds/layout" @@ -73,7 +74,12 @@ func NewElasticSearchCommand( if err != nil { return nil, err } - description.Layers.AppendLayers(glazedParameterLayer, esConnectionLayer, esHelpersLayer) + embeddingsLayer, err := embeddings.NewEmbeddingsFlagsLayer() + if err != nil { + return nil, err + } + + description.Layers.AppendLayers(glazedParameterLayer, esConnectionLayer, esHelpersLayer, embeddingsLayer) return &ElasticSearchCommand{ CommandDescription: description, @@ -109,18 +115,30 @@ func (esc *ElasticSearchCommand) RunIntoGlazeProcessor( // TODO(2022-12-21, manuel): Add explain functionality // See: https://github.com/wesen/sqleton/issues/45 + embeddingsSettings := &embeddings.EmbeddingsConfig{} + err = parsedLayers.InitializeStruct(embeddings.EmbeddingsSlug, embeddingsSettings) + if err != nil { + return err + } + + embeddingsFactory := embeddings.NewSettingsFactory(embeddingsSettings) + + additionalTags := map[string]emrichen.TagFunc{ + "!Embeddings": embeddingsFactory.GetEmbeddingTagFunc(), + } + ps_ := parsedLayers.GetDataMap() if esHelperSettings.PrintQuery { if output == "json" { - query, err := esc.RenderQueryToJSON(ps_) + query, err := esc.RenderQueryToJSON(ps_, additionalTags) if err != nil { return errors.Wrapf(err, "Could not generate query") } fmt.Println(query) return &cmds.ExitWithoutGlazeError{} } else { - query, err := esc.RenderQueryToYAML(ps_) + query, err := esc.RenderQueryToYAML(ps_, additionalTags) if err != nil { return errors.Wrapf(err, "Could not generate query") } @@ -137,12 +155,15 @@ func (esc *ElasticSearchCommand) RunIntoGlazeProcessor( return err } -func (esc *ElasticSearchCommand) renderNodeQuery(parameters map[string]interface{}) (*yaml.Node, error) { +func (esc *ElasticSearchCommand) renderNodeQuery(parameters map[string]interface{}, additionalTags map[string]emrichen.TagFunc) (*yaml.Node, error) { if esc.QueryNodeTemplate == nil { return nil, errors.New("No query template found") } - ei, err := emrichen.NewInterpreter(emrichen.WithVars(parameters)) + ei, err := emrichen.NewInterpreter( + emrichen.WithVars(parameters), + emrichen.WithAdditionalTags(additionalTags), + ) if err != nil { return nil, err } @@ -155,7 +176,7 @@ func (esc *ElasticSearchCommand) renderNodeQuery(parameters map[string]interface return v, nil } -func (esc *ElasticSearchCommand) RenderQueryToYAML(parameters map[string]interface{}) (string, error) { +func (esc *ElasticSearchCommand) RenderQueryToYAML(parameters map[string]interface{}, additionalTags map[string]emrichen.TagFunc) (string, error) { if esc.QueryStringTemplate != "" { tmpl := templating.CreateTemplate("query") tmpl, err := tmpl.Parse(esc.QueryStringTemplate) @@ -171,7 +192,7 @@ func (esc *ElasticSearchCommand) RenderQueryToYAML(parameters map[string]interfa return s, nil } - node, err := esc.renderNodeQuery(parameters) + node, err := esc.renderNodeQuery(parameters, additionalTags) if err != nil { return "", err } @@ -190,16 +211,16 @@ func (esc *ElasticSearchCommand) RenderQueryToYAML(parameters map[string]interfa return string(ys), nil } -func (esc *ElasticSearchCommand) RenderQueryToJSON(parameters map[string]interface{}) (string, error) { +func (esc *ElasticSearchCommand) RenderQueryToJSON(parameters map[string]interface{}, additionalTags map[string]emrichen.TagFunc) (string, error) { if esc.QueryStringTemplate != "" { - ys, err := esc.RenderQueryToYAML(parameters) + ys, err := esc.RenderQueryToYAML(parameters, additionalTags) if err != nil { return "", err } return files.ConvertYAMLMapToJSON(ys) } - node, err := esc.renderNodeQuery(parameters) + node, err := esc.renderNodeQuery(parameters, additionalTags) if err != nil { return "", err } @@ -277,8 +298,18 @@ func (esc *ElasticSearchCommand) RunQueryIntoGlaze( return err } + embeddingsSettings := &embeddings.EmbeddingsConfig{} + err = parsedLayers.InitializeStruct(embeddings.EmbeddingsSlug, embeddingsSettings) + if err != nil { + return err + } + embeddingsFactory := embeddings.NewSettingsFactory(embeddingsSettings) + additionalTags := map[string]emrichen.TagFunc{ + "!Embeddings": embeddingsFactory.GetEmbeddingTagFunc(), + } + ps_ := parsedLayers.GetDataMap() - query, err := esc.RenderQueryToJSON(ps_) + query, err := esc.RenderQueryToJSON(ps_, additionalTags) if err != nil { return errors.Wrapf(err, "Could not generate query") } diff --git a/pkg/cmds/cobra.go b/pkg/cmds/cobra.go index 4781b4b..0c8a044 100644 --- a/pkg/cmds/cobra.go +++ b/pkg/cmds/cobra.go @@ -2,6 +2,7 @@ package cmds import ( "github.com/go-go-golems/escuse-me/pkg/cmds/layers" + "github.com/go-go-golems/geppetto/pkg/embeddings" "github.com/go-go-golems/glazed/pkg/cli" "github.com/go-go-golems/glazed/pkg/cmds" layers2 "github.com/go-go-golems/glazed/pkg/cmds/layers" @@ -16,7 +17,7 @@ func BuildCobraCommandWithEscuseMeMiddlewares( ) (*cobra.Command, error) { options_ := append([]cli.CobraParserOption{ cli.WithCobraMiddlewaresFunc(GetCobraCommandEscuseMeMiddlewares), - cli.WithCobraShortHelpLayers(layers2.DefaultSlug, layers.EsConnectionSlug, layers.ESHelpersSlug), + cli.WithCobraShortHelpLayers(layers2.DefaultSlug, layers.EsConnectionSlug, layers.ESHelpersSlug, embeddings.EmbeddingsSlug), }, options...) return cli.BuildCobraCommandFromCommand(cmd, options_...) @@ -46,6 +47,7 @@ func GetCobraCommandEscuseMeMiddlewares( []string{ layers.EsConnectionSlug, layers.ESHelpersSlug, + embeddings.EmbeddingsSlug, }, middlewares.GatherFlagsFromViper(parameters.WithParseStepSource("viper")), ), From 2dc97674059c2cc316fd851d58d52c90f43cee0d Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Fri, 31 Jan 2025 11:38:05 -0500 Subject: [PATCH 2/4] :sparkles: Add support for embeddings --- .vscode/launch.json | 51 +++- changelog.md | 67 ++++++ cmd/escuse-me/main.go | 6 +- cmd/escuse-me/queries/doc/README | 1 - .../examples/search-summaries-embeddings.yaml | 30 ++- pkg/doc.go | 5 - pkg/doc/doc.go | 14 ++ .../doc/topics/01-escuse-me.md | 0 .../doc/topics/02-writing-yaml-commands.md | 0 .../doc/topics/03-index-management.md | 0 pkg/doc/topics/03-writing-queries.md | 227 ++++++++++++++++++ .../doc/topics/04-document-management.md | 0 12 files changed, 375 insertions(+), 26 deletions(-) create mode 100644 changelog.md delete mode 100644 cmd/escuse-me/queries/doc/README delete mode 100644 pkg/doc.go create mode 100644 pkg/doc/doc.go rename {cmd/escuse-me => pkg}/doc/topics/01-escuse-me.md (100%) rename {cmd/escuse-me => pkg}/doc/topics/02-writing-yaml-commands.md (100%) rename {cmd/escuse-me => pkg}/doc/topics/03-index-management.md (100%) create mode 100644 pkg/doc/topics/03-writing-queries.md rename {cmd/escuse-me => pkg}/doc/topics/04-document-management.md (100%) diff --git a/.vscode/launch.json b/.vscode/launch.json index faf2b20..20ad59a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,7 +4,6 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - { "name": "Launch Package", "type": "go", @@ -18,7 +17,12 @@ "request": "launch", "mode": "auto", "program": "${workspaceFolder}/cmd/escuse-me", - "args": ["mento", "index-stats", "--output", "yaml"], + "args": [ + "mento", + "index-stats", + "--output", + "yaml" + ], "envFile": "${workspaceFolder}/.envrc" }, { @@ -27,7 +31,28 @@ "request": "launch", "mode": "auto", "program": "${workspaceFolder}/cmd/escuse-me", - "args": ["examples", "search-summaries-embeddings", "--query", "test", "--print-query"], + "args": [ + "examples", + "search-summaries-embeddings", + "--query", + "test", + "--print-query" + ], + "envFile": "${workspaceFolder}/.envrc" + }, + { + "name": "Search Summaries Embeddings (Alt)", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/escuse-me", + "args": [ + "examples", + "search-summaries-embeddings", + "--query", + "test", + "--print-query" + ], "envFile": "${workspaceFolder}/.envrc" }, { @@ -37,7 +62,25 @@ "mode": "auto", "program": "${workspaceFolder}/../go-emrichen/cmd/emrichen", "cwd": "${workspaceFolder}/../go-emrichen", - "args": ["process", "test-data/defaults-var-format.yml"] + "args": [ + "process", + "test-data/defaults-var-format.yml" + ] + }, + { + "name": "Search Summaries Embeddings (Alt)", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/escuse-me", + "args": [ + "examples", + "search-summaries-embeddings", + "--query", + "test", + "--print-query" + ], + "envFile": "${workspaceFolder}/.envrc" } ] } \ No newline at end of file diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..16698b7 --- /dev/null +++ b/changelog.md @@ -0,0 +1,67 @@ +## Enhanced Error Handling with Raw Results + +Added support for printing raw error responses when the --raw-results flag is enabled. This helps with debugging by showing the complete error response from Elasticsearch. + +- Print complete error response to stderr when raw-results is enabled +- Print error reason and root cause to stderr when raw-results is disabled + +# Refactor Embeddings Settings Factory + +Simplified the embeddings settings factory to use a minimal configuration struct instead of depending on the full StepSettings. Added backwards compatibility method. + +- Created new EmbeddingsConfig struct for minimal configuration +- Modified SettingsFactory to use EmbeddingsConfig instead of StepSettings +- Added NewSettingsFactoryFromStepSettings for backwards compatibility + +# Fix Embeddings Settings Type Handling + +Fixed type handling in embeddings settings to properly handle pointer types in StepSettings and non-pointer types in EmbeddingsConfig. + +- Updated CreateEmbeddingsConfig to properly dereference pointer types +- Modified NewProvider to handle non-pointer types in EmbeddingsConfig +- Fixed error checks to use empty string checks instead of nil checks + +# Add Provider Options for Embeddings Factory + +Added functional options pattern to the embeddings provider factory for more flexible configuration. + +- Added WithType, WithEngine, WithBaseURL, WithAPIKey, and WithDimensions option functions +- Modified NewProvider to accept variadic options +- Improved configuration handling with options overriding defaults + +# Add Custom Tags Documentation + +Added comprehensive documentation for implementing custom tags in go-emrichen, including: +- Basic tag implementation patterns +- Argument handling and validation +- Environment interaction +- Node processing utilities +- Testing guidelines and best practices +- Conceptual explanations and rationale for design patterns +- Detailed best practices and common patterns +- In-depth discussion of error handling and type safety + +# Fix Custom Tags Documentation Signature + +Updated custom tags documentation to reflect correct function signature: +- Changed tag handler signature to include interpreter parameter +- Clarified pure function nature of tag handlers +- Updated all code examples to use correct signature +- Added explanation of interpreter parameter usage + +# Enhance Custom Tags Documentation with ParseArgs Guidelines + +Added detailed documentation about argument handling and recursive processing: +- Comprehensive guide to using ParseArgs +- Core principles for implementing tag handlers +- Examples of proper argument validation and processing +- Guidelines for recursive processing of nested structures +- Detailed error handling patterns for arguments + +# Update Custom Tags Documentation with Proper Namespace + +Updated custom tags documentation to use proper import paths and namespaces: +- Added proper import statements for all examples +- Updated all type references to use emrichen namespace +- Fixed function signatures to use emrichen.Interpreter +- Updated utility function calls to use emrichen namespace \ No newline at end of file diff --git a/cmd/escuse-me/main.go b/cmd/escuse-me/main.go index 030c66b..234327c 100644 --- a/cmd/escuse-me/main.go +++ b/cmd/escuse-me/main.go @@ -14,6 +14,7 @@ import ( "github.com/go-go-golems/escuse-me/cmd/escuse-me/cmds/indices" es_cmds "github.com/go-go-golems/escuse-me/pkg/cmds" "github.com/go-go-golems/escuse-me/pkg/cmds/layers" + "github.com/go-go-golems/escuse-me/pkg/doc" "github.com/go-go-golems/glazed/pkg/cli" glazed_cmds "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/alias" @@ -113,15 +114,12 @@ var runCommandCmd = &cobra.Command{ }, } -//go:embed doc/* -var docFS embed.FS - //go:embed queries/* var queriesFS embed.FS func initRootCmd() (*help.HelpSystem, error) { helpSystem := help.NewHelpSystem() - err := helpSystem.LoadSectionsFromFS(docFS, ".") + err := doc.AddDocToHelpSystem(helpSystem) cobra.CheckErr(err) helpSystem.SetupCobraRootCommand(rootCmd) diff --git a/cmd/escuse-me/queries/doc/README b/cmd/escuse-me/queries/doc/README deleted file mode 100644 index 0bebca9..0000000 --- a/cmd/escuse-me/queries/doc/README +++ /dev/null @@ -1 +0,0 @@ -GO GO README \ No newline at end of file diff --git a/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml b/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml index 75851a3..73c0e73 100644 --- a/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml +++ b/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml @@ -6,19 +6,25 @@ flags: - name: query type: string help: "Search text to find similar content" + required: true + - name: k + type: int + help: "Number of results to return" + default: 5 default-index: local-testing-multi-document-summarization query: - embeddings: !Embeddings - query: {{query}} - # _source: ["content"] - # size: 5 - # query: - # knn: - # field: content_vector - # query_vector: !Embeddings - # query: {{query}} - # # query_vector: !Embeddings - # # - !Var query - # k: 5 + query: + _source: ["content", "title", "url"] + size: !Var k + knn: + field: content_vector + query_vector: !Embeddings + text: !Var query + config: + type: "openai" + engine: "text-embedding-3-small" + dimensions: 1536 + k: !Var k + num_candidates: 100 diff --git a/pkg/doc.go b/pkg/doc.go deleted file mode 100644 index 0fc01a9..0000000 --- a/pkg/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -package pkg - -func init() { - -} diff --git a/pkg/doc/doc.go b/pkg/doc/doc.go new file mode 100644 index 0000000..aa28923 --- /dev/null +++ b/pkg/doc/doc.go @@ -0,0 +1,14 @@ +package doc + +import ( + "embed" + + "github.com/go-go-golems/glazed/pkg/help" +) + +//go:embed * +var docFS embed.FS + +func AddDocToHelpSystem(helpSystem *help.HelpSystem) error { + return helpSystem.LoadSectionsFromFS(docFS, ".") +} diff --git a/cmd/escuse-me/doc/topics/01-escuse-me.md b/pkg/doc/topics/01-escuse-me.md similarity index 100% rename from cmd/escuse-me/doc/topics/01-escuse-me.md rename to pkg/doc/topics/01-escuse-me.md diff --git a/cmd/escuse-me/doc/topics/02-writing-yaml-commands.md b/pkg/doc/topics/02-writing-yaml-commands.md similarity index 100% rename from cmd/escuse-me/doc/topics/02-writing-yaml-commands.md rename to pkg/doc/topics/02-writing-yaml-commands.md diff --git a/cmd/escuse-me/doc/topics/03-index-management.md b/pkg/doc/topics/03-index-management.md similarity index 100% rename from cmd/escuse-me/doc/topics/03-index-management.md rename to pkg/doc/topics/03-index-management.md diff --git a/pkg/doc/topics/03-writing-queries.md b/pkg/doc/topics/03-writing-queries.md new file mode 100644 index 0000000..bd003f2 --- /dev/null +++ b/pkg/doc/topics/03-writing-queries.md @@ -0,0 +1,227 @@ +--- +Title: Writing Embeddings Queries +Slug: writing-embeddings-queries +Short: Learn how to write effective embeddings queries for escuse-me using YAML +Topics: + - usage + - queries + - yaml +Commands: + - query + - run +IsTemplate: false +IsTopLevel: true +ShowPerDefault: true +SectionType: GeneralTopic +--- + +## Embeddings Query Structure + +Embedding queries are written in YAML and can use emrichen tags for advanced functionality. A basic query consists of: + +1. **Text**: The text to be embedded +2. **Configuration**: Optional settings to customize the embedding process + +Here's a simple example: + +```yaml +!Embeddings + text: "This is the text I want to embed" +``` + +## Configuration Options + +The embedding settings can be configured in three ways, in order of precedence: + +1. Command-line flags +2. Environment variables +3. Configuration file +4. Query-specific configuration (in the YAML file) + +### Command-line Flags + +The following flags are available: + +``` +--embeddings-dimensions Output dimension of embeddings (default: 1536 for OpenAI, 384 for Ollama all-minilm) +--embeddings-engine Model to use (default: "text-embedding-3-small") +--embeddings-type Provider type ("openai" or "ollama") (default: "openai") +--ollama-api-key API key for Ollama +--ollama-base-url Base URL for Ollama (default: "http://localhost:11434") +--openai-api-key API key for OpenAI +--openai-base-url Base URL for OpenAI (default: "https://api.openai.com/v1") +``` + +### Environment Variables + +All settings can be configured through environment variables with the prefix `ESCUSE_ME_`: + +```bash +ESCUSE_ME_EMBEDDINGS_DIMENSIONS=1536 +ESCUSE_ME_EMBEDDINGS_ENGINE=text-embedding-3-small +ESCUSE_ME_EMBEDDINGS_TYPE=openai +ESCUSE_ME_OLLAMA_API_KEY=your-key +ESCUSE_ME_OLLAMA_BASE_URL=http://localhost:11434 +ESCUSE_ME_OPENAI_API_KEY=your-key +ESCUSE_ME_OPENAI_BASE_URL=https://api.openai.com/v1 +``` + +### Configuration File + +The same settings can be specified in a configuration file: + +```yaml +embeddings: + dimensions: 1536 + engine: text-embedding-3-small + type: openai + ollama-api-key: your-key + ollama-base-url: http://localhost:11434 + openai-api-key: your-key + openai-base-url: https://api.openai.com/v1 +``` + +### Query-specific Configuration + +You can override any of the above settings in your query YAML file: + +```yaml +!Embeddings + text: "This is the text I want to embed" + config: + type: "openai" + engine: "text-embedding-3-small" + dimensions: 1536 + base_url: "https://api.openai.com/v1" + api_key: "your-api-key" +``` + +The configuration options are applied in order of precedence, with query-specific settings overriding CLI flags, which override environment variables, which override the config file. + +## Provider-Specific Settings + +### OpenAI + +Default settings for OpenAI: +- dimensions: 1536 +- engine: text-embedding-3-small +- base_url: https://api.openai.com/v1 + +Example configuration: + +```yaml +!Embeddings + text: "My text" + config: + type: "openai" + engine: "text-embedding-3-small" + api_key: "your-openai-api-key" +``` + +### Ollama + +Default settings for Ollama: +- dimensions: 384 (for all-minilm) +- engine: all-minilm +- base_url: http://localhost:11434 + +Example configuration: + +```yaml +!Embeddings + text: "My text" + config: + type: "ollama" + engine: "all-minilm" + base_url: "http://localhost:11434" +``` + +## Using Variables + +You can use emrichen variables in your queries for dynamic content: + +```yaml +!Var text_to_embed: "This is my text" + +result: !Embeddings + text: !Var text_to_embed +``` + +## Best Practices + +1. **Configuration Management**: + - Use environment variables or config files for sensitive data like API keys + - Set common defaults in the config file + - Override specific settings via CLI flags when needed + - Use query-specific config only for one-off changes + +2. **Text Preparation**: + - Keep text concise and focused + - Remove unnecessary formatting + - Consider text length limits of your chosen model + +3. **Error Handling**: + - Validate your YAML syntax + - Check for required fields + - Monitor API rate limits + +## Examples + +### Basic Text Embedding + +```yaml +simple: !Embeddings + text: "Hello, world!" +``` + +### Using Variables and Templates + +```yaml +!Var texts: + - "First text" + - "Second text" + +embeddings: !Loop + over: !Var texts + template: + !Embeddings + text: !Var item +``` + +### Complex Configuration + +```yaml +!Var api_key: !Env OPENAI_API_KEY + +result: !Embeddings + text: "Text to embed" + config: + type: "openai" + engine: "text-embedding-3-small" + dimensions: 1536 + api_key: !Var api_key +``` + +## Troubleshooting + +Common issues and their solutions: + +1. **Configuration Issues**: + - Check configuration precedence (query → CLI → env → config file) + - Verify environment variables are properly set + - Ensure config file is in the correct location + +2. **Missing Required Fields**: + - Ensure 'text' field is present + - Verify provider-specific required config + +3. **API Errors**: + - Check API key validity + - Verify network connectivity + - Review rate limits + +## Next Steps + +- Explore advanced emrichen tags for complex queries +- Learn about batch processing for multiple texts +- Understand embedding storage and retrieval \ No newline at end of file diff --git a/cmd/escuse-me/doc/topics/04-document-management.md b/pkg/doc/topics/04-document-management.md similarity index 100% rename from cmd/escuse-me/doc/topics/04-document-management.md rename to pkg/doc/topics/04-document-management.md From 034c53dfbf7b93d417561fdbf7a7e2b31dc3b7dc Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Fri, 14 Feb 2025 18:24:48 -0500 Subject: [PATCH 3/4] :sparkles: Update to use new split layers --- cmd/escuse-me/cmds/serve.go | 11 +++--- cmd/escuse-me/main.go | 2 + .../queries/examples/search-insights.yaml | 36 +++++++++++++++++ .../examples/search-summaries-embeddings.yaml | 3 +- go.mod | 13 +++++-- go.sum | 33 +++++++++++++--- pkg/cmds/cobra.go | 39 ++++++++++++++++++- 7 files changed, 120 insertions(+), 17 deletions(-) create mode 100644 cmd/escuse-me/queries/examples/search-insights.yaml diff --git a/cmd/escuse-me/cmds/serve.go b/cmd/escuse-me/cmds/serve.go index 6768f3e..50b37a7 100644 --- a/cmd/escuse-me/cmds/serve.go +++ b/cmd/escuse-me/cmds/serve.go @@ -2,6 +2,10 @@ package cmds import ( "context" + "os" + "os/signal" + "path/filepath" + es_cmds "github.com/go-go-golems/escuse-me/pkg/cmds" es_layers "github.com/go-go-golems/escuse-me/pkg/cmds/layers" "github.com/go-go-golems/glazed/pkg/cmds" @@ -17,9 +21,6 @@ import ( "github.com/go-go-golems/parka/pkg/server" "github.com/pkg/errors" "golang.org/x/sync/errgroup" - "os" - "os/signal" - "path/filepath" ) type ServeCommand struct { @@ -147,7 +148,7 @@ func (s *ServeCommand) runWithConfigFile( commandDirHandlerOptions, command_dir.WithGenericCommandHandlerOptions( generic_command.WithParameterFilterOptions( - config.WithLayerDefaults( + config.WithMergeOverrideLayer( esConnectionLayer.Layer.GetSlug(), esConnectionLayer.Parameters.ToMap(), ), @@ -254,7 +255,7 @@ func (s *ServeCommand) Run( command_dir.WithGenericCommandHandlerOptions( generic_command.WithTemplateLookup(datatables.NewDataTablesLookupTemplate()), generic_command.WithParameterFilterOptions( - config.WithLayerDefaults( + config.WithMergeOverrideLayer( esClientLayer.Layer.GetSlug(), esClientLayer.Parameters.ToMap(), ), diff --git a/cmd/escuse-me/main.go b/cmd/escuse-me/main.go index 234327c..c8da57b 100644 --- a/cmd/escuse-me/main.go +++ b/cmd/escuse-me/main.go @@ -187,6 +187,8 @@ func initAllCommands(helpSystem *help.HelpSystem) error { repositories_, cli.WithCobraMiddlewaresFunc(es_cmds.GetCobraCommandEscuseMeMiddlewares), cli.WithCobraShortHelpLayers(glazed_layers.DefaultSlug, layers.EsConnectionSlug, layers.ESHelpersSlug), + cli.WithProfileSettingsLayer(), + cli.WithCreateCommandSettingsLayer(), ) if err != nil { return err diff --git a/cmd/escuse-me/queries/examples/search-insights.yaml b/cmd/escuse-me/queries/examples/search-insights.yaml new file mode 100644 index 0000000..d0ed87f --- /dev/null +++ b/cmd/escuse-me/queries/examples/search-insights.yaml @@ -0,0 +1,36 @@ +name: "search-insights" +short: "Search through summaries using embeddings and keywords" +long: "Search through the summaries using both semantic similarity via embeddings and keyword/fuzzy matching for better results" + +flags: + - name: query + type: string + help: "Search text to find similar content" + required: true + - name: k + type: int + help: "Number of results to return" + default: 5 + +default-index: local-testing-multi-document-summarization + +query: + _source: ["content", "title", "url"] + query: + bool: + should: + - knn: + field: content_vector + query_vector: !Embeddings + text: !Var query + config: + type: "openai" + engine: "text-embedding-3-small" + dimensions: 1536 + k: !Var k + num_candidates: 100 + - match: + content: + query: !Var query + boost: 4 + fuzziness: "AUTO" \ No newline at end of file diff --git a/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml b/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml index 73c0e73..880f323 100644 --- a/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml +++ b/cmd/escuse-me/queries/examples/search-summaries-embeddings.yaml @@ -15,9 +15,8 @@ flags: default-index: local-testing-multi-document-summarization query: + _source: ["content", "title", "url"] query: - _source: ["content", "title", "url"] - size: !Var k knn: field: content_vector query_vector: !Embeddings diff --git a/go.mod b/go.mod index 70dc977..8087418 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,10 @@ toolchain go1.23.3 require ( github.com/elastic/go-elasticsearch/v8 v8.17.0 github.com/go-go-golems/clay v0.1.20 - github.com/go-go-golems/glazed v0.5.24 - github.com/go-go-golems/go-emrichen v0.0.3 - github.com/go-go-golems/parka v0.5.15 + github.com/go-go-golems/geppetto v0.4.34 + github.com/go-go-golems/glazed v0.5.26 + github.com/go-go-golems/go-emrichen v0.0.4 + github.com/go-go-golems/parka v0.5.17 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 @@ -24,6 +25,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect + github.com/ThreeDotsLabs/watermill v1.3.7 // indirect github.com/adrg/frontmatter v0.2.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect @@ -47,6 +49,7 @@ require ( github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/glamour v0.7.0 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect @@ -59,9 +62,11 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/huandu/go-clone v1.7.2 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.12.0 // indirect github.com/itchyny/gojq v0.12.12 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible // indirect @@ -70,6 +75,7 @@ require ( github.com/kucherenkovova/safegroup v1.0.2 // indirect github.com/labstack/echo/v4 v4.12.0 // indirect github.com/labstack/gommon v0.4.2 // indirect + github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -91,6 +97,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sashabaranov/go-openai v1.36.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.0 // indirect diff --git a/go.sum b/go.sum index 0f73300..25c405d 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/ThreeDotsLabs/watermill v1.3.7 h1:NV0PSTmuACVEOV4dMxRnmGXrmbz8U83LENOvpHekN7o= +github.com/ThreeDotsLabs/watermill v1.3.7/go.mod h1:lBnrLbxOjeMRgcJbv+UiZr8Ylz8RkJ4m6i/VN/Nk+to= github.com/adrg/frontmatter v0.2.0 h1:/DgnNe82o03riBd1S+ZDjd43wAmC6W35q67NHeLkPd4= github.com/adrg/frontmatter v0.2.0/go.mod h1:93rQCj3z3ZlwyxxpQioRKC1wDLto4aXHrbqIsnH9wmE= github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= @@ -59,6 +61,8 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/glamour v0.7.0 h1:2BtKGZ4iVJCDfMF229EzbeR1QRKLWztO9dMtjmqZSng= github.com/charmbracelet/glamour v0.7.0/go.mod h1:jUMh5MeihljJPQbJ/wf4ldw2+yBP59+ctV36jASy7ps= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -81,12 +85,14 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-go-golems/clay v0.1.20 h1:KUTbDBA/Q7vgG22B9uBnwDpacwG2+bMavQS8SDwolks= github.com/go-go-golems/clay v0.1.20/go.mod h1:hyQirWoEICmaSTcAiPRy7If1n5JEncPi4WVM6tivjoY= -github.com/go-go-golems/glazed v0.5.24 h1:0jMavScvwdh7MIhoTOZ2hLNjVX06bPirjWS8o6YfzQo= -github.com/go-go-golems/glazed v0.5.24/go.mod h1:px67s8RtyjXsKGuSBWJt6HNny53e2tP9Ew6rRcZHe8k= -github.com/go-go-golems/go-emrichen v0.0.3 h1:Tnk6L6IgRFPM/e2Gwnbful6eiWyu28Jksi25eLg9M4w= -github.com/go-go-golems/go-emrichen v0.0.3/go.mod h1:yYf0DLUYLLZdvCODdpIBYUxgNz0ZomJFlGhhzlg+fs0= -github.com/go-go-golems/parka v0.5.15 h1:14yTJsNXtMw86dOUrZDU7YMEuKjZlvBgOP/V7bCjjFA= -github.com/go-go-golems/parka v0.5.15/go.mod h1:tYskoVDbjNZ6hNqI2wrbFy2AFShrzkCiTwAZ4+nBPj4= +github.com/go-go-golems/geppetto v0.4.34 h1:kVQqVqrXPPa+4tRPqAxTbvOAomM9XKd255nuUdBFCfE= +github.com/go-go-golems/geppetto v0.4.34/go.mod h1:HGEsHKvH8HKH89CLWIcueYm46bue7LdFTtsFos3Uzyo= +github.com/go-go-golems/glazed v0.5.26 h1:/Y+Sq6An0IyRVRG1shjV+FZmcOplJ6NvzbQ1edYw3QU= +github.com/go-go-golems/glazed v0.5.26/go.mod h1:/ZgeDXELDOcAkD505fijARmbF6x5Ev7oewNV4V6Andk= +github.com/go-go-golems/go-emrichen v0.0.4 h1:U8AKGaxBDjMghiZZe/7sRYiw3UPqRkkbAAc/d2Q9rK4= +github.com/go-go-golems/go-emrichen v0.0.4/go.mod h1:yYf0DLUYLLZdvCODdpIBYUxgNz0ZomJFlGhhzlg+fs0= +github.com/go-go-golems/parka v0.5.17 h1:XIyqLFmMwd253+J+jdOTV1F5H8xjVChe6+4UvjDN4SM= +github.com/go-go-golems/parka v0.5.17/go.mod h1:gjUXXumO+yrysFQhbzuwKo+l2u5+eJoo51DRHFjlDoU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -101,20 +107,31 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= +github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= +github.com/huandu/go-clone v1.7.2 h1:3+Aq0Ed8XK+zKkLjE2dfHg0XrpIfcohBE1K+c8Usxoo= +github.com/huandu/go-clone v1.7.2/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/itchyny/gojq v0.12.12 h1:x+xGI9BXqKoJQZkr95ibpe3cdrTbY8D9lonrK433rcA= github.com/itchyny/gojq v0.12.12/go.mod h1:j+3sVkjxwd7A7Z5jrbKibgOLn0ZfLWkV+Awxr/pyzJE= github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= @@ -138,6 +155,8 @@ github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+k github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= +github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= +github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -199,6 +218,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sashabaranov/go-openai v1.36.0 h1:fcSrn8uGuorzPWCBp8L0aCR95Zjb/Dd+ZSML0YZy9EI= +github.com/sashabaranov/go-openai v1.36.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= diff --git a/pkg/cmds/cobra.go b/pkg/cmds/cobra.go index 0c8a044..4700929 100644 --- a/pkg/cmds/cobra.go +++ b/pkg/cmds/cobra.go @@ -1,6 +1,9 @@ package cmds import ( + "fmt" + "os" + "github.com/go-go-golems/escuse-me/pkg/cmds/layers" "github.com/go-go-golems/geppetto/pkg/embeddings" "github.com/go-go-golems/glazed/pkg/cli" @@ -24,10 +27,22 @@ func BuildCobraCommandWithEscuseMeMiddlewares( } func GetCobraCommandEscuseMeMiddlewares( - commandSettings *cli.GlazedCommandSettings, + parsedCommandLayers *layers2.ParsedLayers, cmd *cobra.Command, args []string, ) ([]middlewares.Middleware, error) { + commandSettings := &cli.CommandSettings{} + err := parsedCommandLayers.InitializeStruct(cli.CommandSettingsSlug, commandSettings) + if err != nil { + return nil, err + } + + profileSettings := &cli.ProfileSettings{} + err = parsedCommandLayers.InitializeStruct(cli.ProfileSettingsSlug, profileSettings) + if err != nil { + return nil, err + } + middlewares_ := []middlewares.Middleware{ middlewares.ParseFromCobraCommand(cmd, parameters.WithParseStepSource("cobra"), @@ -42,6 +57,28 @@ func GetCobraCommandEscuseMeMiddlewares( middlewares.LoadParametersFromFile(commandSettings.LoadParametersFromFile)) } + xdgConfigPath, err := os.UserConfigDir() + if err != nil { + return nil, err + } + + defaultProfileFile := fmt.Sprintf("%s/escuse-me/profiles.yaml", xdgConfigPath) + if profileSettings.ProfileFile == "" { + profileSettings.ProfileFile = defaultProfileFile + } + if profileSettings.Profile == "" { + profileSettings.Profile = "default" + } + + middlewares_ = append(middlewares_, + middlewares.GatherFlagsFromProfiles( + defaultProfileFile, + profileSettings.ProfileFile, + profileSettings.Profile, + parameters.WithParseStepSource("profiles"), + ), + ) + middlewares_ = append(middlewares_, middlewares.WrapWithWhitelistedLayers( []string{ From 3801440a0a8a7886a0dd2cb41601f58b58ff75aa Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Fri, 14 Feb 2025 19:10:53 -0500 Subject: [PATCH 4/4] :arrow_up: Bump parka --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 8087418..6da0be7 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,11 @@ toolchain go1.23.3 require ( github.com/elastic/go-elasticsearch/v8 v8.17.0 - github.com/go-go-golems/clay v0.1.20 + github.com/go-go-golems/clay v0.1.27 github.com/go-go-golems/geppetto v0.4.34 - github.com/go-go-golems/glazed v0.5.26 + github.com/go-go-golems/glazed v0.5.29 github.com/go-go-golems/go-emrichen v0.0.4 - github.com/go-go-golems/parka v0.5.17 + github.com/go-go-golems/parka v0.5.18 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 diff --git a/go.sum b/go.sum index 25c405d..abf2eee 100644 --- a/go.sum +++ b/go.sum @@ -83,16 +83,16 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-go-golems/clay v0.1.20 h1:KUTbDBA/Q7vgG22B9uBnwDpacwG2+bMavQS8SDwolks= -github.com/go-go-golems/clay v0.1.20/go.mod h1:hyQirWoEICmaSTcAiPRy7If1n5JEncPi4WVM6tivjoY= +github.com/go-go-golems/clay v0.1.27 h1:iIaD9wzeB0AN8mzYQMFAoaYlJgG4bZ4IQoyfJXREAXE= +github.com/go-go-golems/clay v0.1.27/go.mod h1:sUiDlOnC0zP8W47TX0xJKfOdPhHgk2UVjyQCGMrjzEs= github.com/go-go-golems/geppetto v0.4.34 h1:kVQqVqrXPPa+4tRPqAxTbvOAomM9XKd255nuUdBFCfE= github.com/go-go-golems/geppetto v0.4.34/go.mod h1:HGEsHKvH8HKH89CLWIcueYm46bue7LdFTtsFos3Uzyo= -github.com/go-go-golems/glazed v0.5.26 h1:/Y+Sq6An0IyRVRG1shjV+FZmcOplJ6NvzbQ1edYw3QU= -github.com/go-go-golems/glazed v0.5.26/go.mod h1:/ZgeDXELDOcAkD505fijARmbF6x5Ev7oewNV4V6Andk= +github.com/go-go-golems/glazed v0.5.29 h1:3KaYdZBmdIymFRwhAhscFbTrpwhLx37VIgCHN+0bDX8= +github.com/go-go-golems/glazed v0.5.29/go.mod h1:/ZgeDXELDOcAkD505fijARmbF6x5Ev7oewNV4V6Andk= github.com/go-go-golems/go-emrichen v0.0.4 h1:U8AKGaxBDjMghiZZe/7sRYiw3UPqRkkbAAc/d2Q9rK4= github.com/go-go-golems/go-emrichen v0.0.4/go.mod h1:yYf0DLUYLLZdvCODdpIBYUxgNz0ZomJFlGhhzlg+fs0= -github.com/go-go-golems/parka v0.5.17 h1:XIyqLFmMwd253+J+jdOTV1F5H8xjVChe6+4UvjDN4SM= -github.com/go-go-golems/parka v0.5.17/go.mod h1:gjUXXumO+yrysFQhbzuwKo+l2u5+eJoo51DRHFjlDoU= +github.com/go-go-golems/parka v0.5.18 h1:Rsw/ujSkMQ+Xw8XsZiZ/FoTPF+tzPUAXUFA0Xyqxsuo= +github.com/go-go-golems/parka v0.5.18/go.mod h1:gjUXXumO+yrysFQhbzuwKo+l2u5+eJoo51DRHFjlDoU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=