Add capability to load variables from other yaml files

internal/model/app.go

@@ -123,7 +123,7 @@ func readEnvFile(file string) ([]byte, error) {
 		}
 		return b, nil
 	}
-	return ioutil.ReadFile(file)
+	return os.ReadFile(file)
 }
 
 func loadEnvFiles(app *QbecApp, additionalFiles []string, v *validator) error {
@@ -180,9 +180,69 @@ func loadEnvFiles(app *QbecApp, additionalFiles []string, v *validator) error {
 	return nil
 }
 
+func downloadVarFile(url string) ([]byte, error) {
+	return downloadEnvFile(url)
+}
+
+func readVarFile(file string) ([]byte, error) {
+	if filematcher.IsRemoteFile(file) {
+		b, err := downloadVarFile(file)
+		if err != nil {
+			return nil, errors.Wrapf(err, "download vars from %s", file)
+		}
+		return b, nil
+	}
+	return os.ReadFile(file)
+}
+
+func loadVars(app *QbecApp, v *validator) error {
+	if app.Spec.Vars.Computed == nil {
+		app.Spec.Vars.Computed = []ComputedVar{}
+	}
+	if app.Spec.Vars.External == nil {
+		app.Spec.Vars.External = []ExternalVar{}
+	}
+	if app.Spec.Vars.TopLevel == nil {
+		app.Spec.Vars.TopLevel = []TopLevelVar{}
+	}
+	if app.Spec.DataSources == nil {
+		app.Spec.DataSources = []string{}
+	}
+
+	var varsFiles []string
+	varsFiles = append(varsFiles, app.Spec.VarFiles...)
+	var allVarsFiles []string
+	for _, filePattern := range varsFiles {
+		matchedFiles, err := filematcher.Match(filePattern)
+		if err != nil {
+			return err
+		}
+		allVarsFiles = append(allVarsFiles, matchedFiles...)
+	}
+	for _, file := range allVarsFiles {
+		b, err := readVarFile(file)
+		if err != nil {
+			return err
+		}
+		var qVars QbecVars
+		if err := yaml.Unmarshal(b, &qVars); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("%s: unmarshal YAML", file))
+		}
+		errs := v.validateVarYAML(b)
+		if len(errs) > 0 {
+			return makeValError(file, errs)
+		}
+		app.Spec.Vars.Computed = append(qVars.Spec.Computed, app.Spec.Vars.Computed...)
+		app.Spec.Vars.External = append(qVars.Spec.External, app.Spec.Vars.External...)
+		app.Spec.Vars.TopLevel = append(qVars.Spec.TopLevel, app.Spec.Vars.TopLevel...)
+		app.Spec.DataSources = append(qVars.Spec.DataSources, app.Spec.DataSources...)
+	}
+	return nil
+}
+
 // NewApp returns an app loading its details from the supplied file.
 func NewApp(file string, envFiles []string, tag string) (*App, error) {
-	b, err := ioutil.ReadFile(file)
+	b, err := os.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}
@@ -205,6 +265,10 @@ func NewApp(file string, envFiles []string, tag string) (*App, error) {
 		return nil, err
 	}
 
+	if err := loadVars(&qApp, v); err != nil {
+		return nil, err
+	}
+
 	if len(qApp.Spec.Environments) == 0 {
 		return nil, fmt.Errorf("%s: no environments defined for app", file)
 	}

internal/model/swagger-schema.go

@@ -1,20 +1,6 @@
-// Copyright 2021 Splunk Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 package model
 
-// generated by gen-qbec-swagger from internal/model/swagger.yaml at 2022-07-15 05:48:30.172144 +0000 UTC
+// generated by gen-qbec-swagger from internal/model/swagger.yaml at 2024-06-18 16:07:43.222418 +0000 UTC
 // Do NOT edit this file by hand
 
 var swaggerJSON = `
@@ -98,7 +84,7 @@ var swaggerJSON = `
                     "type": "object"
                 },
                 "envFiles": {
-                    "description": "list of additional files containing environment definitions to load.\nenvironment definitions are merged in the order specified starting with any inline environments.\nAn environment defined in a later file takes precedence over the the same environment already loaded\nand replaces it.",
+                    "description": "list of additional files containing environment definitions to load.\nenvironment definitions are merged in the order specified starting with any inline environments.\nAn environment defined in a later file takes precedence over the the same environment already loaded\nand replaces it. But setting \"mergeImportedEnvs\" changes this behaviour to merge.",
                     "items": {
                         "type": "string"
                     },
@@ -142,6 +128,13 @@ var swaggerJSON = `
                     "description": "file containing jsonnet code that can be used to post-process all objects, typically adding metadata like\nannotations",
                     "type": "string"
                 },
+                "varFiles": {
+                    "description": "list of additional files containing variables to load.\nVariables defined in a later file takes precedence over the the same variables already loaded\nand replaces it.",
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
                 "vars": {
                     "$ref": "#/definitions/qbec.io.v1alpha1.Variables"
                 }
@@ -306,6 +299,22 @@ var swaggerJSON = `
             },
             "title": "Variables is a collection of external and top-level variables.",
             "type": "object"
+        },
+        "qbec.io.v1alpha1.VariablesFile": {
+            "additionalProperties": false,
+            "properties": {
+                "dataSources": {
+                    "$ref": "#/definitions/qbec.io.v1alpha1.AppSpec.dataSources"
+                },
+                "dsExamples": {
+                    "$ref": "#/definitions/qbec.io.v1alpha1.AppSpec.dsExamples"
+                },
+                "vars": {
+                    "$ref": "#/definitions/qbec.io.v1alpha1.Variables"
+                }
+            },
+            "title": "VariablesFile is a collection of variables and data sources.",
+            "type": "object"
         }
     },
     "info": {

internal/model/swagger.yaml

@@ -77,6 +77,14 @@ definitions:
           list of additional files containing environment definitions to load.
           environment definitions are merged in the order specified starting with any inline environments.
           An environment defined in a later file takes precedence over the the same environment already loaded
+          and replaces it. But setting "mergeImportedEnvs" changes this behaviour to merge.
+        items:
+          type: string
+        type: array
+      varFiles:
+        description: |-
+          list of additional files containing variables to load.
+          Variables defined in a later file takes precedence over the the same variables already loaded
           and replaces it.
         items:
           type: string
@@ -226,3 +234,14 @@ definitions:
         type: array
         items:
           $ref: "#/definitions/qbec.io.v1alpha1.ComputedVar"
+  qbec.io.v1alpha1.VariablesFile:
+    additionalProperties: false
+    type: object
+    title: VariablesFile is a collection of variables and data sources.
+    properties:
+      vars:
+        $ref: "#/definitions/qbec.io.v1alpha1.Variables"
+      dataSources:
+        $ref: "#/definitions/qbec.io.v1alpha1.AppSpec.dataSources"
+      dsExamples:
+        $ref: "#/definitions/qbec.io.v1alpha1.AppSpec.dsExamples"

internal/model/types.go

@@ -104,6 +104,8 @@ type AppSpec struct {
 	Environments map[string]Environment `json:"environments"`
 	// additional environments pulled in from external files
 	EnvFiles []string `json:"envFiles,omitempty"`
+	// additional vars pulled in from external files
+	VarFiles []string `json:"varFiles,omitempty"`
 	// list of components to exclude by default for every environment
 	Excludes []string `json:"excludes,omitempty"`
 	// list of library paths to add to the jsonnet VM at evaluation
@@ -162,3 +164,25 @@ type QbecApp struct {
 	// required: true
 	Spec AppSpec `json:"spec"`
 }
+
+type VarsSpec struct {
+	Variables
+	DataSources        []string
+	DataSourceExamples map[string]interface{}
+}
+
+type QbecVars struct {
+	// object kind
+	// required: true
+	// pattern: ^App$
+	Kind string `json:"kind"`
+	// requested API version
+	// required: true
+	APIVersion string `json:"apiVersion"`
+	// app metadata
+	// required: true
+	Metadata AppMeta `json:"metadata,omitempty"`
+	// app specification
+	// required: true
+	Spec VarsSpec `json:"spec"`
+}

internal/model/validator.go

@@ -106,3 +106,33 @@ func (v *validator) validateEnvYAML(content []byte) []error {
 	res := ov.Validate(data)
 	return res.Errors
 }
+
+func (v *validator) validateVarYAML(content []byte) []error {
+	wrap := func(err error) []error {
+		return []error{err}
+	}
+	var data map[string]interface{}
+	if err := yaml.Unmarshal(content, &data); err != nil {
+		return wrap(errors.Wrap(err, "YAML unmarshal"))
+	}
+	apiVersion, ok := data["apiVersion"].(string)
+	if !ok {
+		return wrap(fmt.Errorf("missing or invalid apiVersion property"))
+	}
+	kind, ok := data["kind"].(string)
+	if !ok {
+		return wrap(fmt.Errorf("missing or invalid kind property"))
+	}
+	if kind != "VariablesFile" {
+		return wrap(fmt.Errorf("bad kind property, expected VariablesFile"))
+	}
+
+	dataType := strings.Replace(apiVersion, "/", ".", -1) + "." + kind
+	schema, ok := v.swagger.Definitions[dataType]
+	if !ok {
+		return wrap(fmt.Errorf("no schema found for %s (check for valid apiVersion and kind properties)", dataType))
+	}
+	ov := validate.NewSchemaValidator(&schema, v.swagger, "", strfmt.Default)
+	res := ov.Validate(data)
+	return res.Errors
+}