Change dataset.name to dataset.id and dataset.path (#176)

The dataset name in ECS consists of package.subtype, for example nginx.access. To follow ECS, this is also set now accordingly in the API output of a package.

There are two fields now:

* dataset.id: Unique identifier of a dataset, for example `nginx.access`. This can be configured in a dataset
* dataset.path: This is the name of the directory, for example `access`. This is especially useful for EPM to find the right assets.

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Change `requirements.kibana.version.min/max` to `requirements.kibana.versions: {semver-range}`
 * Encode Kibana objects during packaging. [#157](https://github.com/elastic/integrations-registry/pull/157)
 * Prefix package download url with `/epr/{package-name}`.
+* Remove dataset.name but introduce dataset.id and dataset.path. [#176](https://github.com/elastic/package-registry/pull/176)
 
 ### Bugfixes
 

docs/api/package.json

@@ -50,12 +50,13 @@
   "format_version": "1.0.0",
   "datasets": [
     {
+      "id": "bar",
       "title": "Foo",
-      "name": "foo",
       "release": "beta",
       "type": "logs",
       "ingest_pipeline": "pipeline-entry",
-      "package": "example"
+      "package": "example",
+      "path": "foo"
     }
   ],
   "download": "/epr/example/example-1.0.0.tar.gz",

testdata/package/example-1.0.0/dataset/foo/manifest.yml

@@ -1,3 +1,6 @@
+# This dataset has a different id then the path
+id: bar
+
 title: Foo
 
 # Needs to describe the type of this input

testdata/public/package/default-pipeline-0.0.2/index.json

@@ -18,12 +18,13 @@
   "format_version": "1.0.0",
   "datasets": [
     {
+      "id": "default-pipeline.foo",
       "title": "Foo",
-      "name": "foo",
       "release": "beta",
       "type": "logs",
       "ingest_pipeline": "default",
-      "package": "default-pipeline"
+      "package": "default-pipeline",
+      "path": "foo"
     }
   ],
   "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",

testdata/public/package/example-1.0.0/dataset/foo/manifest.yml

@@ -1,3 +1,6 @@
+# This dataset has a different id then the path
+id: bar
+
 title: Foo
 
 # Needs to describe the type of this input

testdata/public/package/example-1.0.0/index.json

@@ -50,12 +50,13 @@
   "format_version": "1.0.0",
   "datasets": [
     {
+      "id": "bar",
       "title": "Foo",
-      "name": "foo",
       "release": "beta",
       "type": "logs",
       "ingest_pipeline": "pipeline-entry",
-      "package": "example"
+      "package": "example",
+      "path": "foo"
     }
   ],
   "download": "/epr/example/example-1.0.0.tar.gz",

util/dataset.go

@@ -12,14 +12,17 @@ import (
 )
 
 type DataSet struct {
+	ID             string                   `config:"id" json:"id"`
 	Title          string                   `config:"title" json:"title" validate:"required"`
-	Name           string                   `config:"name" json:"name"`
 	Release        string                   `config:"release" json:"release"`
 	Type           string                   `config:"type" json:"type" validate:"required"`
 	IngestPipeline string                   `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty"`
 	Vars           []map[string]interface{} `config:"vars" json:"vars,omitempty"`
 	Streams        []map[string]interface{} `config:"streams" json:"streams,omitempty"`
 	Package        string                   `json:"package"`
+
+	// Generated fields
+	Path string `json:"path"`
 }
 
 type Input struct {
@@ -29,14 +32,14 @@ type Input struct {
 }
 
 func (d *DataSet) Validate() error {
-	pipelineDir := d.Name + "/elasticsearch/ingest-pipeline/"
+	pipelineDir := d.Path + "/elasticsearch/ingest-pipeline/"
 	paths, err := filepath.Glob(pipelineDir + "*")
 	if err != nil {
 		return err
 	}
 
-	if strings.Contains(d.Name, "-") {
-		return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.Name)
+	if strings.Contains(d.ID, "-") {
+		return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.ID)
 	}
 
 	if d.IngestPipeline == "" {
@@ -50,7 +53,7 @@ func (d *DataSet) Validate() error {
 	}
 
 	if d.IngestPipeline == "" && len(paths) > 0 {
-		return fmt.Errorf("Package contains pipelines which are not used: %v, %s", paths, d.Name)
+		return fmt.Errorf("Package contains pipelines which are not used: %v, %s", paths, d.ID)
 	}
 
 	// In case an ingest pipeline is set, check if it is around

util/package.go

@@ -295,30 +295,36 @@ func (p *Package) LoadDataSets(packagePath string) error {
 		return err
 	}
 
-	dataSetNames, err := filepath.Glob("*")
+	datasetPaths, err := filepath.Glob("*")
 	if err != nil {
 		return err
 	}
 
-	for _, dataSetName := range dataSetNames {
+	for _, datasetPath := range datasetPaths {
 		// Check if manifest exists
-		manifestPath := dataSetName + "/manifest.yml"
+		manifestPath := datasetPath + "/manifest.yml"
 		_, err := os.Stat(manifestPath)
 		if err != nil && os.IsNotExist(err) {
 			return errors.Wrapf(err, "manifest does not exist for package: %s", packagePath)
 		}
 
 		manifest, err := yaml.NewConfigWithFile(manifestPath, ucfg.PathSep("."))
 		var d = &DataSet{
-			Name:    dataSetName,
 			Package: p.Name,
+			// This is the name of the directory of the dataset
+			Path: datasetPath,
 		}
 		// go-ucfg automatically calls the `Validate` method on the Dataset object here
 		err = manifest.Unpack(d)
 		if err != nil {
 			return errors.Wrapf(err, "error building dataset in package: %s", p.Name)
 		}
 
+		// if id is not set, {package}.{datasetPath} is the default
+		if d.ID == "" {
+			d.ID = p.Name + "." + datasetPath
+		}
+
 		if d.Release == "" {
 			d.Release = "beta"
 		}