Change dataset.name to dataset.id and dataset.path

The dataset name in ECS consists of package.subtype, for example nginx.access. To follow ECS, this is also set now accordingly in the API output of a package.

There are two fields now:

* dataset.id: Unique identifier of a dataset, for example `nginx.access`. This can be configured in a dataset
* dataset.path: This is the name of the directory, for example `access`. This is especially useful for EPM to find the right assets.

docs/api/package.json

@@ -50,11 +50,12 @@
   "format_version": "1.0.0",
   "datasets": [
     {
+      "id": "bar",
       "title": "Foo",
-      "name": "foo",
       "release": "",
       "type": "logs",
-      "ingest_pipeline": "pipeline-entry"
+      "ingest_pipeline": "pipeline-entry",
+      "path": "foo"
     }
   ],
   "download": "/epr/example/example-1.0.0.tar.gz",

testdata/package/example-1.0.0/dataset/foo/manifest.yml

@@ -1,3 +1,6 @@
+# This dataset has a different id then the path
+id: bar
+
 title: Foo
 
 # Needs to describe the type of this input

testdata/public/package/example-1.0.0/dataset/foo/manifest.yml

@@ -1,3 +1,6 @@
+# This dataset has a different id then the path
+id: bar
+
 title: Foo
 
 # Needs to describe the type of this input

testdata/public/package/example-1.0.0/index.json

@@ -50,11 +50,12 @@
   "format_version": "1.0.0",
   "datasets": [
     {
+      "id": "bar",
       "title": "Foo",
-      "name": "foo",
-      "release": "",
+      "release": "beta",
       "type": "logs",
-      "ingest_pipeline": "pipeline-entry"
+      "ingest_pipeline": "pipeline-entry",
+      "path": "foo"
     }
   ],
   "download": "/epr/example/example-1.0.0.tar.gz",

util/package.go

@@ -6,7 +6,6 @@ package util
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -68,12 +67,15 @@ type Image struct {
 }
 
 type DataSet struct {
+	ID             string                   `config:"id" json:"id"`
 	Title          string                   `config:"title" json:"title" validate:"required"`
-	Name           string                   `config:"name" json:"name"`
 	Release        string                   `config:"release" json:"release"`
 	Type           string                   `config:"type" json:"type" validate:"required"`
 	IngestPipeline string                   `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty"`
 	Vars           []map[string]interface{} `config:"vars" json:"vars,omitempty"`
+
+	// Generated fields
+	Path string `json:"path"`
 }
 
 func (i Image) getPath(p *Package) string {
@@ -293,17 +295,14 @@ func (p *Package) LoadDataSets(packagePath string) error {
 		return err
 	}
 
-	dataSetNames, err := filepath.Glob("*")
+	datasetPaths, err := filepath.Glob("*")
 	if err != nil {
 		return err
 	}
 
-	for _, dataSetName := range dataSetNames {
-		log.Println(dataSetName)
-		// Read manifest file
-
+	for _, datasetPath := range datasetPaths {
 		// Check if manifest exists
-		manifestPath := dataSetName + "/manifest.yml"
+		manifestPath := datasetPath + "/manifest.yml"
 		_, err := os.Stat(manifestPath)
 		if err != nil && os.IsNotExist(err) {
 			return errors.Wrapf(err, "manifest does not exist for package: %s", packagePath)
@@ -315,7 +314,14 @@ func (p *Package) LoadDataSets(packagePath string) error {
 		if err != nil {
 			return err
 		}
-		d.Name = dataSetName
+
+		// This is the name of the directory of the dataset
+		d.Path = datasetPath
+
+		// if id is not set, {package}.{datasetName} is the default
+		if d.ID == "" {
+			d.ID = p.Name + "." + datasetPath
+		}
 
 		if d.Release == "" {
 			d.Release = "beta"