diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index ad4d636164d71..0e239c24dd9cf 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`tests loading fields.yml: base.yml 1`] = ` +exports[`tests loading base.yml: base.yml 1`] = ` { "order": 1, "index_patterns": [ @@ -47,10 +47,12 @@ exports[`tests loading fields.yml: base.yml 1`] = ` "user": { "properties": { "auid": { - "type": "keyword" + "type": "keyword", + "ignore_above": 1024 }, "euid": { - "type": "keyword" + "type": "keyword", + "ignore_above": 1024 } } }, @@ -59,7 +61,10 @@ exports[`tests loading fields.yml: base.yml 1`] = ` "nested": { "properties": { "foo": { - "type": "keyword" + "type": "text" + }, + "bar": { + "type": "long" } } } @@ -68,7 +73,1593 @@ exports[`tests loading fields.yml: base.yml 1`] = ` "nested": { "properties": { "bar": { + "type": "keyword", + "ignore_above": 1024 + }, + "baz": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "myalias": { + "type": "alias", + "path": "user.euid" + }, + "validarray": { + "type": "integer" + } + } + }, + "aliases": {} +} +`; + +exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` +{ + "order": 1, + "index_patterns": [ + "foo-*" + ], + "settings": { + "index": { + "lifecycle": { + "name": "logs-default" + }, + "codec": "best_compression", + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "refresh_interval": "5s", + "number_of_shards": "1", + "query": { + "default_field": [ + "message" + ] + }, + "number_of_routing_shards": "30" + } + }, + "mappings": { + "_meta": { + "package": "foo" + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "coredns": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "query": { + "properties": { + "size": { + "type": "long" + }, + "class": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "response": { + "properties": { + "code": { + "type": "keyword", + "ignore_above": 1024 + }, + "flags": { + "type": "keyword", + "ignore_above": 1024 + }, + "size": { + "type": "long" + } + } + }, + "dnssec_ok": { + "type": "boolean" + } + } + } + } + }, + "aliases": {} +} +`; + +exports[`tests loading system.yml: system.yml 1`] = ` +{ + "order": 1, + "index_patterns": [ + "whatsthis-*" + ], + "settings": { + "index": { + "lifecycle": { + "name": "metrics-default" + }, + "codec": "best_compression", + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "refresh_interval": "5s", + "number_of_shards": "1", + "query": { + "default_field": [ + "message" + ] + }, + "number_of_routing_shards": "30" + } + }, + "mappings": { + "_meta": { + "package": "foo" + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "system": { + "properties": { + "core": { + "properties": { + "id": { + "type": "long" + }, + "user": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + }, + "idle": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "cores": { + "type": "long" + }, + "user": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "idle": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "total": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + } + } + } + } + }, + "diskio": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "serial_number": { + "type": "keyword", + "ignore_above": 1024 + }, + "read": { + "properties": { + "count": { + "type": "long" + }, + "bytes": { + "type": "long" + }, + "time": { + "type": "long" + } + } + }, + "write": { + "properties": { + "count": { + "type": "long" + }, + "bytes": { + "type": "long" + }, + "time": { + "type": "long" + } + } + }, + "io": { + "properties": { + "time": { + "type": "long" + } + } + }, + "iostat": { + "properties": { + "read": { + "properties": { + "request": { + "properties": { + "merges_per_sec": { + "type": "float" + }, + "per_sec": { + "type": "float" + } + } + }, + "per_sec": { + "properties": { + "bytes": { + "type": "float" + } + } + }, + "await": { + "type": "float" + } + } + }, + "write": { + "properties": { + "request": { + "properties": { + "merges_per_sec": { + "type": "float" + }, + "per_sec": { + "type": "float" + } + } + }, + "per_sec": { + "properties": { + "bytes": { + "type": "float" + } + } + }, + "await": { + "type": "float" + } + } + }, + "request": { + "properties": { + "avg_size": { + "type": "float" + } + } + }, + "queue": { + "properties": { + "avg_size": { + "type": "float" + } + } + }, + "await": { + "type": "float" + }, + "service_time": { + "type": "float" + }, + "busy": { + "type": "float" + } + } + } + } + }, + "entropy": { + "properties": { + "available_bits": { + "type": "long" + }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "filesystem": { + "properties": { + "available": { + "type": "long" + }, + "device_name": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "mount_point": { + "type": "keyword", + "ignore_above": 1024 + }, + "files": { + "type": "long" + }, + "free": { + "type": "long" + }, + "free_files": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + } + } + }, + "fsstat": { + "properties": { + "count": { + "type": "long" + }, + "total_files": { + "type": "long" + }, + "total_size": { + "properties": { + "free": { + "type": "long" + }, + "used": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } + }, + "load": { + "properties": { + "1": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "5": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "15": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "norm": { + "properties": { + "1": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "5": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "15": { + "type": "scaled_float", + "scaling_factor": 100 + } + } + }, + "cores": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "free": { + "type": "long" + }, + "actual": { + "properties": { + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "free": { + "type": "long" + } + } + }, + "swap": { + "properties": { + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "free": { + "type": "long" + }, + "out": { + "properties": { + "pages": { + "type": "long" + } + } + }, + "in": { + "properties": { + "pages": { + "type": "long" + } + } + }, + "readahead": { + "properties": { + "pages": { + "type": "long" + }, + "cached": { + "type": "long" + } + } + } + } + }, + "hugepages": { + "properties": { + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "long" + } + } + }, + "free": { + "type": "long" + }, + "reserved": { + "type": "long" + }, + "surplus": { + "type": "long" + }, + "default_size": { + "type": "long" + }, + "swap": { + "properties": { + "out": { + "properties": { + "pages": { + "type": "long" + }, + "fallback": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "network": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "dropped": { + "type": "long" + } + } + }, + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "dropped": { + "type": "long" + } + } + } + } + }, + "network_summary": { + "properties": { + "ip": { + "properties": { + "*": { + "type": "object" + } + } + }, + "tcp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "udp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "udp_lite": { + "properties": { + "*": { + "type": "object" + } + } + }, + "icmp": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "process": { + "properties": { + "state": { + "type": "keyword", + "ignore_above": 1024 + }, + "cmdline": { + "type": "keyword", + "ignore_above": 2048 + }, + "env": { + "type": "object" + }, + "cpu": { + "properties": { + "user": { + "properties": { + "ticks": { + "type": "long" + } + } + }, + "total": { + "properties": { + "value": { + "type": "long" + }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + }, + "norm": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "ticks": { + "type": "long" + } + } + }, + "start_time": { + "type": "date" + } + } + }, + "memory": { + "properties": { + "size": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "share": { + "type": "long" + } + } + }, + "fd": { + "properties": { + "open": { + "type": "long" + }, + "limit": { + "properties": { + "soft": { + "type": "long" + }, + "hard": { + "type": "long" + } + } + } + } + }, + "cgroup": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "path": { + "type": "keyword", + "ignore_above": 1024 + }, + "cpu": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "path": { + "type": "keyword", + "ignore_above": 1024 + }, + "cfs": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "us": { + "type": "long" + } + } + }, + "shares": { + "type": "long" + } + } + }, + "rt": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "runtime": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "stats": { + "properties": { + "periods": { + "type": "long" + }, + "throttled": { + "properties": { + "periods": { + "type": "long" + }, + "ns": { + "type": "long" + } + } + } + } + } + } + }, + "cpuacct": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "path": { + "type": "keyword", + "ignore_above": 1024 + }, + "total": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "stats": { + "properties": { + "user": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "system": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "percpu": { + "type": "object" + } + } + }, + "memory": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "path": { + "type": "keyword", + "ignore_above": 1024 + }, + "mem": { + "properties": { + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "failures": { + "type": "long" + } + } + }, + "memsw": { + "properties": { + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "failures": { + "type": "long" + } + } + }, + "kmem": { + "properties": { + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "failures": { + "type": "long" + } + } + }, + "kmem_tcp": { + "properties": { + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "failures": { + "type": "long" + } + } + }, + "stats": { + "properties": { + "active_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "active_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "cache": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memory_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memsw_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "mapped_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + }, + "major_page_faults": { + "type": "long" + }, + "pages_in": { + "type": "long" + }, + "pages_out": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "rss_huge": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "swap": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "unevictable": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "blkio": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "path": { + "type": "keyword", + "ignore_above": 1024 + }, + "total": { + "properties": { + "bytes": { + "type": "long" + }, + "ios": { + "type": "long" + } + } + } + } + } + } + }, + "summary": { + "properties": { + "total": { + "type": "long" + }, + "running": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "sleeping": { + "type": "long" + }, + "stopped": { + "type": "long" + }, + "zombie": { + "type": "long" + }, + "dead": { + "type": "long" + }, + "unknown": { + "type": "long" + } + } + } + } + }, + "raid": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "status": { + "type": "keyword", + "ignore_above": 1024 + }, + "level": { + "type": "keyword", + "ignore_above": 1024 + }, + "sync_action": { + "type": "keyword", + "ignore_above": 1024 + }, + "disks": { + "properties": { + "active": { + "type": "long" + }, + "total": { + "type": "long" + }, + "spare": { + "type": "long" + }, + "failed": { + "type": "long" + }, + "states": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "blocks": { + "properties": { + "total": { + "type": "long" + }, + "synced": { + "type": "long" + } + } + } + } + }, + "socket": { + "properties": { + "local": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "remote": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "host": { + "type": "keyword", + "ignore_above": 1024 + }, + "etld_plus_one": { + "type": "keyword", + "ignore_above": 1024 + }, + "host_error": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "process": { + "properties": { + "cmdline": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "user": { + "properties": {} + }, + "summary": { + "properties": { + "all": { + "properties": { + "count": { + "type": "long" + }, + "listening": { + "type": "long" + } + } + }, + "tcp": { + "properties": { + "memory": { + "type": "long" + }, + "all": { + "properties": { + "orphan": { + "type": "long" + }, + "count": { + "type": "long" + }, + "listening": { + "type": "long" + }, + "established": { + "type": "long" + }, + "close_wait": { + "type": "long" + }, + "time_wait": { + "type": "long" + }, + "syn_sent": { + "type": "long" + }, + "syn_recv": { + "type": "long" + }, + "fin_wait1": { + "type": "long" + }, + "fin_wait2": { + "type": "long" + }, + "last_ack": { + "type": "long" + }, + "closing": { + "type": "long" + } + } + } + } + }, + "udp": { + "properties": { + "memory": { + "type": "long" + }, + "all": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "uptime": { + "properties": { + "duration": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "users": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "seat": { + "type": "keyword", + "ignore_above": 1024 + }, + "path": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "service": { + "type": "keyword", + "ignore_above": 1024 + }, + "remote": { + "type": "boolean" + }, + "state": { + "type": "keyword", + "ignore_above": 1024 + }, + "scope": { + "type": "keyword", + "ignore_above": 1024 + }, + "leader": { + "type": "long" + }, + "remote_host": { + "type": "keyword", + "ignore_above": 1024 + } + } } } } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts index 005bb78e458e3..de4ba25590c98 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts @@ -12,7 +12,7 @@ import { ElasticsearchAssetType, } from '../../../../types'; import { CallESAsCurrentUser } from '../../../../types'; -import { Field, loadFieldsFromYaml } from '../../fields/field'; +import { Field, loadFieldsFromYaml, processFields } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { generateMappings, generateTemplateName, getTemplate } from './template'; import * as Registry from '../../registry'; @@ -98,7 +98,7 @@ export async function installTemplate({ dataset: Dataset; packageVersion: string; }): Promise { - const mappings = generateMappings(fields); + const mappings = generateMappings(processFields(fields)); const templateName = generateTemplateName(dataset); let pipelineName; if (dataset.ingest_pipeline) { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts index aa5be59b6a5cd..f4e13748641ed 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts @@ -28,15 +28,38 @@ test('get template', () => { expect(template.index_patterns).toStrictEqual([`${templateName}-*`]); }); -test('tests loading fields.yml', () => { - // Load fields.yml file +test('tests loading base.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); const fields: Field[] = safeLoad(fieldsYML); - processFields(fields); - const mappings = generateMappings(fields); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); const template = getTemplate('logs', 'foo', mappings); expect(template).toMatchSnapshot(path.basename(ymlPath)); }); + +test('tests loading coredns.logs.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate('logs', 'foo', mappings); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); +}); + +test('tests loading system.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate('metrics', 'whatsthis', mappings); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); +}); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index f075771e9808a..71c9acc6c10da 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -14,6 +14,10 @@ interface Properties { interface Mappings { properties: any; } + +const DEFAULT_SCALING_FACTOR = 1000; +const DEFAULT_IGNORE_ABOVE = 1024; + /** * getTemplate retrieves the default template but overwrites the index pattern with the given value. * @@ -33,31 +37,98 @@ export function getTemplate( } /** - * Generate mapping takes the given fields array and creates the Elasticsearch + * Generate mapping takes the given nested fields array and creates the Elasticsearch * mapping properties out of it. * + * This assumes that all fields with dotted.names have been expanded in a previous step. + * * @param fields */ export function generateMappings(fields: Field[]): Mappings { const props: Properties = {}; - fields.forEach(field => { - // Are there more fields inside this field? Build them recursively - if (field.fields && field.fields.length > 0) { - props[field.name] = generateMappings(field.fields); - return; - } + // TODO: this can happen when the fields property in fields.yml is present but empty + // Maybe validation should be moved to fields/field.ts + if (fields) { + fields.forEach(field => { + // If type is not defined, assume keyword + const type = field.type || 'keyword'; + + let fieldProps = getDefaultProperties(field); + + switch (type) { + case 'group': + fieldProps = generateMappings(field.fields!); + break; + case 'integer': + fieldProps.type = 'long'; + break; + case 'scaled_float': + fieldProps.type = 'scaled_float'; + fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR; + break; + case 'text': + fieldProps.type = 'text'; + if (field.analyzer) { + fieldProps.analyzer = field.analyzer; + } + if (field.search_analyzer) { + fieldProps.search_analyzer = field.search_analyzer; + } + break; + case 'keyword': + fieldProps.type = 'keyword'; + if (field.ignore_above) { + fieldProps.ignore_above = field.ignore_above; + } else { + fieldProps.ignore_above = DEFAULT_IGNORE_ABOVE; + } + break; + // TODO move handling of multi_fields here? + case 'object': + // TODO improve + fieldProps.type = 'object'; + break; + case 'array': + // this assumes array fields were validated in an earlier step + // adding an array field with no object_type would result in an error + // when the template is added to ES + if (field.object_type) { + fieldProps.type = field.object_type; + } + break; + case 'alias': + // this assumes alias fields were validated in an earlier step + // adding a path to a field that doesn't exist would result in an error + // when the template is added to ES. + fieldProps.type = 'alias'; + fieldProps.path = field.path; + break; + default: + fieldProps.type = type; + } + props[field.name] = fieldProps; + }); + } - // If not type is defined, take keyword - const type = field.type || 'keyword'; - // Only add keyword fields for now - // TODO: add support for other field types - if (type === 'keyword') { - props[field.name] = { type }; - } - }); return { properties: props }; } +function getDefaultProperties(field: Field): Properties { + const properties: Properties = {}; + + if (field.index) { + properties.index = field.index; + } + if (field.doc_values) { + properties.doc_values = field.doc_values; + } + if (field.copy_to) { + properties.copy_to = field.copy_to; + } + + return properties; +} + /** * Generates the template name out of the given information */ diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/__snapshots__/field.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/fields/__snapshots__/field.test.ts.snap index 76991bde77008..5c402b896093a 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/fields/__snapshots__/field.test.ts.snap +++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/__snapshots__/field.test.ts.snap @@ -23,7 +23,12 @@ exports[`tests loading fields.yml: base.yml 1`] = ` "type": "group", "fields": [ { - "name": "foo" + "name": "foo", + "type": "text" + }, + { + "name": "bar", + "type": "integer" } ] } @@ -35,8 +40,21 @@ exports[`tests loading fields.yml: base.yml 1`] = ` "fields": [ { "name": "bar" + }, + { + "name": "baz" } ] + }, + { + "name": "myalias", + "type": "alias", + "path": "user.euid" + }, + { + "name": "validarray", + "type": "array", + "object_type": "integer" } ] `; @@ -54,46 +72,2395 @@ exports[`tests loading fields.yml: coredns.logs.yml 1`] = ` "description": "id of the DNS transaction\\n" }, { - "name": "query.size", - "type": "integer", - "format": "bytes", - "description": "size of the DNS query\\n" + "name": "query", + "type": "group", + "fields": [ + { + "name": "size", + "type": "integer", + "format": "bytes", + "description": "size of the DNS query\\n" + }, + { + "name": "class", + "type": "keyword", + "description": "DNS query class\\n" + }, + { + "name": "name", + "type": "keyword", + "description": "DNS query name\\n" + }, + { + "name": "type", + "type": "keyword", + "description": "DNS query type\\n" + } + ] }, { - "name": "query.class", - "type": "keyword", - "description": "DNS query class\\n" + "name": "response", + "type": "group", + "fields": [ + { + "name": "code", + "type": "keyword", + "description": "DNS response code\\n" + }, + { + "name": "flags", + "type": "keyword", + "description": "DNS response flags\\n" + }, + { + "name": "size", + "type": "integer", + "format": "bytes", + "description": "size of the DNS response\\n" + } + ] }, { - "name": "query.name", - "type": "keyword", - "description": "DNS query name\\n" + "name": "dnssec_ok", + "type": "boolean", + "description": "dnssec flag\\n" + } + ] + } +] +`; + +exports[`tests loading fields.yml: system.yml 1`] = ` +[ + { + "name": "system", + "type": "group", + "fields": [ + { + "name": "core", + "type": "group", + "description": "\`system-core\` contains CPU metrics for a single core of a multi-core system.\\n", + "fields": [ + { + "name": "id", + "type": "long", + "description": "CPU Core number.\\n" + }, + { + "name": "user", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in user space.\\n" + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent in user space.\\n" + } + ] + }, + { + "name": "system", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in kernel space.\\n" + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent in kernel space.\\n" + } + ] + }, + { + "name": "nice", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent on low-priority processes.\\n" + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent on low-priority processes.\\n" + } + ] + }, + { + "name": "idle", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent idle.\\n" + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent idle.\\n" + } + ] + }, + { + "name": "iowait", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in wait (on disk).\\n" + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent in wait (on disk).\\n" + } + ] + }, + { + "name": "irq", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent servicing and handling hardware interrupts.\\n" + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent servicing and handling hardware interrupts.\\n" + } + ] + }, + { + "name": "softirq", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent servicing and handling software interrupts.\\n" + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent servicing and handling software interrupts.\\n" + } + ] + }, + { + "name": "steal", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix.\\n" + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix.\\n" + } + ] + } + ] }, { - "name": "query.type", - "type": "keyword", - "description": "DNS query type\\n" + "name": "cpu", + "type": "group", + "description": "\`cpu\` contains local CPU stats.\\n", + "release": "ga", + "fields": [ + { + "name": "cores", + "type": "long", + "description": "The number of CPU cores present on the host. The non-normalized percentages will have a maximum value of \`100% * cores\`. The normalized percentages already take this value into account and have a maximum value of 100%.\\n" + }, + { + "name": "user", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in user space. On multi-core systems, you can have percentages that are greater than 100%. For example, if 3 cores are at 60% use, then the \`system.cpu.user.pct\` will be 180%.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in user space.\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent in user space.\\n" + } + ] + }, + { + "name": "system", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in kernel space.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in kernel space.\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent in kernel space.\\n" + } + ] + }, + { + "name": "nice", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent on low-priority processes.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent on low-priority processes.\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent on low-priority processes.\\n" + } + ] + }, + { + "name": "idle", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent idle.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent idle.\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent idle.\\n" + } + ] + }, + { + "name": "iowait", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in wait (on disk).\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in wait (on disk).\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent in wait (on disk).\\n" + } + ] + }, + { + "name": "irq", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent servicing and handling hardware interrupts.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent servicing and handling hardware interrupts.\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent servicing and handling hardware interrupts.\\n" + } + ] + }, + { + "name": "softirq", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent servicing and handling software interrupts.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent servicing and handling software interrupts.\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent servicing and handling software interrupts.\\n" + } + ] + }, + { + "name": "steal", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix.\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix.\\n" + } + ] + }, + { + "name": "total", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent in states other than Idle and IOWait.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time in states other than Idle and IOWait, normalised by the number of cores.\\n" + } + ] + } + ] + } + ] }, { - "name": "response.code", - "type": "keyword", - "description": "DNS response code\\n" + "name": "diskio", + "type": "group", + "description": "\`disk\` contains disk IO metrics collected from the operating system.\\n", + "release": "ga", + "fields": [ + { + "name": "name", + "type": "keyword", + "example": "sda1", + "description": "The disk name.\\n" + }, + { + "name": "serial_number", + "type": "keyword", + "description": "The disk's serial number. This may not be provided by all operating systems.\\n" + }, + { + "name": "read", + "type": "group", + "fields": [ + { + "name": "count", + "type": "long", + "description": "The total number of reads completed successfully.\\n" + }, + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The total number of bytes read successfully. On Linux this is the number of sectors read multiplied by an assumed sector size of 512.\\n" + }, + { + "name": "time", + "type": "long", + "description": "The total number of milliseconds spent by all reads.\\n" + } + ] + }, + { + "name": "write", + "type": "group", + "fields": [ + { + "name": "count", + "type": "long", + "description": "The total number of writes completed successfully.\\n" + }, + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The total number of bytes written successfully. On Linux this is the number of sectors written multiplied by an assumed sector size of 512.\\n" + }, + { + "name": "time", + "type": "long", + "description": "The total number of milliseconds spent by all writes.\\n" + } + ] + }, + { + "name": "io", + "type": "group", + "fields": [ + { + "name": "time", + "type": "long", + "description": "The total number of of milliseconds spent doing I/Os.\\n" + } + ] + }, + { + "name": "iostat", + "type": "group", + "fields": [ + { + "name": "read", + "type": "group", + "fields": [ + { + "name": "request", + "type": "group", + "fields": [ + { + "name": "merges_per_sec", + "type": "float", + "description": "The number of read requests merged per second that were queued to the device.\\n" + }, + { + "name": "per_sec", + "type": "float", + "description": "The number of read requests that were issued to the device per second\\n" + } + ] + }, + { + "name": "per_sec", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "float", + "description": "The number of Bytes read from the device per second.\\n", + "format": "bytes" + } + ] + }, + { + "name": "await", + "type": "float", + "description": "The average time spent for read requests issued to the device to be served.\\n" + } + ] + }, + { + "name": "write", + "type": "group", + "fields": [ + { + "name": "request", + "type": "group", + "fields": [ + { + "name": "merges_per_sec", + "type": "float", + "description": "The number of write requests merged per second that were queued to the device.\\n" + }, + { + "name": "per_sec", + "type": "float", + "description": "The number of write requests that were issued to the device per second\\n" + } + ] + }, + { + "name": "per_sec", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "float", + "description": "The number of Bytes write from the device per second.\\n", + "format": "bytes" + } + ] + }, + { + "name": "await", + "type": "float", + "description": "The average time spent for write requests issued to the device to be served.\\n" + } + ] + }, + { + "name": "request", + "type": "group", + "fields": [ + { + "name": "avg_size", + "type": "float", + "description": "The average size (in bytes) of the requests that were issued to the device.\\n" + } + ] + }, + { + "name": "queue", + "type": "group", + "fields": [ + { + "name": "avg_size", + "type": "float", + "description": "The average queue length of the requests that were issued to the device.\\n" + } + ] + }, + { + "name": "await", + "type": "float", + "description": "The average time spent for requests issued to the device to be served.\\n" + }, + { + "name": "service_time", + "type": "float", + "description": "The average service time (in milliseconds) for I/O requests that were issued to the device.\\n" + }, + { + "name": "busy", + "type": "float", + "description": "Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%.\\n" + } + ] + } + ] }, { - "name": "response.flags", - "type": "keyword", - "description": "DNS response flags\\n" + "name": "entropy", + "type": "group", + "description": "Available system entropy\\n", + "release": "ga", + "fields": [ + { + "name": "available_bits", + "type": "long", + "description": "The available bits of entropy\\n" + }, + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of available entropy, relative to the pool size of 4096\\n" + } + ] }, { - "name": "response.size", - "type": "integer", - "format": "bytes", - "description": "size of the DNS response\\n" + "name": "filesystem", + "type": "group", + "description": "\`filesystem\` contains local filesystem stats.\\n", + "release": "ga", + "fields": [ + { + "name": "available", + "type": "long", + "format": "bytes", + "description": "The disk space available to an unprivileged user in bytes.\\n" + }, + { + "name": "device_name", + "type": "keyword", + "description": "The disk name. For example: \`/dev/disk1\`\\n" + }, + { + "name": "type", + "type": "keyword", + "description": "The disk type. For example: \`ext4\`\\n" + }, + { + "name": "mount_point", + "type": "keyword", + "description": "The mounting point. For example: \`/\`\\n" + }, + { + "name": "files", + "type": "long", + "description": "The total number of file nodes in the file system.\\n" + }, + { + "name": "free", + "type": "long", + "format": "bytes", + "description": "The disk space available in bytes.\\n" + }, + { + "name": "free_files", + "type": "long", + "description": "The number of free file nodes in the file system.\\n" + }, + { + "name": "total", + "type": "long", + "format": "bytes", + "description": "The total disk space in bytes.\\n" + }, + { + "name": "used", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The used disk space in bytes.\\n" + }, + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of used disk space.\\n" + } + ] + } + ] }, { - "name": "dnssec_ok", - "type": "boolean", - "description": "dnssec flag\\n" + "name": "fsstat", + "type": "group", + "description": "\`system.fsstat\` contains filesystem metrics aggregated from all mounted filesystems.\\n", + "release": "ga", + "fields": [ + { + "name": "count", + "type": "long", + "description": "Number of file systems found." + }, + { + "name": "total_files", + "type": "long", + "description": "Total number of files." + }, + { + "name": "total_size", + "format": "bytes", + "type": "group", + "description": "Nested file system docs.", + "fields": [ + { + "name": "free", + "type": "long", + "format": "bytes", + "description": "Total free space.\\n" + }, + { + "name": "used", + "type": "long", + "format": "bytes", + "description": "Total used space.\\n" + }, + { + "name": "total", + "type": "long", + "format": "bytes", + "description": "Total space (used plus free).\\n" + } + ] + } + ] + }, + { + "name": "load", + "type": "group", + "description": "CPU load averages.\\n", + "release": "ga", + "fields": [ + { + "name": "1", + "type": "scaled_float", + "scaling_factor": 100, + "description": "Load average for the last minute.\\n" + }, + { + "name": "5", + "type": "scaled_float", + "scaling_factor": 100, + "description": "Load average for the last 5 minutes.\\n" + }, + { + "name": "15", + "type": "scaled_float", + "scaling_factor": 100, + "description": "Load average for the last 15 minutes.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "1", + "type": "scaled_float", + "scaling_factor": 100, + "description": "Load for the last minute divided by the number of cores.\\n" + }, + { + "name": "5", + "type": "scaled_float", + "scaling_factor": 100, + "description": "Load for the last 5 minutes divided by the number of cores.\\n" + }, + { + "name": "15", + "type": "scaled_float", + "scaling_factor": 100, + "description": "Load for the last 15 minutes divided by the number of cores.\\n" + } + ] + }, + { + "name": "cores", + "type": "long", + "description": "The number of CPU cores present on the host.\\n" + } + ] + }, + { + "name": "memory", + "type": "group", + "description": "\`memory\` contains local memory stats.\\n", + "release": "ga", + "fields": [ + { + "name": "total", + "type": "long", + "format": "bytes", + "description": "Total memory.\\n" + }, + { + "name": "used", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Used memory.\\n" + }, + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of used memory.\\n" + } + ] + }, + { + "name": "free", + "type": "long", + "format": "bytes", + "description": "The total amount of free memory in bytes. This value does not include memory consumed by system caches and buffers (see system.memory.actual.free).\\n" + }, + { + "name": "actual", + "type": "group", + "description": "Actual memory used and free.\\n", + "fields": [ + { + "name": "used", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Actual used memory in bytes. It represents the difference between the total and the available memory. The available memory depends on the OS. For more details, please check \`system.actual.free\`.\\n" + }, + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of actual used memory.\\n" + } + ] + }, + { + "name": "free", + "type": "long", + "format": "bytes", + "description": "Actual free memory in bytes. It is calculated based on the OS. On Linux it consists of the free memory plus caches and buffers. On OSX it is a sum of free memory and the inactive memory. On Windows, it is equal to \`system.memory.free\`.\\n" + } + ] + }, + { + "name": "swap", + "type": "group", + "prefix": "[float]", + "description": "This group contains statistics related to the swap memory usage on the system.", + "fields": [ + { + "name": "total", + "type": "long", + "format": "bytes", + "description": "Total swap memory.\\n" + }, + { + "name": "used", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Used swap memory.\\n" + }, + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of used swap memory.\\n" + } + ] + }, + { + "name": "free", + "type": "long", + "format": "bytes", + "description": "Available swap memory.\\n" + }, + { + "name": "out", + "type": "group", + "fields": [ + { + "name": "pages", + "type": "long", + "description": "count of pages swapped out" + } + ] + }, + { + "name": "in", + "type": "group", + "fields": [ + { + "name": "pages", + "type": "long", + "description": "count of pages swapped in" + } + ] + }, + { + "name": "readahead", + "type": "group", + "fields": [ + { + "name": "pages", + "type": "long", + "description": "swap readahead pages" + }, + { + "name": "cached", + "type": "long", + "description": "swap readahead cache hits" + } + ] + } + ] + }, + { + "name": "hugepages", + "type": "group", + "prefix": "[float]", + "description": "This group contains statistics related to huge pages usage on the system.", + "fields": [ + { + "name": "total", + "type": "long", + "format": "number", + "description": "Number of huge pages in the pool.\\n" + }, + { + "name": "used", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Memory used in allocated huge pages.\\n" + }, + { + "name": "pct", + "type": "long", + "format": "percent", + "description": "Percentage of huge pages used.\\n" + } + ] + }, + { + "name": "free", + "type": "long", + "format": "number", + "description": "Number of available huge pages in the pool.\\n" + }, + { + "name": "reserved", + "type": "long", + "format": "number", + "description": "Number of reserved but not allocated huge pages in the pool.\\n" + }, + { + "name": "surplus", + "type": "long", + "format": "number", + "description": "Number of overcommited huge pages.\\n" + }, + { + "name": "default_size", + "type": "long", + "format": "bytes", + "description": "Default size for huge pages.\\n" + }, + { + "name": "swap", + "type": "group", + "fields": [ + { + "name": "out", + "type": "group", + "description": "huge pages swapped out", + "fields": [ + { + "name": "pages", + "type": "long", + "description": "pages swapped out" + }, + { + "name": "fallback", + "type": "long", + "description": "Count of huge pages that must be split before swapout" + } + ] + } + ] + } + ] + } + ] + }, + { + "name": "network", + "type": "group", + "description": "\`network\` contains network IO metrics for a single network interface.\\n", + "release": "ga", + "fields": [ + { + "name": "name", + "type": "keyword", + "example": "eth0", + "description": "The network interface name.\\n" + }, + { + "name": "out", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The number of bytes sent.\\n" + }, + { + "name": "packets", + "type": "long", + "description": "The number of packets sent.\\n" + }, + { + "name": "errors", + "type": "long", + "description": "The number of errors while sending.\\n" + }, + { + "name": "dropped", + "type": "long", + "description": "The number of outgoing packets that were dropped. This value is always 0 on Darwin and BSD because it is not reported by the operating system.\\n" + } + ] + }, + { + "name": "in", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The number of bytes received.\\n" + }, + { + "name": "packets", + "type": "long", + "description": "The number or packets received.\\n" + }, + { + "name": "errors", + "type": "long", + "description": "The number of errors while receiving.\\n" + }, + { + "name": "dropped", + "type": "long", + "description": "The number of incoming packets that were dropped.\\n" + } + ] + } + ] + }, + { + "name": "network_summary", + "type": "group", + "release": "beta", + "description": "Metrics relating to global network activity\\n", + "fields": [ + { + "name": "ip", + "type": "group", + "fields": [ + { + "name": "*", + "type": "object", + "description": "IP counters\\n" + } + ] + }, + { + "name": "tcp", + "type": "group", + "fields": [ + { + "name": "*", + "type": "object", + "description": "TCP counters\\n" + } + ] + }, + { + "name": "udp", + "type": "group", + "fields": [ + { + "name": "*", + "type": "object", + "description": "UDP counters\\n" + } + ] + }, + { + "name": "udp_lite", + "type": "group", + "fields": [ + { + "name": "*", + "type": "object", + "description": "UDP Lite counters\\n" + } + ] + }, + { + "name": "icmp", + "type": "group", + "fields": [ + { + "name": "*", + "type": "object", + "description": "ICMP counters\\n" + } + ] + } + ] + }, + { + "name": "process", + "type": "group", + "description": "\`process\` contains process metadata, CPU metrics, and memory metrics.\\n", + "release": "ga", + "fields": [ + { + "name": "state", + "type": "keyword", + "description": "The process state. For example: \\"running\\".\\n" + }, + { + "name": "cmdline", + "type": "keyword", + "description": "The full command-line used to start the process, including the arguments separated by space.\\n", + "ignore_above": 2048 + }, + { + "name": "env", + "type": "object", + "object_type": "keyword", + "description": "The environment variables used to start the process. The data is available on FreeBSD, Linux, and OS X.\\n" + }, + { + "name": "cpu", + "type": "group", + "prefix": "[float]", + "description": "CPU-specific statistics per process.", + "fields": [ + { + "name": "user", + "type": "group", + "fields": [ + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time the process spent in user space.\\n" + } + ] + }, + { + "name": "total", + "type": "group", + "fields": [ + { + "name": "value", + "type": "long", + "description": "The value of CPU usage since starting the process.\\n" + }, + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent by the process since the last update. Its value is similar to the %CPU value of the process displayed by the top command on Unix systems.\\n" + }, + { + "name": "norm", + "type": "group", + "fields": [ + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of CPU time spent by the process since the last event. This value is normalized by the number of CPU cores and it ranges from 0 to 100%.\\n" + } + ] + }, + { + "name": "ticks", + "type": "long", + "description": "The total CPU time spent by the process.\\n" + } + ] + }, + { + "name": "system", + "type": "group", + "fields": [ + { + "name": "ticks", + "type": "long", + "description": "The amount of CPU time the process spent in kernel space.\\n" + } + ] + }, + { + "name": "start_time", + "type": "date", + "description": "The time when the process was started.\\n" + } + ] + }, + { + "name": "memory", + "type": "group", + "description": "Memory-specific statistics per process.", + "prefix": "[float]", + "fields": [ + { + "name": "size", + "type": "long", + "format": "bytes", + "description": "The total virtual memory the process has.\\n" + }, + { + "name": "rss", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The Resident Set Size. The amount of memory the process occupied in main memory (RAM).\\n" + }, + { + "name": "pct", + "type": "scaled_float", + "format": "percent", + "description": "The percentage of memory the process occupied in main memory (RAM).\\n" + } + ] + }, + { + "name": "share", + "type": "long", + "format": "bytes", + "description": "The shared memory the process uses.\\n" + } + ] + }, + { + "name": "fd", + "type": "group", + "description": "File descriptor usage metrics. This set of metrics is available for Linux and FreeBSD.\\n", + "prefix": "[float]", + "fields": [ + { + "name": "open", + "type": "long", + "description": "The number of file descriptors open by the process." + }, + { + "name": "limit", + "type": "group", + "fields": [ + { + "name": "soft", + "type": "long", + "description": "The soft limit on the number of file descriptors opened by the process. The soft limit can be changed by the process at any time.\\n" + }, + { + "name": "hard", + "type": "long", + "description": "The hard limit on the number of file descriptors opened by the process. The hard limit can only be raised by root.\\n" + } + ] + } + ] + }, + { + "name": "cgroup", + "type": "group", + "description": "Metrics and limits from the cgroup of which the task is a member. cgroup metrics are reported when the process has membership in a non-root cgroup. These metrics are only available on Linux.\\n", + "fields": [ + { + "name": "id", + "type": "keyword", + "description": "The ID common to all cgroups associated with this task. If there isn't a common ID used by all cgroups this field will be absent.\\n" + }, + { + "name": "path", + "type": "keyword", + "description": "The path to the cgroup relative to the cgroup subsystem's mountpoint. If there isn't a common path used by all cgroups this field will be absent.\\n" + }, + { + "name": "cpu", + "type": "group", + "description": "The cpu subsystem schedules CPU access for tasks in the cgroup. Access can be controlled by two separate schedulers, CFS and RT. CFS stands for completely fair scheduler which proportionally divides the CPU time between cgroups based on weight. RT stands for real time scheduler which sets a maximum amount of CPU time that processes in the cgroup can consume during a given period.\\n", + "fields": [ + { + "name": "id", + "type": "keyword", + "description": "ID of the cgroup." + }, + { + "name": "path", + "type": "keyword", + "description": "Path to the cgroup relative to the cgroup subsystem's mountpoint.\\n" + }, + { + "name": "cfs", + "type": "group", + "fields": [ + { + "name": "period", + "type": "group", + "fields": [ + { + "name": "us", + "type": "long", + "description": "Period of time in microseconds for how regularly a cgroup's access to CPU resources should be reallocated.\\n" + } + ] + }, + { + "name": "quota", + "type": "group", + "fields": [ + { + "name": "us", + "type": "long", + "description": "Total amount of time in microseconds for which all tasks in a cgroup can run during one period (as defined by cfs.period.us).\\n" + } + ] + }, + { + "name": "shares", + "type": "long", + "description": "An integer value that specifies a relative share of CPU time available to the tasks in a cgroup. The value specified in the cpu.shares file must be 2 or higher.\\n" + } + ] + }, + { + "name": "rt", + "type": "group", + "fields": [ + { + "name": "period", + "type": "group", + "fields": [ + { + "name": "us", + "type": "long", + "description": "Period of time in microseconds for how regularly a cgroup's access to CPU resources is reallocated.\\n" + } + ] + }, + { + "name": "runtime", + "type": "group", + "fields": [ + { + "name": "us", + "type": "long", + "description": "Period of time in microseconds for the longest continuous period in which the tasks in a cgroup have access to CPU resources.\\n" + } + ] + } + ] + }, + { + "name": "stats", + "type": "group", + "fields": [ + { + "name": "periods", + "type": "long", + "description": "Number of period intervals (as specified in cpu.cfs.period.us) that have elapsed.\\n" + }, + { + "name": "throttled", + "type": "group", + "fields": [ + { + "name": "periods", + "type": "long", + "description": "Number of times tasks in a cgroup have been throttled (that is, not allowed to run because they have exhausted all of the available time as specified by their quota).\\n" + }, + { + "name": "ns", + "type": "long", + "description": "The total time duration (in nanoseconds) for which tasks in a cgroup have been throttled.\\n" + } + ] + } + ] + } + ] + }, + { + "name": "cpuacct", + "type": "group", + "description": "CPU accounting metrics.", + "fields": [ + { + "name": "id", + "type": "keyword", + "description": "ID of the cgroup." + }, + { + "name": "path", + "type": "keyword", + "description": "Path to the cgroup relative to the cgroup subsystem's mountpoint.\\n" + }, + { + "name": "total", + "type": "group", + "fields": [ + { + "name": "ns", + "type": "long", + "description": "Total CPU time in nanoseconds consumed by all tasks in the cgroup.\\n" + } + ] + }, + { + "name": "stats", + "type": "group", + "fields": [ + { + "name": "user", + "type": "group", + "fields": [ + { + "name": "ns", + "type": "long", + "description": "CPU time consumed by tasks in user mode." + } + ] + }, + { + "name": "system", + "type": "group", + "fields": [ + { + "name": "ns", + "type": "long", + "description": "CPU time consumed by tasks in user (kernel) mode." + } + ] + } + ] + }, + { + "name": "percpu", + "type": "object", + "object_type": "long", + "description": "CPU time (in nanoseconds) consumed on each CPU by all tasks in this cgroup.\\n" + } + ] + }, + { + "name": "memory", + "type": "group", + "description": "Memory limits and metrics.", + "fields": [ + { + "name": "id", + "type": "keyword", + "description": "ID of the cgroup." + }, + { + "name": "path", + "type": "keyword", + "description": "Path to the cgroup relative to the cgroup subsystem's mountpoint.\\n" + }, + { + "name": "mem", + "type": "group", + "fields": [ + { + "name": "usage", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Total memory usage by processes in the cgroup (in bytes).\\n" + }, + { + "name": "max", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The maximum memory used by processes in the cgroup (in bytes).\\n" + } + ] + } + ] + }, + { + "name": "limit", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The maximum amount of user memory in bytes (including file cache) that tasks in the cgroup are allowed to use.\\n" + } + ] + }, + { + "name": "failures", + "type": "long", + "description": "The number of times that the memory limit (mem.limit.bytes) was reached.\\n" + } + ] + }, + { + "name": "memsw", + "type": "group", + "fields": [ + { + "name": "usage", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The sum of current memory usage plus swap space used by processes in the cgroup (in bytes).\\n" + }, + { + "name": "max", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The maximum amount of memory and swap space used by processes in the cgroup (in bytes).\\n" + } + ] + } + ] + }, + { + "name": "limit", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The maximum amount for the sum of memory and swap usage that tasks in the cgroup are allowed to use.\\n" + } + ] + }, + { + "name": "failures", + "type": "long", + "description": "The number of times that the memory plus swap space limit (memsw.limit.bytes) was reached.\\n" + } + ] + }, + { + "name": "kmem", + "type": "group", + "fields": [ + { + "name": "usage", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Total kernel memory usage by processes in the cgroup (in bytes).\\n" + }, + { + "name": "max", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The maximum kernel memory used by processes in the cgroup (in bytes).\\n" + } + ] + } + ] + }, + { + "name": "limit", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The maximum amount of kernel memory that tasks in the cgroup are allowed to use.\\n" + } + ] + }, + { + "name": "failures", + "type": "long", + "description": "The number of times that the memory limit (kmem.limit.bytes) was reached.\\n" + } + ] + }, + { + "name": "kmem_tcp", + "type": "group", + "fields": [ + { + "name": "usage", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Total memory usage for TCP buffers in bytes.\\n" + }, + { + "name": "max", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The maximum memory used for TCP buffers by processes in the cgroup (in bytes).\\n" + } + ] + } + ] + }, + { + "name": "limit", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "The maximum amount of memory for TCP buffers that tasks in the cgroup are allowed to use.\\n" + } + ] + }, + { + "name": "failures", + "type": "long", + "description": "The number of times that the memory limit (kmem_tcp.limit.bytes) was reached.\\n" + } + ] + }, + { + "name": "stats", + "type": "group", + "fields": [ + { + "name": "active_anon", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs (shmem), in bytes.\\n" + } + ] + }, + { + "name": "active_file", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "File-backed memory on active LRU list, in bytes." + } + ] + }, + { + "name": "cache", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Page cache, including tmpfs (shmem), in bytes." + } + ] + }, + { + "name": "hierarchical_memory_limit", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Memory limit for the hierarchy that contains the memory cgroup, in bytes.\\n" + } + ] + }, + { + "name": "hierarchical_memsw_limit", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Memory plus swap limit for the hierarchy that contains the memory cgroup, in bytes.\\n" + } + ] + }, + { + "name": "inactive_anon", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Anonymous and swap cache on inactive LRU list, including tmpfs (shmem), in bytes\\n" + } + ] + }, + { + "name": "inactive_file", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "File-backed memory on inactive LRU list, in bytes.\\n" + } + ] + }, + { + "name": "mapped_file", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Size of memory-mapped mapped files, including tmpfs (shmem), in bytes.\\n" + } + ] + }, + { + "name": "page_faults", + "type": "long", + "description": "Number of times that a process in the cgroup triggered a page fault.\\n" + }, + { + "name": "major_page_faults", + "type": "long", + "description": "Number of times that a process in the cgroup triggered a major fault. \\"Major\\" faults happen when the kernel actually has to read the data from disk.\\n" + }, + { + "name": "pages_in", + "type": "long", + "description": "Number of pages paged into memory. This is a counter.\\n" + }, + { + "name": "pages_out", + "type": "long", + "description": "Number of pages paged out of memory. This is a counter.\\n" + }, + { + "name": "rss", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Anonymous and swap cache (includes transparent hugepages), not including tmpfs (shmem), in bytes.\\n" + } + ] + }, + { + "name": "rss_huge", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Number of bytes of anonymous transparent hugepages.\\n" + } + ] + }, + { + "name": "swap", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Swap usage, in bytes.\\n" + } + ] + }, + { + "name": "unevictable", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Memory that cannot be reclaimed, in bytes.\\n" + } + ] + } + ] + } + ] + }, + { + "name": "blkio", + "type": "group", + "description": "Block IO metrics.", + "fields": [ + { + "name": "id", + "type": "keyword", + "description": "ID of the cgroup." + }, + { + "name": "path", + "type": "keyword", + "description": "Path to the cgroup relative to the cgroup subsystems mountpoint.\\n" + }, + { + "name": "total", + "type": "group", + "fields": [ + { + "name": "bytes", + "type": "long", + "format": "bytes", + "description": "Total number of bytes transferred to and from all block devices by processes in the cgroup.\\n" + }, + { + "name": "ios", + "type": "long", + "description": "Total number of I/O operations performed on all devices by processes in the cgroup as seen by the throttling policy.\\n" + } + ] + } + ] + } + ] + }, + { + "name": "summary", + "title": "Process Summary", + "type": "group", + "description": "Summary metrics for the processes running on the host.\\n", + "release": "ga", + "fields": [ + { + "name": "total", + "type": "long", + "description": "Total number of processes on this host.\\n" + }, + { + "name": "running", + "type": "long", + "description": "Number of running processes on this host.\\n" + }, + { + "name": "idle", + "type": "long", + "description": "Number of idle processes on this host.\\n" + }, + { + "name": "sleeping", + "type": "long", + "description": "Number of sleeping processes on this host.\\n" + }, + { + "name": "stopped", + "type": "long", + "description": "Number of stopped processes on this host.\\n" + }, + { + "name": "zombie", + "type": "long", + "description": "Number of zombie processes on this host.\\n" + }, + { + "name": "dead", + "type": "long", + "description": "Number of dead processes on this host. It's very unlikely that it will appear but in some special situations it may happen.\\n" + }, + { + "name": "unknown", + "type": "long", + "description": "Number of processes for which the state couldn't be retrieved or is unknown.\\n" + } + ] + } + ] + }, + { + "name": "raid", + "type": "group", + "description": "raid\\n", + "release": "ga", + "fields": [ + { + "name": "name", + "type": "keyword", + "description": "Name of the device.\\n" + }, + { + "name": "status", + "type": "keyword", + "description": "activity-state of the device.\\n" + }, + { + "name": "level", + "type": "keyword", + "description": "The raid level of the device\\n" + }, + { + "name": "sync_action", + "type": "keyword", + "description": "Current sync action, if the RAID array is redundant\\n" + }, + { + "name": "disks", + "type": "group", + "fields": [ + { + "name": "active", + "type": "long", + "description": "Number of active disks.\\n" + }, + { + "name": "total", + "type": "long", + "description": "Total number of disks the device consists of.\\n" + }, + { + "name": "spare", + "type": "long", + "description": "Number of spared disks.\\n" + }, + { + "name": "failed", + "type": "long", + "description": "Number of failed disks.\\n" + }, + { + "name": "states", + "type": "group", + "fields": [ + { + "name": "*", + "type": "object", + "object_type": "keyword", + "description": "map of raw disk states\\n" + } + ] + } + ] + }, + { + "name": "blocks", + "type": "group", + "fields": [ + { + "name": "total", + "type": "long", + "description": "Number of blocks the device holds, in 1024-byte blocks.\\n" + }, + { + "name": "synced", + "type": "long", + "description": "Number of blocks on the device that are in sync, in 1024-byte blocks.\\n" + } + ] + } + ] + }, + { + "name": "socket", + "type": "group", + "description": "TCP sockets that are active.\\n", + "release": "ga", + "fields": [ + { + "name": "local", + "type": "group", + "fields": [ + { + "name": "ip", + "type": "ip", + "example": "192.0.2.1 or 2001:0DB8:ABED:8536::1", + "description": "Local IP address. This can be an IPv4 or IPv6 address.\\n" + }, + { + "name": "port", + "type": "long", + "example": 22, + "description": "Local port.\\n" + } + ] + }, + { + "name": "remote", + "type": "group", + "fields": [ + { + "name": "ip", + "type": "ip", + "example": "192.0.2.1 or 2001:0DB8:ABED:8536::1", + "description": "Remote IP address. This can be an IPv4 or IPv6 address.\\n" + }, + { + "name": "port", + "type": "long", + "example": 22, + "description": "Remote port.\\n" + }, + { + "name": "host", + "type": "keyword", + "example": "76-211-117-36.nw.example.com.", + "description": "PTR record associated with the remote IP. It is obtained via reverse IP lookup.\\n" + }, + { + "name": "etld_plus_one", + "type": "keyword", + "example": "example.com.", + "description": "The effective top-level domain (eTLD) of the remote host plus one more label. For example, the eTLD+1 for \\"foo.bar.golang.org.\\" is \\"golang.org.\\". The data for determining the eTLD comes from an embedded copy of the data from http://publicsuffix.org.\\n" + }, + { + "name": "host_error", + "type": "keyword", + "description": "Error describing the cause of the reverse lookup failure.\\n" + } + ] + }, + { + "name": "process", + "type": "group", + "fields": [ + { + "name": "cmdline", + "type": "keyword", + "description": "Full command line\\n" + } + ] + }, + { + "name": "user", + "type": "group", + "fields": [] + }, + { + "name": "summary", + "title": "Socket summary", + "type": "group", + "description": "Summary metrics of open sockets in the host system\\n", + "release": "ga", + "fields": [ + { + "name": "all", + "type": "group", + "description": "All connections\\n", + "fields": [ + { + "name": "count", + "type": "integer", + "description": "All open connections\\n" + }, + { + "name": "listening", + "type": "integer", + "description": "All listening ports\\n" + } + ] + }, + { + "name": "tcp", + "type": "group", + "description": "All TCP connections\\n", + "fields": [ + { + "name": "memory", + "type": "integer", + "format": "bytes", + "description": "Memory used by TCP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/tcp_mem. Only available on Linux.\\n" + }, + { + "name": "all", + "type": "group", + "description": "All TCP connections\\n", + "fields": [ + { + "name": "orphan", + "type": "integer", + "description": "A count of all orphaned tcp sockets. Only available on Linux.\\n" + }, + { + "name": "count", + "type": "integer", + "description": "All open TCP connections\\n" + }, + { + "name": "listening", + "type": "integer", + "description": "All TCP listening ports\\n" + }, + { + "name": "established", + "type": "integer", + "description": "Number of established TCP connections\\n" + }, + { + "name": "close_wait", + "type": "integer", + "description": "Number of TCP connections in _close_wait_ state\\n" + }, + { + "name": "time_wait", + "type": "integer", + "description": "Number of TCP connections in _time_wait_ state\\n" + }, + { + "name": "syn_sent", + "type": "integer", + "description": "Number of TCP connections in _syn_sent_ state\\n" + }, + { + "name": "syn_recv", + "type": "integer", + "description": "Number of TCP connections in _syn_recv_ state\\n" + }, + { + "name": "fin_wait1", + "type": "integer", + "description": "Number of TCP connections in _fin_wait1_ state\\n" + }, + { + "name": "fin_wait2", + "type": "integer", + "description": "Number of TCP connections in _fin_wait2_ state\\n" + }, + { + "name": "last_ack", + "type": "integer", + "description": "Number of TCP connections in _last_ack_ state\\n" + }, + { + "name": "closing", + "type": "integer", + "description": "Number of TCP connections in _closing_ state\\n" + } + ] + } + ] + }, + { + "name": "udp", + "type": "group", + "description": "All UDP connections\\n", + "fields": [ + { + "name": "memory", + "type": "integer", + "format": "bytes", + "description": "Memory used by UDP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/udp_mem. Only available on Linux.\\n" + }, + { + "name": "all", + "type": "group", + "description": "All UDP connections\\n", + "fields": [ + { + "name": "count", + "type": "integer", + "description": "All open UDP connections\\n" + } + ] + } + ] + } + ] + } + ] + }, + { + "name": "uptime", + "type": "group", + "description": "\`uptime\` contains the operating system uptime metric.\\n", + "release": "ga", + "fields": [ + { + "name": "duration", + "type": "group", + "fields": [ + { + "name": "ms", + "type": "long", + "format": "duration", + "input_format": "milliseconds", + "description": "The OS uptime in milliseconds.\\n" + } + ] + } + ] + }, + { + "name": "users", + "type": "group", + "release": "beta", + "description": "Logged-in user session data\\n", + "fields": [ + { + "name": "id", + "type": "keyword", + "description": "The ID of the session\\n" + }, + { + "name": "seat", + "type": "keyword", + "description": "An associated logind seat\\n" + }, + { + "name": "path", + "type": "keyword", + "description": "The DBus object path of the session\\n" + }, + { + "name": "type", + "type": "keyword", + "description": "The type of the user session\\n" + }, + { + "name": "service", + "type": "keyword", + "description": "A session associated with the service\\n" + }, + { + "name": "remote", + "type": "boolean", + "description": "A bool indicating a remote session\\n" + }, + { + "name": "state", + "type": "keyword", + "description": "The current state of the session\\n" + }, + { + "name": "scope", + "type": "keyword", + "description": "The associated systemd scope\\n" + }, + { + "name": "leader", + "type": "long", + "description": "The root PID of the session\\n" + }, + { + "name": "remote_host", + "type": "keyword", + "description": "A remote host address for the session\\n" + } + ] } ] } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts index 3cdf011d9d0e3..929f2518ee748 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts @@ -8,7 +8,7 @@ import { readFileSync } from 'fs'; import glob from 'glob'; import { safeLoad } from 'js-yaml'; import path from 'path'; -import { Field, processFields } from './field'; +import { Field, Fields, getField, processFields } from './field'; // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ @@ -27,9 +27,56 @@ test('tests loading fields.yml', () => { for (const file of files) { const fieldsYML = readFileSync(file, 'utf-8'); const fields: Field[] = safeLoad(fieldsYML); - processFields(fields); + const processedFields = processFields(fields); // Check that content file and generated file are equal - expect(fields).toMatchSnapshot(path.basename(file)); + expect(processedFields).toMatchSnapshot(path.basename(file)); } }); + +describe('getField searches recursively for nested field in fields given an array of path parts', () => { + const searchFields: Fields = [ + { + name: '1', + fields: [ + { + name: '1-1', + }, + { + name: '1-2', + }, + ], + }, + { + name: '2', + fields: [ + { + name: '2-1', + }, + { + name: '2-2', + fields: [ + { + name: '2-2-1', + }, + { + name: '2-2-2', + }, + ], + }, + ], + }, + ]; + test('returns undefined when the field does not exist', () => { + expect(getField(searchFields, ['0'])).toBe(undefined); + }); + test('returns undefined if the field is not a leaf node', () => { + expect(getField(searchFields, ['1'])?.name).toBe(undefined); + }); + test('returns undefined searching for a nested field that does not exist', () => { + expect(getField(searchFields, ['1', '1-3'])?.name).toBe(undefined); + }); + test('returns nested field that is a leaf node', () => { + expect(getField(searchFields, ['2', '2-2', '2-2-1'])?.name).toBe('2-2-1'); + }); +}); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts index eb515f5652f36..4a1a84baf6599 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts @@ -21,6 +21,12 @@ export interface Field { required?: boolean; multi_fields?: Fields; doc_values?: boolean; + copy_to?: string; + analyzer?: string; + search_analyzer?: string; + ignore_above?: number; + object_type?: string; + scaling_factor?: number; // Kibana specific analyzed?: boolean; @@ -43,44 +49,140 @@ export interface Field { export type Fields = Field[]; /** - * ProcessFields takes the given fields read from yaml and expands it. + * expandFields takes the given fields read from yaml and expands them. * There are dotted fields in the field.yml like `foo.bar`. These should - * be stored as an object inside an object and is the main purpose of this - * preprocessing. + * be stored as an field within a 'group' field. * - * Note: This function modifies the passed field param. + * Note: This function modifies the passed fields array. */ -export function processFields(fields: Fields) { +export function expandFields(fields: Fields) { fields.forEach((field, key) => { const fieldName = field.name; - // If the field name contains a dot, it means we need to create sub objects + // If the field name contains a dot, it means we need to + // - take the first part of the name + // - create a field of type 'group' with this first part + // - put the original field, named with the rest of the original name in the fields property of the new group field if (fieldName.includes('.')) { // Split up the name by dots to extract first and other parts const nameParts = fieldName.split('.'); // Getting first part of the name for the new field - const newNameTop = nameParts[0]; - delete nameParts[0]; + const groupFieldName = nameParts[0]; // Put back together the parts again for the new field name - const newName = nameParts.length === 1 ? nameParts[0] : nameParts.slice(1).join('.'); + const restFieldName = nameParts.slice(1).join('.'); - field.name = newName; + // keep all properties of the original field, but give it the shortened name + field.name = restFieldName; - // Create the new field with the old field inside - const newField: Field = { - name: newNameTop, + // create a new field of type group with the original field in the fields array + const groupField: Field = { + name: groupFieldName, type: 'group', fields: [field], }; - // Replace the old field in the array - fields[key] = newField; - if (newField.fields) { - processFields(newField.fields); + // check child fields further down the tree + if (groupField.fields) { + expandFields(groupField.fields); } + // Replace the original field in the array with the new one + fields[key] = groupField; + } else { + // even if this field doesn't have dots to expand, its child fields further down the tree might + if (field.fields) { + expandFields(field.fields); + } + } + }); +} +/** + * dedupFields takes the given fields and merges sibling fields with the + * same name together. + * These can result from expandFields when the input contains dotted field + * names that share parts of their hierarchy. + */ +function dedupFields(fields: Fields): Fields { + const dedupedFields: Fields = []; + fields.forEach(field => { + const found = dedupedFields.find(f => { + return f.name === field.name; + }); + if (found) { + if (found.type === 'group' && field.type === 'group' && found.fields && field.fields) { + found.fields = dedupFields(found.fields.concat(field.fields)); + } else { + // only 'group' fields can be merged in this way + // XXX: don't abort on error for now + // see discussion in https://github.com/elastic/kibana/pull/59894 + // throw new Error( + // "Can't merge fields " + JSON.stringify(found) + ' and ' + JSON.stringify(field) + // ); + } + } else { + if (field.fields) { + field.fields = dedupFields(field.fields); + } + dedupedFields.push(field); } }); + return dedupedFields; +} + +/** validateFields takes the given fields and verifies: + * + * - all fields of type alias point to existing fields. + * - all fields of type array have a property object_type + * + * Invalid fields are silently removed. + */ + +function validateFields(fields: Fields, allFields: Fields): Fields { + const validatedFields: Fields = []; + + fields.forEach(field => { + if (field.type === 'alias') { + if (field.path && getField(allFields, field.path.split('.'))) { + validatedFields.push(field); + } + } else if (field.type === 'array') { + if (field.object_type) { + validatedFields.push(field); + } + } else { + validatedFields.push(field); + } + if (field.fields) { + field.fields = validateFields(field.fields, allFields); + } + }); + return validatedFields; +} + +export const getField = (fields: Fields, pathNames: string[]): Field | undefined => { + if (!pathNames.length) return undefined; + // get the first rest of path names + const [name, ...restPathNames] = pathNames; + for (const field of fields) { + if (field.name === name) { + // check field's fields, passing in the remaining path names + if (field.fields && field.fields.length > 0) { + return getField(field.fields, restPathNames); + } + // no nested fields to search, but still more names - not found + if (restPathNames.length) { + return undefined; + } + return field; + } + } + return undefined; +}; + +export function processFields(fields: Fields): Fields { + expandFields(fields); + const dedupedFields = dedupFields(fields); + return validateFields(dedupedFields, dedupedFields); } const isFields = (path: string) => { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/tests/base.yml b/x-pack/plugins/ingest_manager/server/services/epm/fields/tests/base.yml index 86b61245aa3b8..5a71c7dee54dc 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/fields/tests/base.yml +++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/tests/base.yml @@ -4,4 +4,20 @@ - name: auid - name: euid - name: long.nested.foo + type: text +- name: long.nested.bar + type: integer - name: nested.bar +- name: nested.baz +- name: myalias + type: alias + path: user.euid +- name: invalidalias + type: alias + path: euid +- name: validarray + type: array + object_type: integer +- name: invalidarray + type: array + diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/tests/system.yml b/x-pack/plugins/ingest_manager/server/services/epm/fields/tests/system.yml new file mode 100644 index 0000000000000..609914616a683 --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/tests/system.yml @@ -0,0 +1,1625 @@ +- name: system.core + type: group + description: > + `system-core` contains CPU metrics for a single core of a multi-core system. + fields: + - name: id + type: long + description: > + CPU Core number. + + # Percentages + - name: user.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. + + - name: user.ticks + type: long + description: > + The amount of CPU time spent in user space. + + - name: system.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: system.ticks + type: long + description: > + The amount of CPU time spent in kernel space. + + - name: nice.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: nice.ticks + type: long + description: > + The amount of CPU time spent on low-priority processes. + + - name: idle.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: idle.ticks + type: long + description: > + The amount of CPU time spent idle. + + - name: iowait.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: iowait.ticks + type: long + description: > + The amount of CPU time spent in wait (on disk). + + - name: irq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: irq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: softirq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling software interrupts. + + - name: steal.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: steal.ticks + type: long + description: > + The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. +- name: system.cpu + type: group + description: > + `cpu` contains local CPU stats. + release: ga + fields: + - name: cores + type: long + description: > + The number of CPU cores present on the host. The non-normalized + percentages will have a maximum value of `100% * cores`. The + normalized percentages already take this value into account and have + a maximum value of 100%. + + # Percentages + - name: user.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. On multi-core systems, + you can have percentages that are greater than 100%. For example, if 3 + cores are at 60% use, then the `system.cpu.user.pct` will be 180%. + + - name: system.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: nice.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: idle.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: iowait.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: irq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: steal.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: total.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in states other than Idle and IOWait. + + # Normalized Percentages + - name: user.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. + + - name: system.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: nice.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: idle.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: iowait.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: irq.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: steal.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: total.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time in states other than Idle and IOWait, normalised by the number of cores. + + + # Ticks + - name: user.ticks + type: long + description: > + The amount of CPU time spent in user space. + + - name: system.ticks + type: long + description: > + The amount of CPU time spent in kernel space. + + - name: nice.ticks + type: long + description: > + The amount of CPU time spent on low-priority processes. + + - name: idle.ticks + type: long + description: > + The amount of CPU time spent idle. + + - name: iowait.ticks + type: long + description: > + The amount of CPU time spent in wait (on disk). + + - name: irq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling software interrupts. + + - name: steal.ticks + type: long + description: > + The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. +- name: system.diskio + type: group + description: > + `disk` contains disk IO metrics collected from the operating system. + release: ga + fields: + - name: name + type: keyword + example: sda1 + description: > + The disk name. + + - name: serial_number + type: keyword + description: > + The disk's serial number. This may not be provided by all operating + systems. + + - name: read.count + type: long + description: > + The total number of reads completed successfully. + + - name: write.count + type: long + description: > + The total number of writes completed successfully. + + - name: read.bytes + type: long + format: bytes + description: > + The total number of bytes read successfully. On Linux this is + the number of sectors read multiplied by an assumed sector size of 512. + + - name: write.bytes + type: long + format: bytes + description: > + The total number of bytes written successfully. On Linux this is + the number of sectors written multiplied by an assumed sector size of + 512. + + - name: read.time + type: long + description: > + The total number of milliseconds spent by all reads. + + - name: write.time + type: long + description: > + The total number of milliseconds spent by all writes. + + - name: io.time + type: long + description: > + The total number of of milliseconds spent doing I/Os. + + - name: iostat.read.request.merges_per_sec + type: float + description: > + The number of read requests merged per second that were queued to the device. + + - name: iostat.write.request.merges_per_sec + type: float + description: > + The number of write requests merged per second that were queued to the device. + + - name: iostat.read.request.per_sec + type: float + description: > + The number of read requests that were issued to the device per second + + - name: iostat.write.request.per_sec + type: float + description: > + The number of write requests that were issued to the device per second + + - name: iostat.read.per_sec.bytes + type: float + description: > + The number of Bytes read from the device per second. + format: bytes + + - name: iostat.read.await + type: float + description: > + The average time spent for read requests issued to the device to be served. + + - name: iostat.write.per_sec.bytes + type: float + description: > + The number of Bytes write from the device per second. + format: bytes + + - name: iostat.write.await + type: float + description: > + The average time spent for write requests issued to the device to be served. + + - name: iostat.request.avg_size + type: float + description: > + The average size (in bytes) of the requests that were issued to the device. + + - name: iostat.queue.avg_size + type: float + description: > + The average queue length of the requests that were issued to the device. + + - name: iostat.await + type: float + description: > + The average time spent for requests issued to the device to be served. + + - name: iostat.service_time + type: float + description: > + The average service time (in milliseconds) for I/O requests that were issued to the device. + + - name: iostat.busy + type: float + description: > + Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. +- name: system.entropy + type: group + description: > + Available system entropy + release: ga + fields: + - name: available_bits + type: long + description: > + The available bits of entropy + - name: pct + type: scaled_float + format: percent + description: > + The percentage of available entropy, relative to the pool size of 4096 +- name: system.filesystem + type: group + description: > + `filesystem` contains local filesystem stats. + release: ga + fields: + - name: available + type: long + format: bytes + description: > + The disk space available to an unprivileged user in bytes. + - name: device_name + type: keyword + description: > + The disk name. For example: `/dev/disk1` + - name: type + type: keyword + description: > + The disk type. For example: `ext4` + - name: mount_point + type: keyword + description: > + The mounting point. For example: `/` + - name: files + type: long + description: > + The total number of file nodes in the file system. + - name: free + type: long + format: bytes + description: > + The disk space available in bytes. + - name: free_files + type: long + description: > + The number of free file nodes in the file system. + - name: total + type: long + format: bytes + description: > + The total disk space in bytes. + - name: used.bytes + type: long + format: bytes + description: > + The used disk space in bytes. + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used disk space. +- name: system.fsstat + type: group + description: > + `system.fsstat` contains filesystem metrics aggregated from all mounted + filesystems. + release: ga + fields: + - name: count + type: long + description: Number of file systems found. + - name: total_files + type: long + description: Total number of files. + - name: total_size + format: bytes + type: group + description: Nested file system docs. + fields: + - name: free + type: long + format: bytes + description: > + Total free space. + - name: used + type: long + format: bytes + description: > + Total used space. + - name: total + type: long + format: bytes + description: > + Total space (used plus free). +- name: system.load + type: group + description: > + CPU load averages. + release: ga + fields: + - name: "1" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last minute. + - name: "5" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 5 minutes. + - name: "15" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 15 minutes. + + - name: "norm.1" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last minute divided by the number of cores. + + - name: "norm.5" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last 5 minutes divided by the number of cores. + + - name: "norm.15" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last 15 minutes divided by the number of cores. + + - name: "cores" + type: long + description: > + The number of CPU cores present on the host. +- name: system.memory + type: group + description: > + `memory` contains local memory stats. + release: ga + fields: + - name: total + type: long + format: bytes + description: > + Total memory. + + - name: used.bytes + type: long + format: bytes + description: > + Used memory. + + - name: free + type: long + format: bytes + description: > + The total amount of free memory in bytes. This value does not include memory consumed by system caches and + buffers (see system.memory.actual.free). + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used memory. + + - name: actual + type: group + description: > + Actual memory used and free. + fields: + + - name: used.bytes + type: long + format: bytes + description: > + Actual used memory in bytes. It represents the difference between the total and the available memory. The + available memory depends on the OS. For more details, please check `system.actual.free`. + + - name: free + type: long + format: bytes + description: > + Actual free memory in bytes. It is calculated based on the OS. On Linux it consists of the free memory + plus caches and buffers. On OSX it is a sum of free memory and the inactive memory. On Windows, it is equal + to `system.memory.free`. + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of actual used memory. + + - name: swap + type: group + prefix: "[float]" + description: This group contains statistics related to the swap memory usage on the system. + fields: + - name: total + type: long + format: bytes + description: > + Total swap memory. + + - name: used.bytes + type: long + format: bytes + description: > + Used swap memory. + + - name: free + type: long + format: bytes + description: > + Available swap memory. + + - name: out.pages + type: long + description: count of pages swapped out + + - name: in.pages + type: long + description: count of pages swapped in + + - name: readahead.pages + type: long + description: swap readahead pages + + - name: readahead.cached + type: long + description: swap readahead cache hits + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used swap memory. + + - name: hugepages + type: group + prefix: "[float]" + description: This group contains statistics related to huge pages usage on the system. + fields: + - name: total + type: long + format: number + description: > + Number of huge pages in the pool. + + - name: used.bytes + type: long + format: bytes + description: > + Memory used in allocated huge pages. + + - name: used.pct + type: long + format: percent + description: > + Percentage of huge pages used. + + - name: free + type: long + format: number + description: > + Number of available huge pages in the pool. + + - name: reserved + type: long + format: number + description: > + Number of reserved but not allocated huge pages in the pool. + + - name: surplus + type: long + format: number + description: > + Number of overcommited huge pages. + + - name: default_size + type: long + format: bytes + description: > + Default size for huge pages. + + - name: swap.out + type: group + description: huge pages swapped out + fields: + - name: pages + type: long + description: pages swapped out + - name: fallback + type: long + description: Count of huge pages that must be split before swapout +- name: system.network + type: group + description: > + `network` contains network IO metrics for a single network interface. + release: ga + fields: + - name: name + type: keyword + example: eth0 + description: > + The network interface name. + + - name: out.bytes + type: long + format: bytes + description: > + The number of bytes sent. + + - name: in.bytes + type: long + format: bytes + description: > + The number of bytes received. + + - name: out.packets + type: long + description: > + The number of packets sent. + + - name: in.packets + type: long + description: > + The number or packets received. + + - name: in.errors + type: long + description: > + The number of errors while receiving. + + - name: out.errors + type: long + description: > + The number of errors while sending. + + - name: in.dropped + type: long + description: > + The number of incoming packets that were dropped. + + - name: out.dropped + type: long + description: > + The number of outgoing packets that were dropped. This value is always + 0 on Darwin and BSD because it is not reported by the operating system. +- name: system.network_summary + type: group + release: beta + description: > + Metrics relating to global network activity + fields: + - name: ip.* + type: object + description: > + IP counters + - name: tcp.* + type: object + description: > + TCP counters + - name: udp.* + type: object + description: > + UDP counters + - name: udp_lite.* + type: object + description: > + UDP Lite counters + - name: icmp.* + type: object + description: > + ICMP counters +- name: system.process + type: group + description: > + `process` contains process metadata, CPU metrics, and memory metrics. + release: ga + fields: + - name: name + type: alias + path: process.name + migration: true + - name: state + type: keyword + description: > + The process state. For example: "running". + - name: pid + type: alias + path: process.pid + migration: true + - name: ppid + type: alias + path: process.ppid + migration: true + - name: pgid + type: alias + path: process.pgid + migration: true + - name: cmdline + type: keyword + description: > + The full command-line used to start the process, including the + arguments separated by space. + ignore_above: 2048 + - name: username + type: alias + path: user.name + migration: true + - name: cwd + type: alias + path: process.working_directory + migration: true + - name: env + type: object + object_type: keyword + description: > + The environment variables used to start the process. The data is + available on FreeBSD, Linux, and OS X. + - name: cpu + type: group + prefix: "[float]" + description: CPU-specific statistics per process. + fields: + - name: user.ticks + type: long + description: > + The amount of CPU time the process spent in user space. + - name: total.value + type: long + description: > + The value of CPU usage since starting the process. + - name: total.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent by the process since the last update. Its value is similar to the + %CPU value of the process displayed by the top command on Unix systems. + - name: total.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent by the process since the last event. + This value is normalized by the number of CPU cores and it ranges + from 0 to 100%. + - name: system.ticks + type: long + description: > + The amount of CPU time the process spent in kernel space. + - name: total.ticks + type: long + description: > + The total CPU time spent by the process. + - name: start_time + type: date + description: > + The time when the process was started. + - name: memory + type: group + description: Memory-specific statistics per process. + prefix: "[float]" + fields: + - name: size + type: long + format: bytes + description: > + The total virtual memory the process has. + - name: rss.bytes + type: long + format: bytes + description: > + The Resident Set Size. The amount of memory the process occupied in main memory (RAM). + - name: rss.pct + type: scaled_float + format: percent + description: > + The percentage of memory the process occupied in main memory (RAM). + - name: share + type: long + format: bytes + description: > + The shared memory the process uses. + - name: fd + type: group + description: > + File descriptor usage metrics. This set of metrics is available for + Linux and FreeBSD. + prefix: "[float]" + fields: + - name: open + type: long + description: The number of file descriptors open by the process. + - name: limit.soft + type: long + description: > + The soft limit on the number of file descriptors opened by the + process. The soft limit can be changed by the process at any time. + - name: limit.hard + type: long + description: > + The hard limit on the number of file descriptors opened by the + process. The hard limit can only be raised by root. + - name: cgroup + type: group + description: > + Metrics and limits from the cgroup of which the task is a member. + cgroup metrics are reported when the process has membership in a + non-root cgroup. These metrics are only available on Linux. + fields: + - name: id + type: keyword + description: > + The ID common to all cgroups associated with this task. + If there isn't a common ID used by all cgroups this field will be + absent. + + - name: path + type: keyword + description: > + The path to the cgroup relative to the cgroup subsystem's mountpoint. + If there isn't a common path used by all cgroups this field will be + absent. + + - name: cpu + type: group + description: > + The cpu subsystem schedules CPU access for tasks in the cgroup. + Access can be controlled by two separate schedulers, CFS and RT. + CFS stands for completely fair scheduler which proportionally + divides the CPU time between cgroups based on weight. RT stands for + real time scheduler which sets a maximum amount of CPU time that + processes in the cgroup can consume during a given period. + + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's + mountpoint. + + - name: cfs.period.us + type: long + description: > + Period of time in microseconds for how regularly a + cgroup's access to CPU resources should be reallocated. + + - name: cfs.quota.us + type: long + description: > + Total amount of time in microseconds for which all + tasks in a cgroup can run during one period (as defined by + cfs.period.us). + + - name: cfs.shares + type: long + description: > + An integer value that specifies a relative share of CPU time + available to the tasks in a cgroup. The value specified in the + cpu.shares file must be 2 or higher. + + - name: rt.period.us + type: long + description: > + Period of time in microseconds for how regularly a cgroup's + access to CPU resources is reallocated. + + - name: rt.runtime.us + type: long + description: > + Period of time in microseconds for the longest continuous period + in which the tasks in a cgroup have access to CPU resources. + + - name: stats.periods + type: long + description: > + Number of period intervals (as specified in cpu.cfs.period.us) + that have elapsed. + + - name: stats.throttled.periods + type: long + description: > + Number of times tasks in a cgroup have been throttled (that is, + not allowed to run because they have exhausted all of the + available time as specified by their quota). + + - name: stats.throttled.ns + type: long + description: > + The total time duration (in nanoseconds) for which tasks in a + cgroup have been throttled. + + - name: cpuacct + type: group + description: CPU accounting metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's + mountpoint. + + - name: total.ns + type: long + description: > + Total CPU time in nanoseconds consumed by all tasks in the + cgroup. + + - name: stats.user.ns + type: long + description: CPU time consumed by tasks in user mode. + + - name: stats.system.ns + type: long + description: CPU time consumed by tasks in user (kernel) mode. + + - name: percpu + type: object + object_type: long + description: > + CPU time (in nanoseconds) consumed on each CPU by all tasks in + this cgroup. + + - name: memory + type: group + description: Memory limits and metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's mountpoint. + + - name: mem.usage.bytes + type: long + format: bytes + description: > + Total memory usage by processes in the cgroup (in bytes). + + - name: mem.usage.max.bytes + type: long + format: bytes + description: > + The maximum memory used by processes in the cgroup (in bytes). + + - name: mem.limit.bytes + type: long + format: bytes + description: > + The maximum amount of user memory in bytes (including file + cache) that tasks in the cgroup are allowed to use. + + - name: mem.failures + type: long + description: > + The number of times that the memory limit (mem.limit.bytes) was + reached. + + - name: memsw.usage.bytes + type: long + format: bytes + description: > + The sum of current memory usage plus swap space used by + processes in the cgroup (in bytes). + + - name: memsw.usage.max.bytes + type: long + format: bytes + description: > + The maximum amount of memory and swap space used by processes in + the cgroup (in bytes). + + - name: memsw.limit.bytes + type: long + format: bytes + description: > + The maximum amount for the sum of memory and swap usage + that tasks in the cgroup are allowed to use. + + - name: memsw.failures + type: long + description: > + The number of times that the memory plus swap space limit + (memsw.limit.bytes) was reached. + + - name: kmem.usage.bytes + type: long + format: bytes + description: > + Total kernel memory usage by processes in the cgroup (in bytes). + + - name: kmem.usage.max.bytes + type: long + format: bytes + description: > + The maximum kernel memory used by processes in the cgroup (in + bytes). + + - name: kmem.limit.bytes + type: long + format: bytes + description: > + The maximum amount of kernel memory that tasks in the cgroup are + allowed to use. + + - name: kmem.failures + type: long + description: > + The number of times that the memory limit (kmem.limit.bytes) was + reached. + + - name: kmem_tcp.usage.bytes + type: long + format: bytes + description: > + Total memory usage for TCP buffers in bytes. + + - name: kmem_tcp.usage.max.bytes + type: long + format: bytes + description: > + The maximum memory used for TCP buffers by processes in the + cgroup (in bytes). + + - name: kmem_tcp.limit.bytes + type: long + format: bytes + description: > + The maximum amount of memory for TCP buffers that tasks in the + cgroup are allowed to use. + + - name: kmem_tcp.failures + type: long + description: > + The number of times that the memory limit (kmem_tcp.limit.bytes) + was reached. + + - name: stats.active_anon.bytes + type: long + format: bytes + description: > + Anonymous and swap cache on active least-recently-used (LRU) + list, including tmpfs (shmem), in bytes. + + - name: stats.active_file.bytes + type: long + format: bytes + description: File-backed memory on active LRU list, in bytes. + + - name: stats.cache.bytes + type: long + format: bytes + description: Page cache, including tmpfs (shmem), in bytes. + + - name: stats.hierarchical_memory_limit.bytes + type: long + format: bytes + description: > + Memory limit for the hierarchy that contains the memory cgroup, + in bytes. + + - name: stats.hierarchical_memsw_limit.bytes + type: long + format: bytes + description: > + Memory plus swap limit for the hierarchy that contains the + memory cgroup, in bytes. + + - name: stats.inactive_anon.bytes + type: long + format: bytes + description: > + Anonymous and swap cache on inactive LRU list, including tmpfs + (shmem), in bytes + + - name: stats.inactive_file.bytes + type: long + format: bytes + description: > + File-backed memory on inactive LRU list, in bytes. + + - name: stats.mapped_file.bytes + type: long + format: bytes + description: > + Size of memory-mapped mapped files, including tmpfs (shmem), + in bytes. + + - name: stats.page_faults + type: long + description: > + Number of times that a process in the cgroup triggered a page + fault. + + - name: stats.major_page_faults + type: long + description: > + Number of times that a process in the cgroup triggered a major + fault. "Major" faults happen when the kernel actually has to + read the data from disk. + + - name: stats.pages_in + type: long + description: > + Number of pages paged into memory. This is a counter. + + - name: stats.pages_out + type: long + description: > + Number of pages paged out of memory. This is a counter. + + - name: stats.rss.bytes + type: long + format: bytes + description: > + Anonymous and swap cache (includes transparent hugepages), not + including tmpfs (shmem), in bytes. + + - name: stats.rss_huge.bytes + type: long + format: bytes + description: > + Number of bytes of anonymous transparent hugepages. + + - name: stats.swap.bytes + type: long + format: bytes + description: > + Swap usage, in bytes. + + - name: stats.unevictable.bytes + type: long + format: bytes + description: > + Memory that cannot be reclaimed, in bytes. + + - name: blkio + type: group + description: Block IO metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystems mountpoint. + + - name: total.bytes + type: long + format: bytes + description: > + Total number of bytes transferred to and from all block devices + by processes in the cgroup. + + - name: total.ios + type: long + description: > + Total number of I/O operations performed on all devices + by processes in the cgroup as seen by the throttling policy. +- name: system.process.summary + title: Process Summary + type: group + description: > + Summary metrics for the processes running on the host. + release: ga + fields: + - name: total + type: long + description: > + Total number of processes on this host. + - name: running + type: long + description: > + Number of running processes on this host. + - name: idle + type: long + description: > + Number of idle processes on this host. + - name: sleeping + type: long + description: > + Number of sleeping processes on this host. + - name: stopped + type: long + description: > + Number of stopped processes on this host. + - name: zombie + type: long + description: > + Number of zombie processes on this host. + - name: dead + type: long + description: > + Number of dead processes on this host. It's very unlikely that it will appear but in some special situations it may happen. + - name: unknown + type: long + description: > + Number of processes for which the state couldn't be retrieved or is unknown. +- name: system.raid + type: group + description: > + raid + release: ga + fields: + - name: name + type: keyword + description: > + Name of the device. + - name: status + type: keyword + description: > + activity-state of the device. + - name: level + type: keyword + description: > + The raid level of the device + - name: sync_action + type: keyword + description: > + Current sync action, if the RAID array is redundant + - name: disks.active + type: long + description: > + Number of active disks. + - name: disks.total + type: long + description: > + Total number of disks the device consists of. + - name: disks.spare + type: long + description: > + Number of spared disks. + - name: disks.failed + type: long + description: > + Number of failed disks. + - name: disks.states.* + type: object + object_type: keyword + description: > + map of raw disk states + - name: blocks.total + type: long + description: > + Number of blocks the device holds, in 1024-byte blocks. + - name: blocks.synced + type: long + description: > + Number of blocks on the device that are in sync, in 1024-byte blocks. +- name: system.socket + type: group + description: > + TCP sockets that are active. + release: ga + fields: + - name: direction + type: alias + path: network.direction + migration: true + + - name: family + type: alias + path: network.type + migration: true + + - name: local.ip + type: ip + example: 192.0.2.1 or 2001:0DB8:ABED:8536::1 + description: > + Local IP address. This can be an IPv4 or IPv6 address. + + - name: local.port + type: long + example: 22 + description: > + Local port. + + - name: remote.ip + type: ip + example: 192.0.2.1 or 2001:0DB8:ABED:8536::1 + description: > + Remote IP address. This can be an IPv4 or IPv6 address. + + - name: remote.port + type: long + example: 22 + description: > + Remote port. + + - name: remote.host + type: keyword + example: 76-211-117-36.nw.example.com. + description: > + PTR record associated with the remote IP. It is obtained via reverse + IP lookup. + + - name: remote.etld_plus_one + type: keyword + example: example.com. + description: > + The effective top-level domain (eTLD) of the remote host plus one more + label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". + The data for determining the eTLD comes from an embedded copy of the data + from http://publicsuffix.org. + + - name: remote.host_error + type: keyword + description: > + Error describing the cause of the reverse lookup failure. + + - name: process.pid + type: alias + path: process.pid + migration: true + + - name: process.command + type: alias + path: process.name + migration: true + + - name: process.cmdline + type: keyword + description: > + Full command line + + - name: process.exe + type: alias + path: process.executable + migration: true + + - name: user.id + type: alias + path: user.id + migration: true + + - name: user.name + type: alias + path: user.full_name + migration: true +- name: system.socket.summary + title: Socket summary + type: group + description: > + Summary metrics of open sockets in the host system + release: ga + fields: + - name: all + type: group + description: > + All connections + fields: + - name: count + type: integer + description: > + All open connections + - name: listening + type: integer + description: > + All listening ports + - name: tcp + type: group + description: > + All TCP connections + fields: + - name: memory + type: integer + format: bytes + description: > + Memory used by TCP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/tcp_mem. Only available on Linux. + - name: all + type: group + description: > + All TCP connections + fields: + - name: orphan + type: integer + description: > + A count of all orphaned tcp sockets. Only available on Linux. + - name: count + type: integer + description: > + All open TCP connections + - name: listening + type: integer + description: > + All TCP listening ports + - name: established + type: integer + description: > + Number of established TCP connections + - name: close_wait + type: integer + description: > + Number of TCP connections in _close_wait_ state + - name: time_wait + type: integer + description: > + Number of TCP connections in _time_wait_ state + - name: syn_sent + type: integer + description: > + Number of TCP connections in _syn_sent_ state + - name: syn_recv + type: integer + description: > + Number of TCP connections in _syn_recv_ state + - name: fin_wait1 + type: integer + description: > + Number of TCP connections in _fin_wait1_ state + - name: fin_wait2 + type: integer + description: > + Number of TCP connections in _fin_wait2_ state + - name: last_ack + type: integer + description: > + Number of TCP connections in _last_ack_ state + - name: closing + type: integer + description: > + Number of TCP connections in _closing_ state + - name: udp + type: group + description: > + All UDP connections + fields: + - name: memory + type: integer + format: bytes + description: > + Memory used by UDP sockets in bytes, based on number of allocated pages and system page size. Corresponds to limits set in /proc/sys/net/ipv4/udp_mem. Only available on Linux. + - name: all + type: group + description: > + All UDP connections + fields: + - name: count + type: integer + description: > + All open UDP connections +- name: system.uptime + type: group + description: > + `uptime` contains the operating system uptime metric. + release: ga + fields: + - name: duration.ms + type: long + format: duration + input_format: milliseconds + description: > + The OS uptime in milliseconds. +- name: system.users + type: group + release: beta + description: > + Logged-in user session data + fields: + - name: id + type: keyword + description: > + The ID of the session + - name: seat + type: keyword + description: > + An associated logind seat + - name: path + type: keyword + description: > + The DBus object path of the session + - name: type + type: keyword + description: > + The type of the user session + - name: service + type: keyword + description: > + A session associated with the service + - name: remote + type: boolean + description: > + A bool indicating a remote session + - name: state + type: keyword + description: > + The current state of the session + - name: scope + type: keyword + description: > + The associated systemd scope + - name: leader + type: long + description: > + The root PID of the session + - name: remote_host + type: keyword + description: > + A remote host address for the session + + + + +