My

[norfolkislander]

I have setup logstash, elasticsearch and kibana. It all works great however TTL is not always being mapped to indices when they are created (only some of the time)? Initially I tried setting up mappings as per Attaching a TTL field with every log sent via logstash to Elasticsearch . But this just does not seem to work for me? I tried both locations /etc/elasticsearch/mappings/_default/default.json, /etc/elasticsearch/config/mappings/_default/default.json but neither seemed to load the mapping. I did find instructions to create a /etc/elasticsearch/default-mapping.json file which seems to work intermittently.

The json config file always contained the following (on each location attempt):

{
default : {
"_ttl" : { "enabled" : true, "default" : "1d" }
}
}

{
logstash-2013.10.18: {
logs: {
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.09: {
logs: {
_ttl: {
enabled: true,
default: 86400000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.19: {
logs: {
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.07: {
logs: {
_ttl: {
enabled: true,
default: 86400000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
tags: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.14: {
logs: {
_ttl: {
enabled: true,
default: 2592000000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.15: {
logs: {
_ttl: {
enabled: true,
default: 2592000000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.16: {
logs: {
_ttl: {
enabled: true,
default: 2592000000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.17: {
logs: {
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.22: {
logs: {
_ttl: {
enabled: true,
default: 2592000000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.13: {
logs: {
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.21: {
logs: {
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.12: {
logs: {
_ttl: {
enabled: true,
default: 2592000000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
kibana-int: {
temp: {
properties: {
dashboard: {
type: "string"
},
group: {
type: "string"
},
title: {
type: "string"
},
user: {
type: "string"
}
}
}
},
logstash-2013.10.11: {
logs: {
_ttl: {
enabled: true,
default: 2592000000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.10: {
logs: {
_ttl: {
enabled: true,
default: 86400000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
tags: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.23: {
logs: {
_ttl: {
enabled: true,
default: 2592000000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
},
logstash-2013.10.20: {
logs: {
_ttl: {
enabled: true,
default: 2592000000
},
properties: {
@timestamp: {
type: "date",
format: "dateOptionalTime"
},
@Version: {
type: "string"
},
facility: {
type: "long"
},
facility_label: {
type: "string"
},
host: {
type: "string"
},
logsource: {
type: "string"
},
message: {
type: "string"
},
priority: {
type: "long"
},
program: {
type: "string"
},
severity: {
type: "long"
},
severity_label: {
type: "string"
},
timestamp: {
type: "string"
},
type: {
type: "string"
}
}
}
}
}

[bleskes]

Hi @norfolkislander,

Currently ttl is only supported on the document level. There is no ttl support for indices as a whole. There is an issue open requested that feature - so you can watch that for updates: #2114

If you have anymore questions - can you please post them on the mailing list? we try to keep issues on github for feature request or bug reports.

Thx!
Boaz