vector.toml

data_dir = "/etc/vector"

[api]
  enabled = true
  address = "127.0.0.1:8686"

[sources.internal_metrics]
  type = "internal_metrics"

[sources.livingdocs_server_socket]
  type = "socket"
  address = "0.0.0.0:4545"
  max_length = 102400
  mode = "udp"

[sources.file_source]
  type = "file"
  ignore_older_secs = 600
  include = [ "/logs/*.log" ]
  read_from = "beginning"
  # ignore the first bytes for the checksum (because they are equal)
  # otherwise the log files are not imported to grafana
  fingerprint.strategy = "checksum"
  fingerprint.ignored_header_bytes = 1024
  fingerprint.lines = 20

[sources.docker]
  type = "docker_logs"

[transforms.livingdocs_server]
  type = "add_fields"
  inputs = ["livingdocs_server_socket"]
  fields.service = "@livingdocs/server"

[transforms.docker_add_service]
  type = "lua"
  inputs = ["docker"]

  version = "2"
  hooks.process = "process"

  source = """
  function process(event, emit)
    event.log.service = event.log.SERVICE or event.log.IO_RANCHER_STACK_SERVICE_NAME
    if event.log.service then
      return emit(event)
    end

    if event.log.COM_DOCKER_COMPOSE_PROJECT then
      event.log.service = event.log.COM_DOCKER_COMPOSE_PROJECT .. "/" .. event.log.COM_DOCKER_COMPOSE_SERVICE
      return emit(event)
    end

    emit(event)
  end
  """

[transforms.json_detect_message]
  type = "swimlanes"
  inputs = ["file_source", "docker_add_service", "livingdocs_server"]

  [transforms.json_detect_message.lanes.no_json]
    type = "check_fields"
    "message.not_starts_with" = "{"

  [transforms.json_detect_message.lanes.is_json]
    type = "check_fields"
    "message.starts_with" = "{"

[transforms.json_parse_message]
  type = "json_parser"
  inputs = ["json_detect_message.is_json"]
  drop_invalid = false
  drop_field = false
  field = "message"
  target_field = "message_json"

[transforms.json_extract_message_level]
  type = "lua"
  inputs = ["json_parse_message"]
  version = "2"
  hooks.process = "process"

  source = """
  levels = {}
  raw_levels = {trace=10, debug=20, info=30, warn=40, error=50, fatal=60}
  for key, val in pairs(raw_levels) do
    levels[key] = key
    levels[string.upper(key)] = key
    levels[string.gsub(" "..key, "%W%l", string.upper):sub(2)] = key
    levels[tostring(val)] = key
    levels[val] = key
  end

  function process(event, emit)
    if event.log.message_json and event.log.message_json.level then
      event.log.level = levels[event.log.message_json.level]
    end
    emit(event)
  end
  """

[sinks.loki]
  type = "loki"
  inputs = ["json_detect_message.no_json", "json_extract_message_level"]
  healthcheck = true
  encoding = "text"
  endpoint = "http://loki:3100"

  [sinks.loki.labels]
    source_type = "{{ source_type }}"
    service = "{{ service }}"
    container_id = "{{ container_id }}"
    container_name = "{{ container_name }}"
    container_image = "{{ image }}"
    level = "{{ level }}"
    stream = "{{ stream }}"
    file = "{{ file }}"

[sinks.prometheus]
  type = "prometheus_exporter"
  inputs = ["internal_metrics"]
  address = "0.0.0.0:9160"

[[tests]]
  name = "check_json_log"

  [[tests.inputs]]
    insert_at = "livingdocs_server"
    type = "raw"
    value = "{\"level\":30,\"time\":1610722316561,\"req\":{\"id\":\"W4zYBoTFNDSo-206\",\"method\":\"GET\",\"url\":\"/documents?document_type=article&limit=35&sort=-updated_at\",\"userAgent\":\"Mozilla/5.0 (X11; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0\",\"remoteAddress\":\"127.0.0.1\",\"remotePort\":36620,\"ip\":\"127.0.0.1\"},\"traceId\":\"867e90eb8e9d7e98440cb29961399957\",\"res\":{\"statusCode\":304,\"userId\":\"1\",\"projectId\":\"3\",\"tokenId\":\"m4elDRnJwEMQFeBu\"},\"responseTime\":17,\"msg\":\"request completed\"}"

  [[tests.outputs]]
    extract_from = "json_extract_message_level"

    [[tests.outputs.conditions]]
      type = "check_fields"
      "message.exists" = true
      "message_json.exists" = true
      "message_json.time.equals" = 1610722316561
      "level.equals" = "info"
      "service.equals" = "@livingdocs/server"