sample-csv-api-noheader.conf

####################### INPUT ############################
# An input enables a specific source of
# events to be read by Logstash.
##########################################################

input {
  file {
  # Set the path to the source file(s)
     type => "set-type-as-your-table-name"
     path => "/data/sales.csv"
     start_position => "beginning"
     sincedb_path => "/dev/null"
  }
}

####################### FILTER ###########################
# A filter performs intermediary processing on an event.
# Filters are often applied conditionally depending on the
# characteristics of the event.
##########################################################

filter {

 # The CSV filter takes an event field containing CSV data,
 # parses it, and stores it as individual fields (can optionally specify the names).
 # This filter can also parse data with any separator, not just commas.

  csv {

  # Set the comma delimiter
    separator => ","

  # We want to exclude these system columns
    remove_field => [
       "message",
       "host",
       "@timestamp",
       "@version",
       "path"
    ]

  # Define the layout of the input file
    columns => [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands" ]
  }

  ##############################

  # Replaces non-ASCII characters with an ASCII approximation, or if none exists, a replacement character which defaults to ?
  i18n { transliterate =>  [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands" ] }

  #############################

  # Create consistent hashes (fingerprints) of one or more fields and store the result in a new field.
  fingerprint { method => "SHA1" key => "MySecretKeyForMyEyesOnlyOk?" source => "Brands" target => "Brands" }

  #############################

  # The mutate filter allows you to perform general
  # mutations on fields. You can rename, remove, replace
  # and modify fields in your events

  # We need to set the target column to "string" to allow for find and replace
  mutate {
    convert => [ "Sku", "string" ]
  }

  # Strip backslashes, question marks, equals, hashes, and minuses from the target column
  mutate {
    gsub => [ "Sku", "[\\?#=]", "" ]
  }

  # Strip extraneous white space from records
  mutate {
     strip => [ [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands"
     ]
  }

  # Set everything to lowercase
  mutate {
    lowercase => [ [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands"
    ]
  }

  #############################

  # Set rate limits. Do not change unless you know what you are doing
  sleep { time => "1" every => 100 }

}

####################### OUTPUT ###########################
# An output sends event data to a particular
# destination. Outputs are the final stage in the
# event pipeline.
##########################################################

output
{
  http
  {
  # Put the URL for your HTTP endpoint to deliver events to
    url => "https://myapi.foo-api.us-east-1.amazonaws.com/dev/events/teststash?token=774f77b389154fd2ae7cb5131201777&sign=ujguuuljNjBkFGHyNTNmZTIxYjEzMWE5MjgyNzM1ODQ="

    # Leave the settings below untouched.
    http_method => "post"
    format => "json"
    keepalive => "true"
    automatic_retries => 1
    validate_after_inactivity => "45"
    pool_max => "10"
    pool_max_per_route => "5"
    codec => plain { charset => 'UTF-8' }
  }
}