From 56c714b2b051adf2670d760af3da6ffbe2fcea89 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 26 Jun 2017 17:01:18 -0700 Subject: [PATCH 001/154] Add inital skeleton for zipkin input plugin --- plugins/inputs/zipkin/zipkin.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 plugins/inputs/zipkin/zipkin.go diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go new file mode 100644 index 0000000000000..60d32a7c77ffa --- /dev/null +++ b/plugins/inputs/zipkin/zipkin.go @@ -0,0 +1,29 @@ +package zipkin + +import "github.com/influxdata/telegraf" + +type Zipkin struct { + //Add configuration fields here later + Field bool +} + +const sampleConfig = ` + ## + # field = value +` + +func (z Zipkin) Description() string { + return "Allows for the collection of zipkin tracing spans for storage in influxdb" +} + +func (z Zipkin) SampleConfig() string { + return sampleConfig +} + +func (z *Zipkin) Gather(acc telegraf.Accumulator) { + if z.Field { + acc.AddFields("state", map[string]interface{}{"value": "true"}, nil) + } else { + acc.AddFields("state", map[string]interface{}{"value": "false"}, nil) + } +} From ae65f10140a45700e276ead07524fb2d74398c9d Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 27 Jun 2017 16:18:39 -0700 Subject: [PATCH 002/154] Add concurrent zipkin http server to zipkin plugin for data fetching --- plugins/inputs/zipkin/zipkin.go | 29 --- plugins/inputs/zipkin/zipkin/zipkin_server.go | 165 ++++++++++++++++++ 2 files changed, 165 insertions(+), 29 deletions(-) delete mode 100644 plugins/inputs/zipkin/zipkin.go create mode 100644 plugins/inputs/zipkin/zipkin/zipkin_server.go diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go deleted file mode 100644 index 60d32a7c77ffa..0000000000000 --- a/plugins/inputs/zipkin/zipkin.go +++ /dev/null @@ -1,29 +0,0 @@ -package zipkin - -import "github.com/influxdata/telegraf" - -type Zipkin struct { - //Add configuration fields here later - Field bool -} - -const sampleConfig = ` - ## - # field = value -` - -func (z Zipkin) Description() string { - return "Allows for the collection of zipkin tracing spans for storage in influxdb" -} - -func (z Zipkin) SampleConfig() string { - return sampleConfig -} - -func (z *Zipkin) Gather(acc telegraf.Accumulator) { - if z.Field { - acc.AddFields("state", map[string]interface{}{"value": "true"}, nil) - } else { - acc.AddFields("state", map[string]interface{}{"value": "false"}, nil) - } -} diff --git a/plugins/inputs/zipkin/zipkin/zipkin_server.go b/plugins/inputs/zipkin/zipkin/zipkin_server.go new file mode 100644 index 0000000000000..16d2fbb8f5b50 --- /dev/null +++ b/plugins/inputs/zipkin/zipkin/zipkin_server.go @@ -0,0 +1,165 @@ +package zipkin + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "strconv" + "time" + + "github.com/apache/thrift/lib/go/thrift" + "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" + "github.com/tylerb/graceful" +) + +const version = "1.0" +const port = "9411" + +type Server struct { + errorChan chan error + dataChan chan SpanData + Port string + Done chan struct{} + listener net.Listener +} + +type SpanData []*zipkincore.Span + +func NewHTTPServer(port int, e chan error, d chan SpanData, f chan struct{}) *Server { + return &Server{ + errorChan: e, + dataChan: d, + Port: strconv.Itoa(port), + Done: f, + } +} + +// Version adds a version header to response +func Version(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Proxy-Version", version) + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +// Logger is middleware that logs the request +func Logger(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + log.Printf("Received request from url %s\n", r.URL.String()) + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +// MainHandler returns a closure with access to a *Server pointer +// for use as an http server handler +func (s *Server) MainHandler() http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + log.Printf("Received request from: %s", r.URL.String()) + log.Printf("Raw request data is: %#+v", r) + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + if err != nil { + e := fmt.Errorf("Encoutered error: %s", err) + log.Println(e) + s.errorChan <- e + } + buffer := thrift.NewTMemoryBuffer() + if _, err = buffer.Write(body); err != nil { + log.Println(err) + s.errorChan <- err + } + transport := thrift.NewTBinaryProtocolTransport(buffer) + _, size, err := transport.ReadListBegin() + if err != nil { + log.Printf("%s", err) + s.errorChan <- err + return + } + var spans []*zipkincore.Span + for i := 0; i < size; i++ { + zs := &zipkincore.Span{} + if err = zs.Read(transport); err != nil { + log.Printf("%s", err) + s.errorChan <- err + return + } + spans = append(spans, zs) + } + err = transport.ReadListEnd() + if err != nil { + log.Printf("%s", err) + s.errorChan <- err + return + } + out, _ := json.MarshalIndent(spans, "", " ") + log.Println(string(out)) + s.dataChan <- SpanData(spans) + w.WriteHeader(http.StatusNoContent) + } + return http.HandlerFunc(fn) +} + +func (s *Server) HandleZipkinRequests() { + log.Printf("Starting zipkin HTTP server on %s\n", s.Port) + mux := http.NewServeMux() + // The func MainHandler returns has been closure-ified + mux.Handle("/api/v1/spans", s.MainHandler()) + + listener, err := net.Listen("tcp", ":"+s.Port) + if err != nil { + //Print error out for debugging purposes + e := fmt.Errorf("Error listening on port 9411 %v", err) + //Send error through channel to caller + s.errorChan <- e + return + } + + s.addListener(listener) + go s.ListenForStop() + + httpServer := &graceful.Server{Server: new(http.Server)} + httpServer.SetKeepAlivesEnabled(true) + httpServer.TCPKeepAlive = 5 * time.Second + httpServer.Handler = Version(Logger(mux)) + log.Fatal(httpServer.Serve(listener)) +} + +func (s *Server) addListener(l net.Listener) { + s.listener = l +} + +func (s *Server) ListenForStop() { + if s.listener == nil { + log.Fatal("Listen called without listener instance") + return + } + select { + case <-s.Done: + log.Println("closing up listener...") + s.listener.Close() + return + } +} + +func (s *Server) CloseAllChannels() { + log.Printf("Closing all communication channels...\n") + close(s.dataChan) + close(s.errorChan) + close(s.Done) +} + +/*func main() { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + server() + }() + wg.Wait() +}*/ From 83ff0e7e4e8ed0c551eb6673e4a42a76b5ece68c Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 27 Jun 2017 16:24:09 -0700 Subject: [PATCH 003/154] Add new skeleton for telegraf.ServiceInput to zipkin plugin --- plugins/inputs/zipkin/zipkin/zipkin.go | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 plugins/inputs/zipkin/zipkin/zipkin.go diff --git a/plugins/inputs/zipkin/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin/zipkin.go new file mode 100644 index 0000000000000..af1c639640dcc --- /dev/null +++ b/plugins/inputs/zipkin/zipkin/zipkin.go @@ -0,0 +1,37 @@ +package zipkin + +import "github.com/influxdata/telegraf" + +type Zipkin struct { + //Add configuration fields here later + Field bool +} + +const sampleConfig = ` + ## + # field = value +` + +func (z Zipkin) Description() string { + return "Allows for the collection of zipkin tracing spans for storage in influxdb" +} + +func (z Zipkin) SampleConfig() string { + return sampleConfig +} + +func (z *Zipkin) Gather(acc telegraf.Accumulator) { + if z.Field { + acc.AddFields("state", map[string]interface{}{"value": "true"}, nil) + } else { + acc.AddFields("state", map[string]interface{}{"value": "false"}, nil) + } +} + +func (z *Zipkin) Start(acc telegraf.Accumulator) { + //start collecting data +} + +func (z *Zipkin) Stop() { + //clean up any channels, etc. +} From 4d74b4da3b9a9898e8eb19d9b6336212bd996f9d Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 27 Jun 2017 16:24:42 -0700 Subject: [PATCH 004/154] Add simple test program for zipkin http server to zipkin plugin --- plugins/inputs/zipkin/main.go | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 plugins/inputs/zipkin/main.go diff --git a/plugins/inputs/zipkin/main.go b/plugins/inputs/zipkin/main.go new file mode 100644 index 0000000000000..0cab7e28bb99c --- /dev/null +++ b/plugins/inputs/zipkin/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + + "github.com/influxdata/telegraf/plugins/inputs/zipkin/zipkin" +) + +func main() { + e := make(chan error) + d := make(chan zipkin.SpanData) + f := make(chan struct{}) + s := zipkin.NewHTTPServer(9411, e, d, f) + go s.HandleZipkinRequests() + + sigChan := make(chan os.Signal) + signal.Notify(sigChan, os.Interrupt) + sigHandle(sigChan, s) + + for { + select { + case err := <-e: + fmt.Println("error: ", err) + case data := <-d: + fmt.Println("Got zipkin data: %#+v", data) + } + } + +} + +func sigHandle(c chan os.Signal, server *zipkin.Server) { + select { + case <-c: + fmt.Println("received SIGINT, stopping server") + server.Done <- struct{}{} + server.CloseAllChannels() + } +} From bba89223cc1e108815ca7f88b56e1259d835b8d0 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 10:47:58 -0700 Subject: [PATCH 005/154] Add Godoc comments to zipkin_server.go in zipkin input plugin --- plugins/inputs/zipkin/zipkin/zipkin_server.go | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin/zipkin_server.go b/plugins/inputs/zipkin/zipkin/zipkin_server.go index 16d2fbb8f5b50..d65b8cd1f4eab 100644 --- a/plugins/inputs/zipkin/zipkin/zipkin_server.go +++ b/plugins/inputs/zipkin/zipkin/zipkin_server.go @@ -26,8 +26,12 @@ type Server struct { listener net.Listener } +// SpanData is an alias for a slice of references to zipkincore.Span +// created for better abstraction of internal package types type SpanData []*zipkincore.Span +// NewHTTPServer creates a new Zipkin http server given a port and a set of +// channels func NewHTTPServer(port int, e chan error, d chan SpanData, f chan struct{}) *Server { return &Server{ errorChan: e, @@ -105,6 +109,10 @@ func (s *Server) MainHandler() http.Handler { return http.HandlerFunc(fn) } +// HandleZipkinRequests starts a zipkin http server on the port specified +// wthin the *Server it is called on. It receives data from zipkin, and sends +// it back to the caller over the channels the caller constructed the *Server +// with func (s *Server) HandleZipkinRequests() { log.Printf("Starting zipkin HTTP server on %s\n", s.Port) mux := http.NewServeMux() @@ -134,6 +142,8 @@ func (s *Server) addListener(l net.Listener) { s.listener = l } +// ListenForStop selects over the Server.Done channel, and stops the +// server's internal net.Listener when a singnal is received. func (s *Server) ListenForStop() { if s.listener == nil { log.Fatal("Listen called without listener instance") @@ -147,19 +157,10 @@ func (s *Server) ListenForStop() { } } +// CloseAllChannels closes the Server's communication channels on the server's end. func (s *Server) CloseAllChannels() { log.Printf("Closing all communication channels...\n") close(s.dataChan) close(s.errorChan) close(s.Done) } - -/*func main() { - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - server() - }() - wg.Wait() -}*/ From 89c471ad2d0449986af5911daa6afac2193256f2 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 14:37:18 -0700 Subject: [PATCH 006/154] Update directory structure in zipkin plugin --- plugins/inputs/zipkin/{ => cmd}/main.go | 0 plugins/inputs/zipkin/{zipkin => }/zipkin.go | 0 plugins/inputs/zipkin/{zipkin => }/zipkin_server.go | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/{ => cmd}/main.go (100%) rename plugins/inputs/zipkin/{zipkin => }/zipkin.go (100%) rename plugins/inputs/zipkin/{zipkin => }/zipkin_server.go (100%) diff --git a/plugins/inputs/zipkin/main.go b/plugins/inputs/zipkin/cmd/main.go similarity index 100% rename from plugins/inputs/zipkin/main.go rename to plugins/inputs/zipkin/cmd/main.go diff --git a/plugins/inputs/zipkin/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go similarity index 100% rename from plugins/inputs/zipkin/zipkin/zipkin.go rename to plugins/inputs/zipkin/zipkin.go diff --git a/plugins/inputs/zipkin/zipkin/zipkin_server.go b/plugins/inputs/zipkin/zipkin_server.go similarity index 100% rename from plugins/inputs/zipkin/zipkin/zipkin_server.go rename to plugins/inputs/zipkin/zipkin_server.go From 72668e935c91f2f48f2bd05fec940bda5ceb63ec Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 14:39:07 -0700 Subject: [PATCH 007/154] Rename zipkin_server.go to server.go --- plugins/inputs/zipkin/{zipkin_server.go => server.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/{zipkin_server.go => server.go} (100%) diff --git a/plugins/inputs/zipkin/zipkin_server.go b/plugins/inputs/zipkin/server.go similarity index 100% rename from plugins/inputs/zipkin/zipkin_server.go rename to plugins/inputs/zipkin/server.go From 158e13c0637f7bd84a4980ecba76535b2864b667 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 14:40:56 -0700 Subject: [PATCH 008/154] Add TODO comments to server.go in zipkin plugin --- plugins/inputs/zipkin/server.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index d65b8cd1f4eab..7c062c0b3b41d 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -42,6 +42,7 @@ func NewHTTPServer(port int, e chan error, d chan SpanData, f chan struct{}) *Se } // Version adds a version header to response +// Delete later func Version(h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { w.Header().Add("X-Proxy-Version", version) @@ -51,6 +52,8 @@ func Version(h http.Handler) http.Handler { } // Logger is middleware that logs the request +// delete later, re-implement in a better way +// inspired by the httptrace package func Logger(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { log.Printf("Received request from url %s\n", r.URL.String()) @@ -129,13 +132,23 @@ func (s *Server) HandleZipkinRequests() { } s.addListener(listener) + //TODO: put a sync group around this ListenForStop() + // create wait group; add to wait group (wg.Add(1)) + // pass in to ListenForStop() + //wg.Add() + //go func(){ go s.ListenForStop() + //wg.Done() + //}() + // TODO: don't need to use graceful anymore in go 1.8 (there is graceful Server + // shutdown) httpServer := &graceful.Server{Server: new(http.Server)} httpServer.SetKeepAlivesEnabled(true) httpServer.TCPKeepAlive = 5 * time.Second httpServer.Handler = Version(Logger(mux)) log.Fatal(httpServer.Serve(listener)) + } func (s *Server) addListener(l net.Listener) { From f7acf33080a647d730abd693ee149eb9d805648d Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 28 Jun 2017 15:59:11 -0700 Subject: [PATCH 009/154] Fix import path in main.go in zipkin plugin Signed-off-by: Adam Perlin --- plugins/inputs/zipkin/cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/cmd/main.go b/plugins/inputs/zipkin/cmd/main.go index 0cab7e28bb99c..c2dacd6494fae 100644 --- a/plugins/inputs/zipkin/cmd/main.go +++ b/plugins/inputs/zipkin/cmd/main.go @@ -5,7 +5,7 @@ import ( "os" "os/signal" - "github.com/influxdata/telegraf/plugins/inputs/zipkin/zipkin" + "github.com/influxdata/telegraf/plugins/inputs/zipkin" ) func main() { From 8a858d9f175318acade6d16acbee90ded1f8e59f Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 16:01:06 -0700 Subject: [PATCH 010/154] Fix handler in server.go so that it returns correct http status in zipkin plugin --- plugins/inputs/zipkin/server.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index 7c062c0b3b41d..f6e2fcbb147e3 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -75,17 +75,22 @@ func (s *Server) MainHandler() http.Handler { e := fmt.Errorf("Encoutered error: %s", err) log.Println(e) s.errorChan <- e + w.WriteHeader(http.StatusNoContent) + return } buffer := thrift.NewTMemoryBuffer() if _, err = buffer.Write(body); err != nil { log.Println(err) s.errorChan <- err + w.WriteHeader(http.StatusNoContent) + return } transport := thrift.NewTBinaryProtocolTransport(buffer) _, size, err := transport.ReadListBegin() if err != nil { log.Printf("%s", err) s.errorChan <- err + w.WriteHeader(http.StatusNoContent) return } var spans []*zipkincore.Span @@ -94,6 +99,7 @@ func (s *Server) MainHandler() http.Handler { if err = zs.Read(transport); err != nil { log.Printf("%s", err) s.errorChan <- err + w.WriteHeader(http.StatusNoContent) return } spans = append(spans, zs) @@ -102,6 +108,7 @@ func (s *Server) MainHandler() http.Handler { if err != nil { log.Printf("%s", err) s.errorChan <- err + w.WriteHeader(http.StatusNoContent) return } out, _ := json.MarshalIndent(spans, "", " ") From 6ecefac85a4b73392bff00d8da7f2b4947d4338b Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 16:03:33 -0700 Subject: [PATCH 011/154] Add test for zipkin http server Signed-off-by: Ayrdrie Palmer --- plugins/inputs/zipkin/server_test.go | 60 ++++++++++++++++++++++ plugins/inputs/zipkin/test/threespans.dat | Bin 0 -> 616 bytes 2 files changed, 60 insertions(+) create mode 100644 plugins/inputs/zipkin/server_test.go create mode 100644 plugins/inputs/zipkin/test/threespans.dat diff --git a/plugins/inputs/zipkin/server_test.go b/plugins/inputs/zipkin/server_test.go new file mode 100644 index 0000000000000..315892176ecd4 --- /dev/null +++ b/plugins/inputs/zipkin/server_test.go @@ -0,0 +1,60 @@ +package zipkin + +import ( + "bytes" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +func TestMainHandler(t *testing.T) { + dat, err := ioutil.ReadFile("test/threespans.dat") + if err != nil { + t.Fatal("Could not find threespans.dat") + } + e := make(chan error, 1) + d := make(chan SpanData, 1) + f := make(chan struct{}, 1) + s := NewHTTPServer(9411, e, d, f) + w := httptest.NewRecorder() + r := httptest.NewRequest( + "POST", + "http://server.local/api/v1/spans", + ioutil.NopCloser( + bytes.NewReader(dat))) + handler := s.MainHandler() + handler.ServeHTTP(w, r) + if w.Code != http.StatusNoContent { + t.Errorf("MainHandler did not return StatusNoContent %d", w.Code) + } + + spans := <-d + if len(spans) != 3 { + t.Fatalf("Expected 3 spans received len(spans) %d", len(spans)) + } + if spans[0].ID != 8090652509916334619 { + t.Errorf("Expected 8090652509916334619 but received ID %d ", spans[0].ID) + } + if spans[0].TraceID != 2505404965370368069 { + t.Errorf("Expected 2505404965370368069 but received TraceID %d ", spans[0].TraceID) + } + if spans[0].Name != "Child" { + t.Errorf("Expected Child but recieved name %s", spans[0].Name) + } + if *(spans[0].ParentID) != 22964302721410078 { + t.Errorf("Expected 22964302721410078 but recieved parent id %d", spans[0].ParentID) + } + serviceName := spans[0].GetBinaryAnnotations()[0].GetHost().GetServiceName() + if serviceName != "trivial" { + t.Errorf("Expected trivial but recieved service name %s", serviceName) + } + + if spans[0].GetTimestamp() != 1498688360851331 { + t.Errorf("Expected timestamp %d but recieved timestamp %d", 1498688360851331, spans[0].GetTimestamp()) + } + + if spans[0].GetDuration() != 53106 { + t.Errorf("Expected duration %d but recieved duration %d", 53106, spans[0].GetDuration()) + } +} diff --git a/plugins/inputs/zipkin/test/threespans.dat b/plugins/inputs/zipkin/test/threespans.dat new file mode 100644 index 0000000000000000000000000000000000000000..3fe5fb2ef0714d1968157cc3639fbf754ed5d788 GIT binary patch literal 616 zcmd;KU|?Y8VqjD{@~3Na+5}f_241FG?q`4Sa83L!iOtMUo z<7Z&w0V-kOXW#(QjNA;2K#D0RnVW$L$Y3ui$}G!F%;5kV1XRkx!N6D#RL2IAh8Pc1 z!@$4=gsj0llM|LVa{(p6;CvBK5SN1(3lv_5uGs&S)(%Q6&wxAV$SX_-u>~X+rRJ3Y zg9t5LK!FMMQY|Dz1cOTwi%K%{(iM_Hp{k&4z{9|dKTsehB$g-QFu{;W6OwESASSRn VD){856VOHF7%hVZAmda91^{IfXEFc) literal 0 HcmV?d00001 From 898b3b672e66168f17033055d9f9f4789418a855 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 17:27:38 -0700 Subject: [PATCH 012/154] Modify internal http server in zipkin plugin to use go 1.8 features for graceful Refactor serve and shutdown methods Remove unused channel --- plugins/inputs/zipkin/server.go | 76 ++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index f6e2fcbb147e3..0b76ef4a00579 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -1,29 +1,37 @@ package zipkin import ( + "context" "encoding/json" "fmt" "io/ioutil" "log" - "net" "net/http" + "os" "strconv" "time" "github.com/apache/thrift/lib/go/thrift" "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" - "github.com/tylerb/graceful" ) const version = "1.0" const port = "9411" +// Zipkin writes to this route by default +const DefaultRoute = "/api/v1/spans" + +// Provides a shutdown timout in order to gracefully shutdown our http server +const DefaultShutdownTimeout = 5 * time.Second + type Server struct { errorChan chan error dataChan chan SpanData Port string - Done chan struct{} - listener net.Listener + //Done chan struct{} + //listener net.Listener + HTTPServer *http.Server + logger *log.Logger } // SpanData is an alias for a slice of references to zipkincore.Span @@ -32,12 +40,13 @@ type SpanData []*zipkincore.Span // NewHTTPServer creates a new Zipkin http server given a port and a set of // channels -func NewHTTPServer(port int, e chan error, d chan SpanData, f chan struct{}) *Server { +func NewHTTPServer(port int, e chan error, d chan SpanData) *Server { + logger := log.New(os.Stdout, "", 0) return &Server{ errorChan: e, dataChan: d, Port: strconv.Itoa(port), - Done: f, + logger: logger, } } @@ -67,20 +76,20 @@ func Logger(next http.Handler) http.Handler { func (s *Server) MainHandler() http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - log.Printf("Received request from: %s", r.URL.String()) - log.Printf("Raw request data is: %#+v", r) + s.logger.Printf("Received request from: %s", r.URL.String()) + s.logger.Printf("Raw request data is: %#+v", r) defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) if err != nil { e := fmt.Errorf("Encoutered error: %s", err) - log.Println(e) + s.logger.Println(e) s.errorChan <- e w.WriteHeader(http.StatusNoContent) return } buffer := thrift.NewTMemoryBuffer() if _, err = buffer.Write(body); err != nil { - log.Println(err) + s.logger.Println(err) s.errorChan <- err w.WriteHeader(http.StatusNoContent) return @@ -88,7 +97,7 @@ func (s *Server) MainHandler() http.Handler { transport := thrift.NewTBinaryProtocolTransport(buffer) _, size, err := transport.ReadListBegin() if err != nil { - log.Printf("%s", err) + s.logger.Printf("%s", err) s.errorChan <- err w.WriteHeader(http.StatusNoContent) return @@ -97,7 +106,7 @@ func (s *Server) MainHandler() http.Handler { for i := 0; i < size; i++ { zs := &zipkincore.Span{} if err = zs.Read(transport); err != nil { - log.Printf("%s", err) + s.logger.Printf("%s", err) s.errorChan <- err w.WriteHeader(http.StatusNoContent) return @@ -106,24 +115,48 @@ func (s *Server) MainHandler() http.Handler { } err = transport.ReadListEnd() if err != nil { - log.Printf("%s", err) + s.logger.Printf("%s", err) s.errorChan <- err w.WriteHeader(http.StatusNoContent) return } out, _ := json.MarshalIndent(spans, "", " ") - log.Println(string(out)) + s.logger.Println(string(out)) s.dataChan <- SpanData(spans) w.WriteHeader(http.StatusNoContent) } return http.HandlerFunc(fn) } +// Serve creates an internal http.Server and calls its ListenAndServe method to +// start serving. It uses the MainHandler as the route handler. +func (s *Server) Serve() { + mux := http.NewServeMux() + mux.Handle(DefaultRoute, s.MainHandler()) + + s.HTTPServer = &http.Server{ + Addr: ":" + s.Port, + Handler: mux, + } + + s.logger.Printf("Starting zipkin HTTP server on %s\n", s.Port) + s.logger.Fatal(s.HTTPServer.ListenAndServe()) +} + +// Shutdown gracefully shuts down the internal http server +func (s *Server) Shutdown() { + ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) + defer cancel() + + s.HTTPServer.Shutdown(ctx) +} + // HandleZipkinRequests starts a zipkin http server on the port specified // wthin the *Server it is called on. It receives data from zipkin, and sends // it back to the caller over the channels the caller constructed the *Server // with -func (s *Server) HandleZipkinRequests() { + +/*func (s *Server) HandleZipkinRequests() { log.Printf("Starting zipkin HTTP server on %s\n", s.Port) mux := http.NewServeMux() // The func MainHandler returns has been closure-ified @@ -154,17 +187,18 @@ func (s *Server) HandleZipkinRequests() { httpServer.SetKeepAlivesEnabled(true) httpServer.TCPKeepAlive = 5 * time.Second httpServer.Handler = Version(Logger(mux)) + log.Fatal(httpServer.Serve(listener)) -} +}*/ -func (s *Server) addListener(l net.Listener) { +/*func (s *Server) addListener(l net.Listener) { s.listener = l -} +}*/ // ListenForStop selects over the Server.Done channel, and stops the // server's internal net.Listener when a singnal is received. -func (s *Server) ListenForStop() { +/*func (s *Server) ListenForStop() { if s.listener == nil { log.Fatal("Listen called without listener instance") return @@ -175,12 +209,12 @@ func (s *Server) ListenForStop() { s.listener.Close() return } -} +}*/ // CloseAllChannels closes the Server's communication channels on the server's end. func (s *Server) CloseAllChannels() { log.Printf("Closing all communication channels...\n") close(s.dataChan) close(s.errorChan) - close(s.Done) + //close(s.Done) } From 157b20090b3cec168a9e5f977ac7b3cf94c3b1f7 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 17:32:12 -0700 Subject: [PATCH 013/154] Update main.go to use new methods in zipkin server --- plugins/inputs/zipkin/cmd/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/zipkin/cmd/main.go b/plugins/inputs/zipkin/cmd/main.go index c2dacd6494fae..74ab92afe5cc1 100644 --- a/plugins/inputs/zipkin/cmd/main.go +++ b/plugins/inputs/zipkin/cmd/main.go @@ -11,9 +11,8 @@ import ( func main() { e := make(chan error) d := make(chan zipkin.SpanData) - f := make(chan struct{}) - s := zipkin.NewHTTPServer(9411, e, d, f) - go s.HandleZipkinRequests() + s := zipkin.NewHTTPServer(9411, e, d) + go s.Serve() sigChan := make(chan os.Signal) signal.Notify(sigChan, os.Interrupt) @@ -34,7 +33,8 @@ func sigHandle(c chan os.Signal, server *zipkin.Server) { select { case <-c: fmt.Println("received SIGINT, stopping server") - server.Done <- struct{}{} + //server.Done <- struct{}{} + server.Shutdown() server.CloseAllChannels() } } From b3dda370188c8f5f125de58303c86e9e4a0d011a Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 17:33:18 -0700 Subject: [PATCH 014/154] Update server_test.go to call zipkin server constructor with the correct arguments --- plugins/inputs/zipkin/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/server_test.go b/plugins/inputs/zipkin/server_test.go index 315892176ecd4..370ad3a2bca9d 100644 --- a/plugins/inputs/zipkin/server_test.go +++ b/plugins/inputs/zipkin/server_test.go @@ -15,8 +15,8 @@ func TestMainHandler(t *testing.T) { } e := make(chan error, 1) d := make(chan SpanData, 1) - f := make(chan struct{}, 1) - s := NewHTTPServer(9411, e, d, f) + //f := make(chan struct{}, 1) + s := NewHTTPServer(9411, e, d) w := httptest.NewRecorder() r := httptest.NewRequest( "POST", From b6ebe5c9d38702be2e9643ff9939a1c3656e79ca Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 17:36:55 -0700 Subject: [PATCH 015/154] Add TODO comment to server.go in zipkin plugin --- plugins/inputs/zipkin/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index 0b76ef4a00579..446e218a383e4 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -11,6 +11,7 @@ import ( "strconv" "time" + //TODO: must add below deps to Godeps file "github.com/apache/thrift/lib/go/thrift" "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) From 1147bcf5101454ab010b617492ddee8859af83e3 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 28 Jun 2017 17:48:24 -0700 Subject: [PATCH 016/154] Update Godeps --- Godeps | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Godeps b/Godeps index aa4a5b0e8005d..f6de2fe99f5ab 100644 --- a/Godeps +++ b/Godeps @@ -3,6 +3,7 @@ github.com/Shopify/sarama c01858abb625b73a3af51d0798e4ad42c8147093 github.com/Sirupsen/logrus 61e43dc76f7ee59a82bdf3d71033dc12bea4c77d github.com/aerospike/aerospike-client-go 95e1ad7791bdbca44707fedbb29be42024900d9c github.com/amir/raidman c74861fe6a7bb8ede0a010ce4485bdbb4fc4c985 +github.com/apache/thrift 4aaa92ece8503a6da9bc6701604f69acf2b99d07 github.com/aws/aws-sdk-go c861d27d0304a79f727e9a8a4e2ac1e74602fdc0 github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 github.com/bsm/sarama-cluster ccdc0803695fbce22f1706d04ded46cd518fd832 @@ -37,6 +38,7 @@ github.com/nats-io/go-nats ea9585611a4ab58a205b9b125ebd74c389a6b898 github.com/nats-io/nats ea9585611a4ab58a205b9b125ebd74c389a6b898 github.com/nats-io/nuid 289cccf02c178dc782430d534e3c1f5b72af807f github.com/nsqio/go-nsq a53d495e81424aaf7a7665a9d32a97715c40e953 +github.com/openzipkin/zipkin-go-opentracing 1cafbdfde94fbf2b373534764e0863aa3bd0bf7b github.com/pierrec/lz4 5c9560bfa9ace2bf86080bf40d46b34ae44604df github.com/pierrec/xxHash 5a004441f897722c627870a981d02b29924215fa github.com/pkg/errors 645ef00459ed84a119197bfb8d8205042c6df63d From 53eaa0218331d87204ec1ac702f9c824e62d140c Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 29 Jun 2017 14:59:47 -0700 Subject: [PATCH 017/154] Begin to add interfaces to zipkin plugin --- plugins/inputs/zipkin/zipkin.go | 209 ++++++++++++++++++++++++++++++-- 1 file changed, 201 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index af1c639640dcc..c1401b3c193a4 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -1,10 +1,207 @@ package zipkin -import "github.com/influxdata/telegraf" +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "strconv" + "sync" + "time" + + "github.com/apache/thrift/lib/go/thrift" + "github.com/gorilla/mux" + "github.com/influxdata/telegraf" + "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" +) + +type BinaryAnnotation struct { + //stuff +} + +type Annotation struct { + Timestamp time.Time + Value string + Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port + ServiceName string +} + +type Span struct { + ID string // zipkin traceid high concat with traceid + Name string + ParentID *int64 + Timestamp time.Time // If zipkin input is nil then time.Now() + Duration *int64 + TraceIDHigh *int64 + Annotations []Annotation +} + +type Trace []Span + +type Tracer interface { + Record(Trace) error + Error(error) +} + +type Service interface { + Register(router *mux.Router, tracer Tracer) +} type Zipkin struct { - //Add configuration fields here later - Field bool + ServiceAddress string + Path string + tracing Service + server *http.Server +} + +type Server struct { + Path string + Port string + tracer Tracer + waitGroup *sync.WaitGroup +} + +func NewServer(path string, port int) *Server { + return &Server{ + Path: path, + Port: strconv.Itoa(port), + } +} + +func (s *Server) Register(router *mux.Router, tracer Tracer) error { + router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") + s.tracer = tracer + return nil +} + +func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { + var trace Trace + for _, span := range spans { + + s := &Span{} + s.ID = strconv.FormatInt(span.GetID(), 10) + s.Annotations = UnmarshalAnnotations(span.GetAnnotations()) + s.Name = span.GetName() + if span.GetTimestamp() == 0 { + s.Timestamp = time.Now() + } else { + s.Timestamp = time.Unix(span.GetTimestamp(), 0) + } + + //duration, parent id, and trace id high are all optional fields. + // below, we check to see if any of these fields are non-zero, and if they are, + // we set the repsective fields in our Span structure to the address of + // these values. Otherwise, we just leave them as nil + + duration := span.GetDuration() + if duration != 0 { + s.Duration = &duration + } + + parentID := span.GetParentID() + if parentID != 0 { + s.ID += strconv.FormatInt(parentID, 10) + s.ParentID = &parentID + } + + traceIDHIGH := span.GetTraceIDHigh() + if traceIDHIGH != 0 { + s.TraceIDHigh = &traceIDHIGH + } + } + + return trace, nil +} + +func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { + var formatted []Annotation + for _, annotation := range annotations { + a := Annotation{} + endpoint := annotation.GetHost() + if endpoint != nil { + a.Host = strconv.Itoa(int(endpoint.GetIpv4())) + a.ServiceName = endpoint.GetServiceName() + } else { + a.Host, a.ServiceName = "", "" + } + + a.Timestamp = time.Unix(annotation.GetTimestamp(), 0) + a.Value = annotation.GetValue() + formatted = append(formatted, a) + } + return formatted +} + +func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { + log.Printf("Received request from: %s", r.URL.String()) + log.Printf("Raw request data is: %#+v", r) + defer r.Body.Close() + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + e := fmt.Errorf("Encoutered error: %s", err) + log.Println(e) + s.tracer.Error(e) + w.WriteHeader(http.StatusNoContent) + return + } + + buffer := thrift.NewTMemoryBuffer() + if _, err = buffer.Write(body); err != nil { + log.Println(err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + + transport := thrift.NewTBinaryProtocolTransport(buffer) + _, size, err := transport.ReadListBegin() + if err != nil { + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + + var spans []*zipkincore.Span + for i := 0; i < size; i++ { + zs := &zipkincore.Span{} + if err = zs.Read(transport); err != nil { + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + spans = append(spans, zs) + } + + err = transport.ReadListEnd() + if err != nil { + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + + //marshal json for debugging purposes + out, _ := json.MarshalIndent(spans, "", " ") + log.Println(string(out)) + + trace, err := UnmarshalZipkinResponse(spans) + if err != nil { + log.Println("Error: ", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + + if err = s.tracer.Record(trace); err != nil { + log.Println(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) } const sampleConfig = ` @@ -21,11 +218,7 @@ func (z Zipkin) SampleConfig() string { } func (z *Zipkin) Gather(acc telegraf.Accumulator) { - if z.Field { - acc.AddFields("state", map[string]interface{}{"value": "true"}, nil) - } else { - acc.AddFields("state", map[string]interface{}{"value": "false"}, nil) - } + } func (z *Zipkin) Start(acc telegraf.Accumulator) { From 76d49792a7ed51d3c5ae9f9ae6642f598f246af9 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 29 Jun 2017 17:24:14 -0700 Subject: [PATCH 018/154] Update zipkin.go to include server and and interfaces/method implementations necessary to unmarshal zipkin data --- plugins/inputs/zipkin/zipkin.go | 70 +++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index c1401b3c193a4..7c1d1657609f9 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -17,7 +17,11 @@ import ( ) type BinaryAnnotation struct { - //stuff + Key string + Value string + Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port + ServiceName string + Type string } type Annotation struct { @@ -28,13 +32,14 @@ type Annotation struct { } type Span struct { - ID string // zipkin traceid high concat with traceid - Name string - ParentID *int64 - Timestamp time.Time // If zipkin input is nil then time.Now() - Duration *int64 - TraceIDHigh *int64 - Annotations []Annotation + ID string // zipkin traceid high concat with traceid + Name string + ParentID *int64 + Timestamp time.Time // If zipkin input is nil then time.Now() + Duration *int64 + TraceIDHigh *int64 + Annotations []Annotation + BinaryAnnotations []BinaryAnnotation } type Trace []Span @@ -79,9 +84,15 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { var trace Trace for _, span := range spans { - s := &Span{} + s := Span{} s.ID = strconv.FormatInt(span.GetID(), 10) s.Annotations = UnmarshalAnnotations(span.GetAnnotations()) + + var err error + s.BinaryAnnotations, err = UnmarshalBinaryAnnotations(span.GetBinaryAnnotations()) + if err != nil { + return nil, err + } s.Name = span.GetName() if span.GetTimestamp() == 0 { s.Timestamp = time.Now() @@ -95,20 +106,25 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { // these values. Otherwise, we just leave them as nil duration := span.GetDuration() + fmt.Println("Duration: ", duration) if duration != 0 { s.Duration = &duration } parentID := span.GetParentID() + fmt.Println("Parent ID: ", parentID) if parentID != 0 { - s.ID += strconv.FormatInt(parentID, 10) s.ParentID = &parentID } traceIDHIGH := span.GetTraceIDHigh() + fmt.Println("Trace id high: ", traceIDHIGH) if traceIDHIGH != 0 { + s.ID += strconv.FormatInt(traceIDHIGH, 10) s.TraceIDHigh = &traceIDHIGH } + fmt.Println("ID:", s.ID) + trace = append(trace, s) } return trace, nil @@ -120,7 +136,7 @@ func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { a := Annotation{} endpoint := annotation.GetHost() if endpoint != nil { - a.Host = strconv.Itoa(int(endpoint.GetIpv4())) + a.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) a.ServiceName = endpoint.GetServiceName() } else { a.Host, a.ServiceName = "", "" @@ -130,9 +146,41 @@ func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { a.Value = annotation.GetValue() formatted = append(formatted, a) } + fmt.Println("formatted annotations: ", formatted) return formatted } +func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryAnnotation, error) { + var formatted []BinaryAnnotation + for _, annotation := range annotations { + b := BinaryAnnotation{} + endpoint := annotation.GetHost() + if endpoint != nil { + b.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) + b.ServiceName = endpoint.GetServiceName() + + fmt.Println("Binary Annotation host and service name: ", b.Host, b.ServiceName) + } else { + b.Host, b.ServiceName = "", "" + } + + val := annotation.GetValue() + /*log.Println("Value: ", string(val)) + dst := make([]byte, base64.StdEncoding.DecodedLen(len(val))) + _, err := base64.StdEncoding.Decode(dst, annotation.GetValue()) + if err != nil { + return nil, err + }*/ + + b.Key = annotation.GetKey() + b.Value = string(val) + b.Type = annotation.GetAnnotationType().String() + formatted = append(formatted, b) + } + + return formatted, nil +} + func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { log.Printf("Received request from: %s", r.URL.String()) log.Printf("Raw request data is: %#+v", r) From b8697b47be853e277343d6a4464d17e90d806a8c Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 29 Jun 2017 17:24:44 -0700 Subject: [PATCH 019/154] Add test for new unmarshalling features added to zipkin.go --- plugins/inputs/zipkin/new_server_test.go | 138 +++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 plugins/inputs/zipkin/new_server_test.go diff --git a/plugins/inputs/zipkin/new_server_test.go b/plugins/inputs/zipkin/new_server_test.go new file mode 100644 index 0000000000000..1db5b0e8db022 --- /dev/null +++ b/plugins/inputs/zipkin/new_server_test.go @@ -0,0 +1,138 @@ +package zipkin + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" +) + +type MockTracer struct { + Data Trace + Err error +} + +func (m *MockTracer) Record(t Trace) error { + fmt.Println("Adding trace ", t) + m.Data = t + return nil +} + +func (m *MockTracer) Error(err error) { + m.Err = err +} + +func TestZipkinServer(t *testing.T) { + dat, err := ioutil.ReadFile("test/threespans.dat") + if err != nil { + t.Fatal("Could not find file %s\n", "test/threespans.dat") + } + + s := NewServer("/api/v1/spans", 9411) + mockTracer := &MockTracer{} + s.tracer = mockTracer + w := httptest.NewRecorder() + r := httptest.NewRequest( + "POST", + "http://server.local/api/v1/spans", + ioutil.NopCloser( + bytes.NewReader(dat))) + handler := s.SpanHandler + handler(w, r) + if w.Code != http.StatusNoContent { + t.Errorf("MainHandler did not return StatusNoContent %d", w.Code) + } + + got := mockTracer.Data + + d := int64(53106) + d1 := int64(50410) + d2 := int64(103680) + parentID1 := int64(22964302721410078) + want := Trace{ + Span{ + Name: "Child", + ID: "8090652509916334619", + TraceIDHigh: nil, + ParentID: &parentID1, + Timestamp: time.Unix(1498688360851331, 0), + Duration: &d, + //note: []Annotation(nil) is different than + // []Annotation{} + Annotations: []Annotation(nil), + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "trivial", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, + Span{ + Name: "Child", + ID: "103618986556047333", + TraceIDHigh: nil, + ParentID: &parentID1, + Timestamp: time.Unix(1498688360904552, 0), + Duration: &d1, + Annotations: []Annotation(nil), + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "trivial", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, + Span{ + Name: "Parent", + ID: "22964302721410078", + Timestamp: time.Unix(1498688360851318, 0), + Duration: &d2, + Annotations: []Annotation{ + Annotation{ + Timestamp: time.Unix(1498688360851325, 0), + Value: "Starting child #0", + Host: "2130706433:0", + ServiceName: "trivial", + }, + Annotation{ + Timestamp: time.Unix(1498688360904545, 0), + Value: "Starting child #1", + Host: "2130706433:0", + ServiceName: "trivial", + }, + Annotation{ + Timestamp: time.Unix(1498688360954992, 0), + Value: "A Log", + Host: "2130706433:0", + ServiceName: "trivial", + }, + }, + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "trivial", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, + } + + fmt.Println("BINARY ANNOTATIONS FOR TESTING: ") + fmt.Println(got[0].BinaryAnnotations, want[0].BinaryAnnotations) + + if !reflect.DeepEqual(got, want) { + t.Fatal("Got != want, Fields weren't unmarshalled correctly") + } +} From bec51bf16d2f714175c64112c2981e0d5c8e04cd Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 29 Jun 2017 17:32:30 -0700 Subject: [PATCH 020/154] Rename main struct in server.go to avoidpackage level naming conflict in zipking plugin --- plugins/inputs/zipkin/server.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index 446e218a383e4..e2c3808176b88 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -25,7 +25,7 @@ const DefaultRoute = "/api/v1/spans" // Provides a shutdown timout in order to gracefully shutdown our http server const DefaultShutdownTimeout = 5 * time.Second -type Server struct { +type ZipkinServer struct { errorChan chan error dataChan chan SpanData Port string @@ -41,9 +41,9 @@ type SpanData []*zipkincore.Span // NewHTTPServer creates a new Zipkin http server given a port and a set of // channels -func NewHTTPServer(port int, e chan error, d chan SpanData) *Server { +func NewHTTPServer(port int, e chan error, d chan SpanData) *ZipkinServer { logger := log.New(os.Stdout, "", 0) - return &Server{ + return &ZipkinServer{ errorChan: e, dataChan: d, Port: strconv.Itoa(port), @@ -72,9 +72,9 @@ func Logger(next http.Handler) http.Handler { return http.HandlerFunc(fn) } -// MainHandler returns a closure with access to a *Server pointer -// for use as an http server handler -func (s *Server) MainHandler() http.Handler { +// MainHandler returns a closure with access to a *ZipkinServer pointer +// for use as an http ZipkinServer handler +func (s *ZipkinServer) MainHandler() http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { s.logger.Printf("Received request from: %s", r.URL.String()) @@ -131,7 +131,7 @@ func (s *Server) MainHandler() http.Handler { // Serve creates an internal http.Server and calls its ListenAndServe method to // start serving. It uses the MainHandler as the route handler. -func (s *Server) Serve() { +func (s *ZipkinServer) Serve() { mux := http.NewServeMux() mux.Handle(DefaultRoute, s.MainHandler()) @@ -145,7 +145,7 @@ func (s *Server) Serve() { } // Shutdown gracefully shuts down the internal http server -func (s *Server) Shutdown() { +func (s *ZipkinServer) Shutdown() { ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) defer cancel() @@ -213,7 +213,7 @@ func (s *Server) Shutdown() { }*/ // CloseAllChannels closes the Server's communication channels on the server's end. -func (s *Server) CloseAllChannels() { +func (s *ZipkinServer) CloseAllChannels() { log.Printf("Closing all communication channels...\n") close(s.dataChan) close(s.errorChan) From 38820502ed2edc9400295e76b77df31ddd255c0c Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 29 Jun 2017 17:45:27 -0700 Subject: [PATCH 021/154] fix formatting directive in new_server_test.go in zipking plugin --- plugins/inputs/zipkin/new_server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/new_server_test.go b/plugins/inputs/zipkin/new_server_test.go index 1db5b0e8db022..1cc174e379963 100644 --- a/plugins/inputs/zipkin/new_server_test.go +++ b/plugins/inputs/zipkin/new_server_test.go @@ -29,7 +29,7 @@ func (m *MockTracer) Error(err error) { func TestZipkinServer(t *testing.T) { dat, err := ioutil.ReadFile("test/threespans.dat") if err != nil { - t.Fatal("Could not find file %s\n", "test/threespans.dat") + t.Fatalf("Could not find file %s\n", "test/threespans.dat") } s := NewServer("/api/v1/spans", 9411) From 1c799ccbf72eeb29a3b15145e0cf1f62581a4d0a Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 12:46:21 -0700 Subject: [PATCH 022/154] Enable zipkin plugin in all.go --- plugins/inputs/all/all.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 0a68235b8e6f1..cdfe1fcce002c 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -90,4 +90,5 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters" _ "github.com/influxdata/telegraf/plugins/inputs/zfs" _ "github.com/influxdata/telegraf/plugins/inputs/zookeeper" + _ "github.com/influxdata/telegraf/plugins/inputs/zipkin" ) From 5a1d7adcf961abe40ca9e727ba4329d2bfb9f282 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 12:46:52 -0700 Subject: [PATCH 023/154] Delete outdated main.go file in zipkin plugin --- plugins/inputs/zipkin/cmd/main.go | 40 ------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 plugins/inputs/zipkin/cmd/main.go diff --git a/plugins/inputs/zipkin/cmd/main.go b/plugins/inputs/zipkin/cmd/main.go deleted file mode 100644 index 74ab92afe5cc1..0000000000000 --- a/plugins/inputs/zipkin/cmd/main.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "fmt" - "os" - "os/signal" - - "github.com/influxdata/telegraf/plugins/inputs/zipkin" -) - -func main() { - e := make(chan error) - d := make(chan zipkin.SpanData) - s := zipkin.NewHTTPServer(9411, e, d) - go s.Serve() - - sigChan := make(chan os.Signal) - signal.Notify(sigChan, os.Interrupt) - sigHandle(sigChan, s) - - for { - select { - case err := <-e: - fmt.Println("error: ", err) - case data := <-d: - fmt.Println("Got zipkin data: %#+v", data) - } - } - -} - -func sigHandle(c chan os.Signal, server *zipkin.Server) { - select { - case <-c: - fmt.Println("received SIGINT, stopping server") - //server.Done <- struct{}{} - server.Shutdown() - server.CloseAllChannels() - } -} From 9575625c3983ea8cf1c5afb8e283881ba4ea8865 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 12:47:30 -0700 Subject: [PATCH 024/154] Remove redeclared constant in server.go in zipkin plugin --- plugins/inputs/zipkin/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index e2c3808176b88..436befbf416a9 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -20,7 +20,7 @@ const version = "1.0" const port = "9411" // Zipkin writes to this route by default -const DefaultRoute = "/api/v1/spans" +//const DefaultRoute = "/api/v1/spans" // Provides a shutdown timout in order to gracefully shutdown our http server const DefaultShutdownTimeout = 5 * time.Second From 4672463ab4a4fb3c0c0dd632c3d0750e4a767026 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 12:49:45 -0700 Subject: [PATCH 025/154] Finish interface implementations in main zipkin plugin file --- plugins/inputs/zipkin/zipkin.go | 83 ++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 7c1d1657609f9..92cf019b8df5e 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -1,6 +1,7 @@ package zipkin import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -13,9 +14,13 @@ import ( "github.com/apache/thrift/lib/go/thrift" "github.com/gorilla/mux" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) +const DefaultPort = 9411 +const DefaultRoute = "/api/v1/spans" + type BinaryAnnotation struct { Key string Value string @@ -50,14 +55,16 @@ type Tracer interface { } type Service interface { - Register(router *mux.Router, tracer Tracer) + Register(router *mux.Router, tracer Tracer) error } type Zipkin struct { ServiceAddress string + Port int Path string tracing Service server *http.Server + waitGroup *sync.WaitGroup } type Server struct { @@ -254,7 +261,8 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { const sampleConfig = ` ## - # field = value + # path = /path/your/zipkin/impl/posts/to + # port = ` func (z Zipkin) Description() string { @@ -265,14 +273,75 @@ func (z Zipkin) SampleConfig() string { return sampleConfig } -func (z *Zipkin) Gather(acc telegraf.Accumulator) { +func (z *Zipkin) Gather(acc telegraf.Accumulator) error { return nil } -} +func (z *Zipkin) Start(acc telegraf.Accumulator) error { + if z.tracing == nil { + t := NewServer(z.Path, z.Port) + z.tracing = t + } + + var wg sync.WaitGroup + z.waitGroup = &wg -func (z *Zipkin) Start(acc telegraf.Accumulator) { - //start collecting data + go func() { + wg.Add(1) + defer wg.Done() + + z.Listen(acc) + }() + + return nil } func (z *Zipkin) Stop() { - //clean up any channels, etc. + ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) + defer cancel() + + defer z.waitGroup.Wait() + z.server.Shutdown(ctx) +} + +type LineProtocolConverter struct { + acc telegraf.Accumulator +} + +func (l *LineProtocolConverter) Record(t Trace) error { + log.Printf("received trace: %#+v\n", t) + return nil +} + +func (l *LineProtocolConverter) Error(err error) { + +} + +func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { + return &LineProtocolConverter{ + acc: acc, + } +} + +func (z *Zipkin) Listen(acc telegraf.Accumulator) { + r := mux.NewRouter() + converter := NewLineProtocolConverter(acc) + z.tracing.Register(r, converter) + + if z.server == nil { + z.server = &http.Server{ + Addr: ":" + strconv.Itoa(z.Port), + Handler: r, + } + } + if err := z.server.ListenAndServe(); err != nil { + acc.AddError(fmt.Errorf("E! Error listening: %v\n", err)) + } +} + +func init() { + inputs.Add("zipkin", func() telegraf.Input { + return &Zipkin{ + Path: DefaultRoute, + Port: DefaultPort, + } + }) } From 438e710692a1b24e9a042603c14c3d58d25bc04e Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 12:52:19 -0700 Subject: [PATCH 026/154] Rename test data directory --- .../inputs/zipkin/{test => testdata}/threespans.dat | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/{test => testdata}/threespans.dat (100%) diff --git a/plugins/inputs/zipkin/test/threespans.dat b/plugins/inputs/zipkin/testdata/threespans.dat similarity index 100% rename from plugins/inputs/zipkin/test/threespans.dat rename to plugins/inputs/zipkin/testdata/threespans.dat From 9239d640f337b96f9d65fce300b27f3ab9c54fc2 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 14:08:11 -0700 Subject: [PATCH 027/154] Re-order method and type declaration in zipkin.go and add Godoc comments --- plugins/inputs/zipkin/zipkin.go | 182 +++++++++++++++++++++----------- 1 file changed, 119 insertions(+), 63 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 92cf019b8df5e..3e2d58d8739d3 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -18,9 +18,31 @@ import ( "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) -const DefaultPort = 9411 -const DefaultRoute = "/api/v1/spans" +const ( + // DefaultPort is the default port zipkin listens on, which zipkin implementations + // expect. + DefaultPort = 9411 + + // DefaultRoute is the default route zipkin uses, and zipkin implementations + // expect. + DefaultRoute = "/api/v1/spans" +) + +// Tracer represents a type which can record zipkin trace data as well as +// any accompanying errors, and process that data. +type Tracer interface { + Record(Trace) error + Error(error) +} +// Service represents a type which can register itself with a router for +// http routing, and a Tracer for trace data collection. +type Service interface { + Register(router *mux.Router, tracer Tracer) error +} + +// BinaryAnnotation represents a zipkin binary annotation. It contains +// all of the same fields as might be found in its zipkin counterpart. type BinaryAnnotation struct { Key string Value string @@ -29,6 +51,8 @@ type BinaryAnnotation struct { Type string } +// Annotation represents an ordinary zipkin annotation. It contains the data fields +// which will become fields/tags in influxdb type Annotation struct { Timestamp time.Time Value string @@ -36,6 +60,9 @@ type Annotation struct { ServiceName string } +//Span represents a specific zipkin span. It holds the majority of the same +// data as a zipkin span sent via the thrift protocol, but is presented in a +// format which is more straightforward for storage purposes. type Span struct { ID string // zipkin traceid high concat with traceid Name string @@ -47,46 +74,34 @@ type Span struct { BinaryAnnotations []BinaryAnnotation } +// Trace is an array (or a series) of spans type Trace []Span -type Tracer interface { - Record(Trace) error - Error(error) -} - -type Service interface { - Register(router *mux.Router, tracer Tracer) error -} - -type Zipkin struct { - ServiceAddress string - Port int - Path string - tracing Service - server *http.Server - waitGroup *sync.WaitGroup -} - +// Server is an implementation of tracer which is a helper for running an +// http server which accepts zipkin requests type Server struct { Path string - Port string tracer Tracer waitGroup *sync.WaitGroup } -func NewServer(path string, port int) *Server { +//NewServer returns a new server instance given path to handle +func NewServer(path string) *Server { return &Server{ Path: path, - Port: strconv.Itoa(port), } } +// Register allows server to implement the Service interface. Server's register metod +// registers its handler on mux, and sets the servers tracer with tracer func (s *Server) Register(router *mux.Router, tracer Tracer) error { router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") s.tracer = tracer return nil } +//UnmarshalZipkinResponse is a helper method for unmarhsalling a slice of []*zipkincore.Spans +// into a Trace (technically a []Span) func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { var trace Trace for _, span := range spans { @@ -137,6 +152,8 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { return trace, nil } +// UnmarshalAnnotations is a helper method for unmarshalling a slice of +// *zipkincore.Annotation into a slice of Annotations func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { var formatted []Annotation for _, annotation := range annotations { @@ -157,6 +174,8 @@ func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { return formatted } +// UnmarshalBinaryAnnotations is very similar to UnmarshalAnnotations, but it +// Unmarshalls zipkincore.BinaryAnnotations instead of the normal zipkincore.Annotation func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryAnnotation, error) { var formatted []BinaryAnnotation for _, annotation := range annotations { @@ -188,6 +207,7 @@ func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]B return formatted, nil } +// SpanHandler is the handler Server uses for handling zipkin POST requests func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { log.Printf("Received request from: %s", r.URL.String()) log.Printf("Raw request data is: %#+v", r) @@ -259,25 +279,95 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } +type LineProtocolConverter struct { + acc telegraf.Accumulator +} + +func (l *LineProtocolConverter) Record(t Trace) error { + log.Printf("received trace: %#+v\n", t) + log.Printf("...But converter implementation is not yet done. Here's some example data") + + fields := map[string]interface{}{ + "Duration": "1060", + "Timestamp": time.Unix(1498852876, 0), + "Annotations": []string{"An annotation"}, + "binary_annotations": []string{"A binary annotation"}, + } + + tags := map[string]string{ + "host": "http://hostname.com", + "port": "5555", + } + + l.acc.AddFields("zipkin", fields, tags) + return nil +} + +func (l *LineProtocolConverter) Error(err error) { + +} + +func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { + return &LineProtocolConverter{ + acc: acc, + } +} + +// Zipkin is a telegraf configuration structure for the zipkin input plugin, +// but it also contains fields for the management of a separate, concurrent +// zipkin http server +type Zipkin struct { + ServiceAddress string + Port int + Path string + tracing Service + server *http.Server + waitGroup *sync.WaitGroup +} + +// Description: necessary method implementation from telegraf.ServiceInput +func (z Zipkin) Description() string { + return "Allows for the collection of zipkin tracing spans for storage in influxdb" +} + const sampleConfig = ` ## # path = /path/your/zipkin/impl/posts/to # port = ` -func (z Zipkin) Description() string { - return "Allows for the collection of zipkin tracing spans for storage in influxdb" -} - +// SampleConfig: necessary method implementation from telegraf.ServiceInput func (z Zipkin) SampleConfig() string { return sampleConfig } +// Gather is empty for the zipkin plugin; all gathering is done through +// the separate goroutine launched in (*Zipkin).Start() func (z *Zipkin) Gather(acc telegraf.Accumulator) error { return nil } +// Listen creates an http server on the zipkin instance it is called with, and +// serves http until it is stopped by Zipkin's (*Zipkin).Stop() method. +func (z *Zipkin) Listen(acc telegraf.Accumulator) { + r := mux.NewRouter() + converter := NewLineProtocolConverter(acc) + z.tracing.Register(r, converter) + + if z.server == nil { + z.server = &http.Server{ + Addr: ":" + strconv.Itoa(z.Port), + Handler: r, + } + } + if err := z.server.ListenAndServe(); err != nil { + acc.AddError(fmt.Errorf("E! Error listening: %v\n", err)) + } +} + +// Start launches a separate goroutine for collecting zipkin client http requests, +// passing in a telegraf.Accumulator such that data can be collected. func (z *Zipkin) Start(acc telegraf.Accumulator) error { if z.tracing == nil { - t := NewServer(z.Path, z.Port) + t := NewServer(z.Path) z.tracing = t } @@ -294,6 +384,7 @@ func (z *Zipkin) Start(acc telegraf.Accumulator) error { return nil } +// Stop shuts the internal http server down with via context.Context func (z *Zipkin) Stop() { ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) defer cancel() @@ -302,41 +393,6 @@ func (z *Zipkin) Stop() { z.server.Shutdown(ctx) } -type LineProtocolConverter struct { - acc telegraf.Accumulator -} - -func (l *LineProtocolConverter) Record(t Trace) error { - log.Printf("received trace: %#+v\n", t) - return nil -} - -func (l *LineProtocolConverter) Error(err error) { - -} - -func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { - return &LineProtocolConverter{ - acc: acc, - } -} - -func (z *Zipkin) Listen(acc telegraf.Accumulator) { - r := mux.NewRouter() - converter := NewLineProtocolConverter(acc) - z.tracing.Register(r, converter) - - if z.server == nil { - z.server = &http.Server{ - Addr: ":" + strconv.Itoa(z.Port), - Handler: r, - } - } - if err := z.server.ListenAndServe(); err != nil { - acc.AddError(fmt.Errorf("E! Error listening: %v\n", err)) - } -} - func init() { inputs.Add("zipkin", func() telegraf.Input { return &Zipkin{ From e96bfba1c5a745f686dcbb26713b8e395e8e0145 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 14:08:48 -0700 Subject: [PATCH 028/154] Update data file path in new_server_test.go --- plugins/inputs/zipkin/new_server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/new_server_test.go b/plugins/inputs/zipkin/new_server_test.go index 1cc174e379963..2ef1e198a0f57 100644 --- a/plugins/inputs/zipkin/new_server_test.go +++ b/plugins/inputs/zipkin/new_server_test.go @@ -27,12 +27,12 @@ func (m *MockTracer) Error(err error) { } func TestZipkinServer(t *testing.T) { - dat, err := ioutil.ReadFile("test/threespans.dat") + dat, err := ioutil.ReadFile("testdata/threespans.dat") if err != nil { t.Fatalf("Could not find file %s\n", "test/threespans.dat") } - s := NewServer("/api/v1/spans", 9411) + s := NewServer("/api/v1/spans") mockTracer := &MockTracer{} s.tracer = mockTracer w := httptest.NewRecorder() From f2adbe99de456f4bce3b90152dc7cf0e9f83cafd Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 16:50:39 -0700 Subject: [PATCH 029/154] Add todo comments to zipkin.go in zipkin plugin --- plugins/inputs/zipkin/zipkin.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 3e2d58d8739d3..3a75381fb3b12 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -28,6 +28,10 @@ const ( DefaultRoute = "/api/v1/spans" ) +// TODO: connect methods lexically; method implementations should go right under +// struct definition. Maybe change order of structs, organize where structs are +// declared based on when their type is used + // Tracer represents a type which can record zipkin trace data as well as // any accompanying errors, and process that data. type Tracer interface { @@ -95,6 +99,7 @@ func NewServer(path string) *Server { // Register allows server to implement the Service interface. Server's register metod // registers its handler on mux, and sets the servers tracer with tracer func (s *Server) Register(router *mux.Router, tracer Tracer) error { + // TODO: potentially move router into Server if appropriate router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") s.tracer = tracer return nil @@ -215,9 +220,10 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - e := fmt.Errorf("Encoutered error: %s", err) + e := fmt.Errorf("Encountered error: %s", err) log.Println(e) s.tracer.Error(e) + //TODO: Change http status that is sent back to client w.WriteHeader(http.StatusNoContent) return } @@ -327,7 +333,7 @@ type Zipkin struct { // Description: necessary method implementation from telegraf.ServiceInput func (z Zipkin) Description() string { - return "Allows for the collection of zipkin tracing spans for storage in influxdb" + return "Allows for the collection of zipkin tracing spans for storage in InfluxDB" } const sampleConfig = ` From 4ee9246d10d76f9ac9b1c932db534ca6e36139e3 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 30 Jun 2017 16:51:24 -0700 Subject: [PATCH 030/154] Disable server_test.go for current testing in zipkin plugin --- plugins/inputs/zipkin/server_test.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/plugins/inputs/zipkin/server_test.go b/plugins/inputs/zipkin/server_test.go index 370ad3a2bca9d..eba9214e2262e 100644 --- a/plugins/inputs/zipkin/server_test.go +++ b/plugins/inputs/zipkin/server_test.go @@ -1,14 +1,6 @@ package zipkin -import ( - "bytes" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" -) - -func TestMainHandler(t *testing.T) { +/*func TestMainHandler(t *testing.T) { dat, err := ioutil.ReadFile("test/threespans.dat") if err != nil { t.Fatal("Could not find threespans.dat") @@ -57,4 +49,4 @@ func TestMainHandler(t *testing.T) { if spans[0].GetDuration() != 53106 { t.Errorf("Expected duration %d but recieved duration %d", 53106, spans[0].GetDuration()) } -} +}*/ From 401786372b3f87eaeaec37e25edd267b80bb0548 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 5 Jul 2017 10:52:47 -0700 Subject: [PATCH 031/154] Refactor zipkin server implementation into multiple files --- plugins/inputs/zipkin/server_new.go | 111 ++++++++++++++++++++ plugins/inputs/zipkin/zipkin.go | 152 +++++----------------------- 2 files changed, 137 insertions(+), 126 deletions(-) create mode 100644 plugins/inputs/zipkin/server_new.go diff --git a/plugins/inputs/zipkin/server_new.go b/plugins/inputs/zipkin/server_new.go new file mode 100644 index 0000000000000..70f705b37bbc4 --- /dev/null +++ b/plugins/inputs/zipkin/server_new.go @@ -0,0 +1,111 @@ +package zipkin + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "sync" + + "github.com/apache/thrift/lib/go/thrift" + "github.com/gorilla/mux" + "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" +) + +// Server is an implementation of tracer which is a helper for running an +// http server which accepts zipkin requests +type Server struct { + Path string + tracer Tracer + waitGroup *sync.WaitGroup +} + +// Register allows server to implement the Service interface. Server's register metod +// registers its handler on mux, and sets the servers tracer with tracer +func (s *Server) Register(router *mux.Router, tracer Tracer) error { + // TODO: potentially move router into Server if appropriate + router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") + s.tracer = tracer + return nil +} + +// SpanHandler is the handler Server uses for handling zipkin POST requests +func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { + log.Printf("Received request from: %s", r.URL.String()) + log.Printf("Raw request data is: %#+v", r) + defer r.Body.Close() + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + e := fmt.Errorf("Encountered error: %s", err) + log.Println(e) + s.tracer.Error(e) + //TODO: Change http status that is sent back to client + w.WriteHeader(http.StatusNoContent) + return + } + + buffer := thrift.NewTMemoryBuffer() + if _, err = buffer.Write(body); err != nil { + log.Println(err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + + transport := thrift.NewTBinaryProtocolTransport(buffer) + _, size, err := transport.ReadListBegin() + if err != nil { + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + + var spans []*zipkincore.Span + for i := 0; i < size; i++ { + zs := &zipkincore.Span{} + if err = zs.Read(transport); err != nil { + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + spans = append(spans, zs) + } + + err = transport.ReadListEnd() + if err != nil { + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + + //marshal json for debugging purposes + out, _ := json.MarshalIndent(spans, "", " ") + log.Println(string(out)) + + trace, err := UnmarshalZipkinResponse(spans) + if err != nil { + log.Println("Error: ", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusNoContent) + return + } + + if err = s.tracer.Record(trace); err != nil { + log.Println(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) +} + +//NewServer returns a new server instance given path to handle +func NewServer(path string) *Server { + return &Server{ + Path: path, + } +} diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 3a75381fb3b12..e5e59402b31e5 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -2,16 +2,13 @@ package zipkin import ( "context" - "encoding/json" "fmt" - "io/ioutil" "log" "net/http" "strconv" "sync" "time" - "github.com/apache/thrift/lib/go/thrift" "github.com/gorilla/mux" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" @@ -81,30 +78,6 @@ type Span struct { // Trace is an array (or a series) of spans type Trace []Span -// Server is an implementation of tracer which is a helper for running an -// http server which accepts zipkin requests -type Server struct { - Path string - tracer Tracer - waitGroup *sync.WaitGroup -} - -//NewServer returns a new server instance given path to handle -func NewServer(path string) *Server { - return &Server{ - Path: path, - } -} - -// Register allows server to implement the Service interface. Server's register metod -// registers its handler on mux, and sets the servers tracer with tracer -func (s *Server) Register(router *mux.Router, tracer Tracer) error { - // TODO: potentially move router into Server if appropriate - router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") - s.tracer = tracer - return nil -} - //UnmarshalZipkinResponse is a helper method for unmarhsalling a slice of []*zipkincore.Spans // into a Trace (technically a []Span) func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { @@ -212,79 +185,6 @@ func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]B return formatted, nil } -// SpanHandler is the handler Server uses for handling zipkin POST requests -func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { - log.Printf("Received request from: %s", r.URL.String()) - log.Printf("Raw request data is: %#+v", r) - defer r.Body.Close() - - body, err := ioutil.ReadAll(r.Body) - if err != nil { - e := fmt.Errorf("Encountered error: %s", err) - log.Println(e) - s.tracer.Error(e) - //TODO: Change http status that is sent back to client - w.WriteHeader(http.StatusNoContent) - return - } - - buffer := thrift.NewTMemoryBuffer() - if _, err = buffer.Write(body); err != nil { - log.Println(err) - s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) - return - } - - transport := thrift.NewTBinaryProtocolTransport(buffer) - _, size, err := transport.ReadListBegin() - if err != nil { - log.Printf("%s", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) - return - } - - var spans []*zipkincore.Span - for i := 0; i < size; i++ { - zs := &zipkincore.Span{} - if err = zs.Read(transport); err != nil { - log.Printf("%s", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) - return - } - spans = append(spans, zs) - } - - err = transport.ReadListEnd() - if err != nil { - log.Printf("%s", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) - return - } - - //marshal json for debugging purposes - out, _ := json.MarshalIndent(spans, "", " ") - log.Println(string(out)) - - trace, err := UnmarshalZipkinResponse(spans) - if err != nil { - log.Println("Error: ", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) - return - } - - if err = s.tracer.Record(trace); err != nil { - log.Println(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - w.WriteHeader(http.StatusNoContent) -} - type LineProtocolConverter struct { acc telegraf.Accumulator } @@ -319,6 +219,12 @@ func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { } } +const sampleConfig = ` + ## + # path = /path/your/zipkin/impl/posts/to + # port = +` + // Zipkin is a telegraf configuration structure for the zipkin input plugin, // but it also contains fields for the management of a separate, concurrent // zipkin http server @@ -331,18 +237,12 @@ type Zipkin struct { waitGroup *sync.WaitGroup } -// Description: necessary method implementation from telegraf.ServiceInput +// Description is a necessary method implementation from telegraf.ServiceInput func (z Zipkin) Description() string { return "Allows for the collection of zipkin tracing spans for storage in InfluxDB" } -const sampleConfig = ` - ## - # path = /path/your/zipkin/impl/posts/to - # port = -` - -// SampleConfig: necessary method implementation from telegraf.ServiceInput +// SampleConfig is a necessary method implementation from telegraf.ServiceInput func (z Zipkin) SampleConfig() string { return sampleConfig } @@ -351,24 +251,6 @@ func (z Zipkin) SampleConfig() string { // the separate goroutine launched in (*Zipkin).Start() func (z *Zipkin) Gather(acc telegraf.Accumulator) error { return nil } -// Listen creates an http server on the zipkin instance it is called with, and -// serves http until it is stopped by Zipkin's (*Zipkin).Stop() method. -func (z *Zipkin) Listen(acc telegraf.Accumulator) { - r := mux.NewRouter() - converter := NewLineProtocolConverter(acc) - z.tracing.Register(r, converter) - - if z.server == nil { - z.server = &http.Server{ - Addr: ":" + strconv.Itoa(z.Port), - Handler: r, - } - } - if err := z.server.ListenAndServe(); err != nil { - acc.AddError(fmt.Errorf("E! Error listening: %v\n", err)) - } -} - // Start launches a separate goroutine for collecting zipkin client http requests, // passing in a telegraf.Accumulator such that data can be collected. func (z *Zipkin) Start(acc telegraf.Accumulator) error { @@ -390,6 +272,24 @@ func (z *Zipkin) Start(acc telegraf.Accumulator) error { return nil } +// Listen creates an http server on the zipkin instance it is called with, and +// serves http until it is stopped by Zipkin's (*Zipkin).Stop() method. +func (z *Zipkin) Listen(acc telegraf.Accumulator) { + r := mux.NewRouter() + converter := NewLineProtocolConverter(acc) + z.tracing.Register(r, converter) + + if z.server == nil { + z.server = &http.Server{ + Addr: ":" + strconv.Itoa(z.Port), + Handler: r, + } + } + if err := z.server.ListenAndServe(); err != nil { + acc.AddError(fmt.Errorf("E! Error listening: %v", err)) + } +} + // Stop shuts the internal http server down with via context.Context func (z *Zipkin) Stop() { ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) From 7b60eef93a2063b2d058373822129af210548a65 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 5 Jul 2017 10:55:27 -0700 Subject: [PATCH 032/154] Change http header status written on error condition in zipkin server http handler --- plugins/inputs/zipkin/server_new.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/inputs/zipkin/server_new.go b/plugins/inputs/zipkin/server_new.go index 70f705b37bbc4..ea21910939e27 100644 --- a/plugins/inputs/zipkin/server_new.go +++ b/plugins/inputs/zipkin/server_new.go @@ -42,7 +42,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { log.Println(e) s.tracer.Error(e) //TODO: Change http status that is sent back to client - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(http.StatusInternalServerError) return } @@ -50,7 +50,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { if _, err = buffer.Write(body); err != nil { log.Println(err) s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(http.StatusInternalServerError) return } @@ -59,7 +59,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { if err != nil { log.Printf("%s", err) s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(http.StatusInternalServerError) return } @@ -69,7 +69,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { if err = zs.Read(transport); err != nil { log.Printf("%s", err) s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(http.StatusInternalServerError) return } spans = append(spans, zs) @@ -79,7 +79,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { if err != nil { log.Printf("%s", err) s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(http.StatusInternalServerError) return } @@ -91,7 +91,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { if err != nil { log.Println("Error: ", err) s.tracer.Error(err) - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(http.StatusInternalServerError) return } From 9f4a8375848f3e4b0c758c440794922b548d501c Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 5 Jul 2017 17:16:48 -0700 Subject: [PATCH 033/154] Add initial telegraf conversion code to zipkin.go --- plugins/inputs/zipkin/zipkin.go | 116 ++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 14 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index e5e59402b31e5..5dff3a9ffe24f 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -191,10 +191,12 @@ type LineProtocolConverter struct { func (l *LineProtocolConverter) Record(t Trace) error { log.Printf("received trace: %#+v\n", t) - log.Printf("...But converter implementation is not yet done. Here's some example data") + //log.Printf("...But converter implementation is not yet done. Here's some example data") + log.Printf("Writing to telegraf...\n") - fields := map[string]interface{}{ - "Duration": "1060", + // Example fields and tags + /*fields := map[string]interface{}{ + "Duration": "1060", // "Timestamp": time.Unix(1498852876, 0), "Annotations": []string{"An annotation"}, "binary_annotations": []string{"A binary annotation"}, @@ -203,14 +205,100 @@ func (l *LineProtocolConverter) Record(t Trace) error { tags := map[string]string{ "host": "http://hostname.com", "port": "5555", + }*/ + //l.acc.AddFields("zipkin", fields, tags) + /* type Span struct { + ID string // zipkin traceid high concat with traceid Done + Name string Done + ParentID *int64 Done + Timestamp time.Time // If zipkin input is nil then time.Now() Done + Duration *int64 Done + TraceIDHigh *int64 Don't worry about + Annotations []Annotation + BinaryAnnotations []BinaryAnnotation + } + + */ + + /* + type BinaryAnnotation struct { + Key string + Value string + Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port + ServiceName string + Type string + } + */ + + for _, s := range t { + //Do some conversion + + fields := map[string]interface{}{ + "timestamp": s.Timestamp.Unix(), + } + + if s.Duration != nil { + fields["Duration"] = *s.Duration + log.Printf("Duration is: %d", *s.Duration) + } + + tags := map[string]string{ + "id": s.ID, + "name": s.Name, + } + + if s.ParentID == nil { + tags["parent_id"] = s.ID + } else { + tags["parent_id"] = strconv.Itoa(int(*s.ParentID)) + } + l.acc.AddFields("zipkin", fields, tags) + + /* + type Annotation struct { + Timestamp time.Time + Value string + Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port + ServiceName string + } + + */ + for _, a := range s.Annotations { + tags = map[string]string{ + "span_id": s.ID, + "host": a.Host, + "service_name": a.ServiceName, + } + + fields = map[string]interface{}{ + "timestamp": a.Timestamp.Unix(), + "value": a.Value, + } + l.acc.AddFields("zipkin_annotations", fields, tags) + } + + for _, b := range s.BinaryAnnotations { + tags = map[string]string{ + "annotation_type": "binary", + "host": b.Host, + "service_name": b.ServiceName, + "type": b.Type, + "span_id": s.ID, + } + + fields = map[string]interface{}{ + "value": b.Value, + "key": b.Key, + } + l.acc.AddFields("zipkin_binary_annotations", fields, tags) + } } - l.acc.AddFields("zipkin", fields, tags) return nil } func (l *LineProtocolConverter) Error(err error) { - + l.acc.AddError(err) } func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { @@ -272,6 +360,15 @@ func (z *Zipkin) Start(acc telegraf.Accumulator) error { return nil } +// Stop shuts the internal http server down with via context.Context +func (z *Zipkin) Stop() { + ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) + defer cancel() + + defer z.waitGroup.Wait() + z.server.Shutdown(ctx) +} + // Listen creates an http server on the zipkin instance it is called with, and // serves http until it is stopped by Zipkin's (*Zipkin).Stop() method. func (z *Zipkin) Listen(acc telegraf.Accumulator) { @@ -290,15 +387,6 @@ func (z *Zipkin) Listen(acc telegraf.Accumulator) { } } -// Stop shuts the internal http server down with via context.Context -func (z *Zipkin) Stop() { - ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) - defer cancel() - - defer z.waitGroup.Wait() - z.server.Shutdown(ctx) -} - func init() { inputs.Add("zipkin", func() telegraf.Input { return &Zipkin{ From a6ed636b2fba70858d969d4d46f8159111ad017d Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 7 Jul 2017 15:49:14 -0700 Subject: [PATCH 034/154] Update zipkin plugin line protocol schema --- plugins/inputs/zipkin/zipkin.go | 266 +++++++++++++++++++++++--------- 1 file changed, 193 insertions(+), 73 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 5dff3a9ffe24f..a5877d1f519b9 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -52,6 +52,16 @@ type BinaryAnnotation struct { Type string } +func (b BinaryAnnotation) ToMeta() MetaAnnotation { + return MetaAnnotation{ + Key: b.Key, + Value: b.Value, + Host: b.Host, + ServiceName: b.ServiceName, + Type: b.Type, + } +} + // Annotation represents an ordinary zipkin annotation. It contains the data fields // which will become fields/tags in influxdb type Annotation struct { @@ -61,16 +71,37 @@ type Annotation struct { ServiceName string } +func (a Annotation) ToMeta() MetaAnnotation { + return MetaAnnotation{ + Key: a.Value, + Value: "NONE", + Type: "NONE", + Timestamp: a.Timestamp, + Host: a.Host, + ServiceName: a.ServiceName, + } +} + +type MetaAnnotation struct { + Key string + Value string + Type string + Timestamp time.Time + Host string + HostIPV6 string + ServiceName string +} + //Span represents a specific zipkin span. It holds the majority of the same // data as a zipkin span sent via the thrift protocol, but is presented in a // format which is more straightforward for storage purposes. type Span struct { - ID string // zipkin traceid high concat with traceid + ID string + TraceID string // zipkin traceid high concat with traceid Name string - ParentID *int64 + ParentID string Timestamp time.Time // If zipkin input is nil then time.Now() - Duration *int64 - TraceIDHigh *int64 + Duration time.Duration Annotations []Annotation BinaryAnnotations []BinaryAnnotation } @@ -86,6 +117,8 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { s := Span{} s.ID = strconv.FormatInt(span.GetID(), 10) + s.TraceID = strconv.FormatInt(span.GetTraceIDHigh(), 10) + strconv.FormatInt(span.GetTraceID(), 10) + s.Annotations = UnmarshalAnnotations(span.GetAnnotations()) var err error @@ -97,7 +130,7 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { if span.GetTimestamp() == 0 { s.Timestamp = time.Now() } else { - s.Timestamp = time.Unix(span.GetTimestamp(), 0) + s.Timestamp = time.Unix(0, span.GetTimestamp()*int64(time.Microsecond)) } //duration, parent id, and trace id high are all optional fields. @@ -105,24 +138,18 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { // we set the repsective fields in our Span structure to the address of // these values. Otherwise, we just leave them as nil - duration := span.GetDuration() + duration := time.Duration(span.GetDuration()) fmt.Println("Duration: ", duration) - if duration != 0 { - s.Duration = &duration - } + s.Duration = duration * time.Microsecond parentID := span.GetParentID() fmt.Println("Parent ID: ", parentID) - if parentID != 0 { - s.ParentID = &parentID + if parentID == 0 { + s.ParentID = s.ID + } else { + s.ParentID = strconv.FormatInt(parentID, 10) } - traceIDHIGH := span.GetTraceIDHigh() - fmt.Println("Trace id high: ", traceIDHIGH) - if traceIDHIGH != 0 { - s.ID += strconv.FormatInt(traceIDHIGH, 10) - s.TraceIDHigh = &traceIDHIGH - } fmt.Println("ID:", s.ID) trace = append(trace, s) } @@ -152,6 +179,9 @@ func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { return formatted } +func (s Span) MergeAnnotations() { +} + // UnmarshalBinaryAnnotations is very similar to UnmarshalAnnotations, but it // Unmarshalls zipkincore.BinaryAnnotations instead of the normal zipkincore.Annotation func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryAnnotation, error) { @@ -189,70 +219,122 @@ type LineProtocolConverter struct { acc telegraf.Accumulator } -func (l *LineProtocolConverter) Record(t Trace) error { - log.Printf("received trace: %#+v\n", t) - //log.Printf("...But converter implementation is not yet done. Here's some example data") - log.Printf("Writing to telegraf...\n") +/*func (l *LineProtocolConverter) Record(t Trace) error { +log.Printf("received trace: %#+v\n", t) +//log.Printf("...But converter implementation is not yet done. Here's some example data") +log.Printf("Writing to telegraf...\n") + +// Example fields and tags +/*fields := map[string]interface{}{ + "Duration": "1060", // + "Timestamp": time.Unix(1498852876, 0), + "Annotations": []string{"An annotation"}, + "binary_annotations": []string{"A binary annotation"}, +} + +tags := map[string]string{ + "host": "http://hostname.com", + "port": "5555", +}*/ +//l.acc.AddFields("zipkin", fields, tags) +/* type Span struct { + ID string // zipkin traceid high concat with traceid Done + Name string Done + ParentID *int64 Done + Timestamp time.Time // If zipkin input is nil then time.Now() Done + Duration *int64 Done + TraceIDHigh *int64 Don't worry about + Annotations []Annotation + BinaryAnnotations []BinaryAnnotation +} - // Example fields and tags - /*fields := map[string]interface{}{ - "Duration": "1060", // - "Timestamp": time.Unix(1498852876, 0), - "Annotations": []string{"An annotation"}, - "binary_annotations": []string{"A binary annotation"}, +*/ + +/* + type BinaryAnnotation struct { + Key string + Value string + Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port + ServiceName string + Type string } +*/ + +/*for _, s := range t { +//Do some conversion + +fields := map[string]interface{}{ + "timestamp": s.Timestamp.Unix(), +} + +if s.Duration != nil { + fields["Duration"] = *s.Duration + log.Printf("Duration is: %d", *s.Duration) +} + +tags := map[string]string{ + "id": s.ID, + "name": s.Name, +} - tags := map[string]string{ - "host": "http://hostname.com", - "port": "5555", - }*/ - //l.acc.AddFields("zipkin", fields, tags) - /* type Span struct { - ID string // zipkin traceid high concat with traceid Done - Name string Done - ParentID *int64 Done - Timestamp time.Time // If zipkin input is nil then time.Now() Done - Duration *int64 Done - TraceIDHigh *int64 Don't worry about - Annotations []Annotation - BinaryAnnotations []BinaryAnnotation +if s.ParentID == nil { + tags["parent_id"] = s.ID +} else { + tags["parent_id"] = strconv.Itoa(int(*s.ParentID)) +} +l.acc.AddFields("zipkin", fields, tags) + +/* + type Annotation struct { + Timestamp time.Time + Value string + Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port + ServiceName string } - */ +*/ +/* for _, a := range s.Annotations { + tags = map[string]string{ + "span_id": s.ID, + "host": a.Host, + "service_name": a.ServiceName, + } - /* - type BinaryAnnotation struct { - Key string - Value string - Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port - ServiceName string - Type string + fields = map[string]interface{}{ + "timestamp": a.Timestamp.Unix(), + "value": a.Value, + } + l.acc.AddFields("zipkin_annotations", fields, tags) } - */ - for _, s := range t { - //Do some conversion + for _, b := range s.BinaryAnnotations { + tags = map[string]string{ + "annotation_type": "binary", + "host": b.Host, + "service_name": b.ServiceName, + "type": b.Type, + "span_id": s.ID, + } - fields := map[string]interface{}{ - "timestamp": s.Timestamp.Unix(), + fields = map[string]interface{}{ + "value": b.Value, + "key": b.Key, + } + l.acc.AddFields("zipkin_binary_annotations", fields, tags) } + } - if s.Duration != nil { - fields["Duration"] = *s.Duration - log.Printf("Duration is: %d", *s.Duration) - } + return nil +}*/ - tags := map[string]string{ - "id": s.ID, - "name": s.Name, - } +func (l *LineProtocolConverter) Record(t Trace) error { + log.Printf("received trace: %#+v\n", t) + //log.Printf("...But converter implementation is not yet done. Here's some example data") + log.Printf("Writing to telegraf...\n") + for _, s := range t { + //Do some conversion - if s.ParentID == nil { - tags["parent_id"] = s.ID - } else { - tags["parent_id"] = strconv.Itoa(int(*s.ParentID)) - } - l.acc.AddFields("zipkin", fields, tags) + //l.acc.AddFields("zipkin", fields, tags) /* type Annotation struct { @@ -264,7 +346,7 @@ func (l *LineProtocolConverter) Record(t Trace) error { */ for _, a := range s.Annotations { - tags = map[string]string{ + /*tags = map[string]string{ "span_id": s.ID, "host": a.Host, "service_name": a.ServiceName, @@ -273,12 +355,30 @@ func (l *LineProtocolConverter) Record(t Trace) error { fields = map[string]interface{}{ "timestamp": a.Timestamp.Unix(), "value": a.Value, + }*/ + + fields := map[string]interface{}{ + // Maybe we don't need "annotation_timestamp"? + "annotation_timestamp": a.Timestamp.Unix(), + "duration": s.Duration, } - l.acc.AddFields("zipkin_annotations", fields, tags) + + log.Printf("Duration is: %d", s.Duration) + + tags := map[string]string{ + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + "service_name": a.ServiceName, + "annotation_value": a.Value, + "endpoint_host": a.Host, + } + l.acc.AddFields("zipkin", fields, tags, s.Timestamp) } for _, b := range s.BinaryAnnotations { - tags = map[string]string{ + /*tags := map[string]string{ "annotation_type": "binary", "host": b.Host, "service_name": b.ServiceName, @@ -286,15 +386,35 @@ func (l *LineProtocolConverter) Record(t Trace) error { "span_id": s.ID, } - fields = map[string]interface{}{ + fields := map[string]interface{}{ "value": b.Value, "key": b.Key, + }*/ + + fields := map[string]interface{}{ + "duration": s.Duration, } - l.acc.AddFields("zipkin_binary_annotations", fields, tags) + + log.Printf("Duration is: %d", s.Duration) + + tags := map[string]string{ + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + "service_name": b.ServiceName, + "annotation_value": b.Value, + "endpoint_host": b.Host, + "key": b.Key, + "type": b.Type, + } + + l.acc.AddFields("zipkin_binary_annotations", fields, tags, s.Timestamp) } } return nil + } func (l *LineProtocolConverter) Error(err error) { From c066ac1f379212fcff6e09d082a3c375293f221a Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 7 Jul 2017 15:49:43 -0700 Subject: [PATCH 035/154] Fix invalid struct fields in new_server_test.go for zipkin plugin --- plugins/inputs/zipkin/new_server_test.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/inputs/zipkin/new_server_test.go b/plugins/inputs/zipkin/new_server_test.go index 2ef1e198a0f57..292e7cdcec01a 100644 --- a/plugins/inputs/zipkin/new_server_test.go +++ b/plugins/inputs/zipkin/new_server_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "strconv" "testing" "time" ) @@ -55,12 +56,11 @@ func TestZipkinServer(t *testing.T) { parentID1 := int64(22964302721410078) want := Trace{ Span{ - Name: "Child", - ID: "8090652509916334619", - TraceIDHigh: nil, - ParentID: &parentID1, - Timestamp: time.Unix(1498688360851331, 0), - Duration: &d, + Name: "Child", + ID: "8090652509916334619", + ParentID: strconv.FormatInt(parentID1, 10), + Timestamp: time.Unix(1498688360851331, 0), + Duration: time.Duration(d), //note: []Annotation(nil) is different than // []Annotation{} Annotations: []Annotation(nil), @@ -77,10 +77,9 @@ func TestZipkinServer(t *testing.T) { Span{ Name: "Child", ID: "103618986556047333", - TraceIDHigh: nil, - ParentID: &parentID1, + ParentID: strconv.FormatInt(parentID1, 10), Timestamp: time.Unix(1498688360904552, 0), - Duration: &d1, + Duration: time.Duration(d1), Annotations: []Annotation(nil), BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ @@ -96,7 +95,7 @@ func TestZipkinServer(t *testing.T) { Name: "Parent", ID: "22964302721410078", Timestamp: time.Unix(1498688360851318, 0), - Duration: &d2, + Duration: time.Duration(d2), Annotations: []Annotation{ Annotation{ Timestamp: time.Unix(1498688360851325, 0), From ca91421ae17ec0a3d4f712f3a813c83a83740a08 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 09:04:03 -0700 Subject: [PATCH 036/154] Add zero check when working with traceid and remove excess comments in zipkin plugin --- plugins/inputs/zipkin/zipkin.go | 157 +------------------------------- 1 file changed, 4 insertions(+), 153 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index a5877d1f519b9..b8e0faf1d3898 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -117,7 +117,10 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { s := Span{} s.ID = strconv.FormatInt(span.GetID(), 10) - s.TraceID = strconv.FormatInt(span.GetTraceIDHigh(), 10) + strconv.FormatInt(span.GetTraceID(), 10) + s.TraceID = strconv.FormatInt(span.GetTraceID(), 10) + if span.GetTraceIDHigh() != 0 { + s.TraceID = strconv.FormatInt(span.GetTraceIDHigh(), 10) + s.TraceID + } s.Annotations = UnmarshalAnnotations(span.GetAnnotations()) @@ -199,13 +202,6 @@ func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]B } val := annotation.GetValue() - /*log.Println("Value: ", string(val)) - dst := make([]byte, base64.StdEncoding.DecodedLen(len(val))) - _, err := base64.StdEncoding.Decode(dst, annotation.GetValue()) - if err != nil { - return nil, err - }*/ - b.Key = annotation.GetKey() b.Value = string(val) b.Type = annotation.GetAnnotationType().String() @@ -219,144 +215,12 @@ type LineProtocolConverter struct { acc telegraf.Accumulator } -/*func (l *LineProtocolConverter) Record(t Trace) error { -log.Printf("received trace: %#+v\n", t) -//log.Printf("...But converter implementation is not yet done. Here's some example data") -log.Printf("Writing to telegraf...\n") - -// Example fields and tags -/*fields := map[string]interface{}{ - "Duration": "1060", // - "Timestamp": time.Unix(1498852876, 0), - "Annotations": []string{"An annotation"}, - "binary_annotations": []string{"A binary annotation"}, -} - -tags := map[string]string{ - "host": "http://hostname.com", - "port": "5555", -}*/ -//l.acc.AddFields("zipkin", fields, tags) -/* type Span struct { - ID string // zipkin traceid high concat with traceid Done - Name string Done - ParentID *int64 Done - Timestamp time.Time // If zipkin input is nil then time.Now() Done - Duration *int64 Done - TraceIDHigh *int64 Don't worry about - Annotations []Annotation - BinaryAnnotations []BinaryAnnotation -} - -*/ - -/* - type BinaryAnnotation struct { - Key string - Value string - Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port - ServiceName string - Type string - } -*/ - -/*for _, s := range t { -//Do some conversion - -fields := map[string]interface{}{ - "timestamp": s.Timestamp.Unix(), -} - -if s.Duration != nil { - fields["Duration"] = *s.Duration - log.Printf("Duration is: %d", *s.Duration) -} - -tags := map[string]string{ - "id": s.ID, - "name": s.Name, -} - -if s.ParentID == nil { - tags["parent_id"] = s.ID -} else { - tags["parent_id"] = strconv.Itoa(int(*s.ParentID)) -} -l.acc.AddFields("zipkin", fields, tags) - -/* - type Annotation struct { - Timestamp time.Time - Value string - Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port - ServiceName string - } - -*/ -/* for _, a := range s.Annotations { - tags = map[string]string{ - "span_id": s.ID, - "host": a.Host, - "service_name": a.ServiceName, - } - - fields = map[string]interface{}{ - "timestamp": a.Timestamp.Unix(), - "value": a.Value, - } - l.acc.AddFields("zipkin_annotations", fields, tags) - } - - for _, b := range s.BinaryAnnotations { - tags = map[string]string{ - "annotation_type": "binary", - "host": b.Host, - "service_name": b.ServiceName, - "type": b.Type, - "span_id": s.ID, - } - - fields = map[string]interface{}{ - "value": b.Value, - "key": b.Key, - } - l.acc.AddFields("zipkin_binary_annotations", fields, tags) - } - } - - return nil -}*/ - func (l *LineProtocolConverter) Record(t Trace) error { log.Printf("received trace: %#+v\n", t) //log.Printf("...But converter implementation is not yet done. Here's some example data") log.Printf("Writing to telegraf...\n") for _, s := range t { - //Do some conversion - - //l.acc.AddFields("zipkin", fields, tags) - - /* - type Annotation struct { - Timestamp time.Time - Value string - Host string // annotation.endpoint.ipv4 + ":" + annotation.endpoint.port - ServiceName string - } - - */ for _, a := range s.Annotations { - /*tags = map[string]string{ - "span_id": s.ID, - "host": a.Host, - "service_name": a.ServiceName, - } - - fields = map[string]interface{}{ - "timestamp": a.Timestamp.Unix(), - "value": a.Value, - }*/ - fields := map[string]interface{}{ // Maybe we don't need "annotation_timestamp"? "annotation_timestamp": a.Timestamp.Unix(), @@ -378,19 +242,6 @@ func (l *LineProtocolConverter) Record(t Trace) error { } for _, b := range s.BinaryAnnotations { - /*tags := map[string]string{ - "annotation_type": "binary", - "host": b.Host, - "service_name": b.ServiceName, - "type": b.Type, - "span_id": s.ID, - } - - fields := map[string]interface{}{ - "value": b.Value, - "key": b.Key, - }*/ - fields := map[string]interface{}{ "duration": s.Duration, } From 23b6798601888779c373e97714d11611dc666251 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 09:05:00 -0700 Subject: [PATCH 037/154] Update new_server_test.go to account for changes to zipkin schema --- plugins/inputs/zipkin/new_server_test.go | 26 +++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/zipkin/new_server_test.go b/plugins/inputs/zipkin/new_server_test.go index 292e7cdcec01a..38c5100f090b6 100644 --- a/plugins/inputs/zipkin/new_server_test.go +++ b/plugins/inputs/zipkin/new_server_test.go @@ -58,9 +58,10 @@ func TestZipkinServer(t *testing.T) { Span{ Name: "Child", ID: "8090652509916334619", + TraceID: "2505404965370368069", ParentID: strconv.FormatInt(parentID1, 10), - Timestamp: time.Unix(1498688360851331, 0), - Duration: time.Duration(d), + Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)), + Duration: time.Duration(d) * time.Microsecond, //note: []Annotation(nil) is different than // []Annotation{} Annotations: []Annotation(nil), @@ -77,9 +78,10 @@ func TestZipkinServer(t *testing.T) { Span{ Name: "Child", ID: "103618986556047333", + TraceID: "2505404965370368069", ParentID: strconv.FormatInt(parentID1, 10), - Timestamp: time.Unix(1498688360904552, 0), - Duration: time.Duration(d1), + Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)), + Duration: time.Duration(d1) * time.Microsecond, Annotations: []Annotation(nil), BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ @@ -94,8 +96,10 @@ func TestZipkinServer(t *testing.T) { Span{ Name: "Parent", ID: "22964302721410078", - Timestamp: time.Unix(1498688360851318, 0), - Duration: time.Duration(d2), + TraceID: "2505404965370368069", + ParentID: "22964302721410078", + Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)), + Duration: time.Duration(d2) * time.Microsecond, Annotations: []Annotation{ Annotation{ Timestamp: time.Unix(1498688360851325, 0), @@ -131,7 +135,15 @@ func TestZipkinServer(t *testing.T) { fmt.Println("BINARY ANNOTATIONS FOR TESTING: ") fmt.Println(got[0].BinaryAnnotations, want[0].BinaryAnnotations) - if !reflect.DeepEqual(got, want) { + /* if !reflect.DeepEqual(got, want) { t.Fatal("Got != want, Fields weren't unmarshalled correctly") + }*/ + + for i, s := range got { + if !reflect.DeepEqual(s, want[i]) { + fmt.Printf("index %d wasn't equal", i) + fmt.Println(s, want[i]) + t.Fatal("Got != want, Fields weren't unmarshalled correctly") + } } } From e643d12c7ee69ce6486ff80c4b1b1b4dc3f7713b Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 13:32:56 -0700 Subject: [PATCH 038/154] Add initial unit test for zipkin plugin --- plugins/inputs/zipkin/plugin_test.go | 197 +++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 plugins/inputs/zipkin/plugin_test.go diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go new file mode 100644 index 0000000000000..d1a57f1283e10 --- /dev/null +++ b/plugins/inputs/zipkin/plugin_test.go @@ -0,0 +1,197 @@ +package zipkin + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "net/http" + "reflect" + "testing" + "time" + + "github.com/influxdata/telegraf/testutil" +) + +type UnitTest struct { + datfile string + tags []map[string]string + expectedFields []map[string]interface{} +} + +func TestZipkinPlugin(t *testing.T) { + var acc testutil.Accumulator + z := &Zipkin{ + Path: "/api/v1/test", + Port: 9411, + } + err := z.Start(&acc) + if err != nil { + t.Fatal("Failed to start zipkin server") + } + defer z.Stop() + + postTestData(t, "testdata/threespans.dat") + acc.Wait(6) + + for _, m := range acc.Metrics { + log.Println(m) + } + + if len(acc.Errors) != 0 { + for _, e := range acc.Errors { + fmt.Println(e) + } + t.Fatal("Errors were added during request") + } + + tags := map[string]string{ + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + } + + //map[endpoint_host:2130706433:0 name:Child service_name:trivial trace_id:2505404965370368069 annotation_value:trivial key:lc type:STRING id:8090652509916334619 parent_id:22964302721410078] + + //map[trace_id:2505404965370368069 key:lc type:STRING id:103618986556047333 name:Child service_name:trivial annotation_value:trivial endpoint_host:2130706433:0 parent_id:22964302721410078] + + assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(53106)*time.Microsecond, tags) + + tags = map[string]string{ + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + } + assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(50410)*time.Microsecond, tags) + + tags = map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "Starting child #0", + "endpoint_host": "2130706433:0", + } + assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360851325, tags) + tags["annotation_value"] = "Starting child #1" + assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360904545, tags) + tags["annotation_value"] = "A Log" + assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360954992, tags) + + tags = map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + } + + assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(103680)*time.Microsecond, tags) +} + +func assertContainsTaggedDuration( + t *testing.T, + acc *testutil.Accumulator, + measurement string, + field string, + expectedValue time.Duration, + tags map[string]string, +) { + log.Println("going through tagged ") + var actualValue interface{} + log.Println(acc.Metrics) + for _, pt := range acc.Metrics { + log.Println("looping, point is : ", pt) + log.Println("point tags are : ", pt.Tags) + if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { + log.Println("found measurement") + for fieldname, value := range pt.Fields { + fmt.Println("looping through fields") + if fieldname == field { + fmt.Println("found field: ", field) + actualValue = value + fmt.Println("Value: ", value) + if value == expectedValue { + return + } + t.Errorf("Expected value %d\n got value %d\n", expectedValue, value) + } + } + } + } + msg := fmt.Sprintf( + "Could not find measurement \"%s\" with requested tags within %s, Actual: %d", + measurement, field, actualValue) + t.Fatal(msg) +} + +func assertContainsTaggedInt64( + t *testing.T, + acc *testutil.Accumulator, + measurement string, + field string, + expectedValue int64, + tags map[string]string, +) { + log.Println("going through tagged ") + var actualValue interface{} + log.Println(acc.Metrics) + for _, pt := range acc.Metrics { + log.Println("looping, point is : ", pt) + log.Println("point tags are : ", pt.Tags) + if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { + log.Println("found measurement") + for fieldname, value := range pt.Fields { + fmt.Println("looping through fields") + if fieldname == field { + fmt.Println("found field: ", field) + actualValue = value + fmt.Println("Value: ", value) + if value == expectedValue { + return + } + t.Errorf("Expected value %v\n got value %v\n", expectedValue, value) + } + } + } + } + msg := fmt.Sprintf( + "Could not find measurement \"%s\" with requested tags within %s, Actual: %d", + measurement, field, actualValue) + t.Fatal(msg) +} + +func postTestData(t *testing.T, datafile string) { + dat, err := ioutil.ReadFile("testdata/threespans.dat") + if err != nil { + t.Fatal("Could not read from data file") + } + + req, err := http.NewRequest("POST", "http://localhost:9411/api/v1/test", bytes.NewReader(dat)) + + if err != nil { + t.Fatal("bad http request") + } + client := &http.Client{} + _, err = client.Do(req) + if err != nil { + t.Fatal("http request failed") + } +} From c876696785208629257e595d570280198d5b4b5e Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 13:37:05 -0700 Subject: [PATCH 039/154] Add additional log statement to server_new.go --- plugins/inputs/zipkin/server_new.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/zipkin/server_new.go b/plugins/inputs/zipkin/server_new.go index ea21910939e27..c3502e5cf082a 100644 --- a/plugins/inputs/zipkin/server_new.go +++ b/plugins/inputs/zipkin/server_new.go @@ -37,6 +37,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) + log.Printf("body=%s\n", string(body)) if err != nil { e := fmt.Errorf("Encountered error: %s", err) log.Println(e) From 24ceba6ac42417169764f2c94dac56c0088728f8 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 13:38:46 -0700 Subject: [PATCH 040/154] Change name of measurement for zipkin binary annotations --- plugins/inputs/zipkin/zipkin.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index b8e0faf1d3898..80f148f5736b2 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -260,7 +260,7 @@ func (l *LineProtocolConverter) Record(t Trace) error { "type": b.Type, } - l.acc.AddFields("zipkin_binary_annotations", fields, tags, s.Timestamp) + l.acc.AddFields("zipkin", fields, tags, s.Timestamp) } } @@ -313,6 +313,7 @@ func (z *Zipkin) Gather(acc telegraf.Accumulator) error { return nil } // Start launches a separate goroutine for collecting zipkin client http requests, // passing in a telegraf.Accumulator such that data can be collected. func (z *Zipkin) Start(acc telegraf.Accumulator) error { + log.Println("starting...") if z.tracing == nil { t := NewServer(z.Path) z.tracing = t From 36198dedeb3bb0b2b22e7d40a37cbc81a3155c67 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 15:48:51 -0700 Subject: [PATCH 041/154] Add clarifying comments to zipkin server test, fix hardcoded path name bug --- plugins/inputs/zipkin/plugin_test.go | 50 +++++++++++++++++++--------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index d1a57f1283e10..003ff4e0ff719 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -13,13 +13,9 @@ import ( "github.com/influxdata/telegraf/testutil" ) -type UnitTest struct { - datfile string - tags []map[string]string - expectedFields []map[string]interface{} -} +var tests []func(*testing.T) -func TestZipkinPlugin(t *testing.T) { +func testBasicSpans(t *testing.T) { var acc testutil.Accumulator z := &Zipkin{ Path: "/api/v1/test", @@ -34,10 +30,6 @@ func TestZipkinPlugin(t *testing.T) { postTestData(t, "testdata/threespans.dat") acc.Wait(6) - for _, m := range acc.Metrics { - log.Println(m) - } - if len(acc.Errors) != 0 { for _, e := range acc.Errors { fmt.Println(e) @@ -45,6 +37,11 @@ func TestZipkinPlugin(t *testing.T) { t.Fatal("Errors were added during request") } + // Actual testing: + + // The tags we will be querying by: + + // Test for the first span in the trace: tags := map[string]string{ "id": "8090652509916334619", "parent_id": "22964302721410078", @@ -57,12 +54,16 @@ func TestZipkinPlugin(t *testing.T) { "type": "STRING", } - //map[endpoint_host:2130706433:0 name:Child service_name:trivial trace_id:2505404965370368069 annotation_value:trivial key:lc type:STRING id:8090652509916334619 parent_id:22964302721410078] - - //map[trace_id:2505404965370368069 key:lc type:STRING id:103618986556047333 name:Child service_name:trivial annotation_value:trivial endpoint_host:2130706433:0 parent_id:22964302721410078] + // assertContainsTaggedDuration asserts that the specified field which corresponds to `tags` has + // the specified value. In this case, we are testing that the measurement zipkin with tags `tags` has a + // field called `duration` with the value 53106 microseconds assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(53106)*time.Microsecond, tags) + // Test for the second span in the trace: + + //Update tags in order to perform our next test + tags = map[string]string{ "id": "103618986556047333", "parent_id": "22964302721410078", @@ -74,8 +75,12 @@ func TestZipkinPlugin(t *testing.T) { "key": "lc", "type": "STRING", } + + //Similar test as above, but with different tags assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(50410)*time.Microsecond, tags) + //test for the third span in the trace (with three annotations) + tags = map[string]string{ "id": "22964302721410078", "parent_id": "22964302721410078", @@ -85,9 +90,15 @@ func TestZipkinPlugin(t *testing.T) { "annotation_value": "Starting child #0", "endpoint_host": "2130706433:0", } + + // test for existence of annotation specific fields assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360851325, tags) + + // test for existence of annotation specific fields tags["annotation_value"] = "Starting child #1" assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360904545, tags) + + //test for existence of annotation specific fields tags["annotation_value"] = "A Log" assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360954992, tags) @@ -102,10 +113,19 @@ func TestZipkinPlugin(t *testing.T) { "key": "lc", "type": "STRING", } - + // test for existence of span time stamp in third span, using binary annotation specific values. assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(103680)*time.Microsecond, tags) } +func TestZipkinPlugin(t *testing.T) { + tests = append(tests, testBasicSpans) + + // iterate through all tests to run each test in sequence + for _, test := range tests { + t.Run("Trivial Test", test) + } +} + func assertContainsTaggedDuration( t *testing.T, acc *testutil.Accumulator, @@ -179,7 +199,7 @@ func assertContainsTaggedInt64( } func postTestData(t *testing.T, datafile string) { - dat, err := ioutil.ReadFile("testdata/threespans.dat") + dat, err := ioutil.ReadFile(datafile) if err != nil { t.Fatal("Could not read from data file") } From 981b464b80e1a9ec321f33ccf34cea6bf0be6d95 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 16:22:22 -0700 Subject: [PATCH 042/154] Add timestamp test for last metric in zipkin plugin unit test --- plugins/inputs/zipkin/plugin_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index 003ff4e0ff719..caa65974d9493 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -115,6 +115,10 @@ func testBasicSpans(t *testing.T) { } // test for existence of span time stamp in third span, using binary annotation specific values. assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(103680)*time.Microsecond, tags) + log.Println("end") + log.Println("TIMESTAMP: ", acc.Metrics[5].Time) + assertTimeIs(t, &acc, "zipkin", time.Unix(1498688360, 851318*int64(time.Microsecond)), tags) + } func TestZipkinPlugin(t *testing.T) { @@ -126,6 +130,29 @@ func TestZipkinPlugin(t *testing.T) { } } +func assertTimeIs(t *testing.T, acc *testutil.Accumulator, + measurement string, + expectedValue time.Time, + tags map[string]string) { + var actualValue time.Time + for _, pt := range acc.Metrics { + if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { + actualValue = pt.Time + if actualValue == expectedValue { + return + } + + t.Errorf("Expected value %d\n got value %d\n", expectedValue, actualValue) + + } + } + + msg := fmt.Sprintf( + "Could not find measurement \"%s\" with requested tags and time: %v, Actual: %v", + measurement, expectedValue, actualValue) + t.Fatal(msg) +} + func assertContainsTaggedDuration( t *testing.T, acc *testutil.Accumulator, From 7e828f6140466f189f6d201acd136a17d3c22ba2 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 16:24:00 -0700 Subject: [PATCH 043/154] Remove extraneous debug statement from new_server_test.go in zipkin plugin --- plugins/inputs/zipkin/new_server_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/inputs/zipkin/new_server_test.go b/plugins/inputs/zipkin/new_server_test.go index 38c5100f090b6..33189c3fa4ba1 100644 --- a/plugins/inputs/zipkin/new_server_test.go +++ b/plugins/inputs/zipkin/new_server_test.go @@ -132,9 +132,6 @@ func TestZipkinServer(t *testing.T) { }, } - fmt.Println("BINARY ANNOTATIONS FOR TESTING: ") - fmt.Println(got[0].BinaryAnnotations, want[0].BinaryAnnotations) - /* if !reflect.DeepEqual(got, want) { t.Fatal("Got != want, Fields weren't unmarshalled correctly") }*/ From 71d4b2f4809c5486214eee272cde7c32daa7d014 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 10 Jul 2017 16:24:28 -0700 Subject: [PATCH 044/154] Comment out debug statements in zipkin.go --- plugins/inputs/zipkin/zipkin.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 80f148f5736b2..3fac0be226698 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -142,18 +142,18 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { // these values. Otherwise, we just leave them as nil duration := time.Duration(span.GetDuration()) - fmt.Println("Duration: ", duration) + // fmt.Println("Duration: ", duration) s.Duration = duration * time.Microsecond parentID := span.GetParentID() - fmt.Println("Parent ID: ", parentID) + // fmt.Println("Parent ID: ", parentID) if parentID == 0 { s.ParentID = s.ID } else { s.ParentID = strconv.FormatInt(parentID, 10) } - fmt.Println("ID:", s.ID) + // fmt.Println("ID:", s.ID) trace = append(trace, s) } @@ -178,7 +178,7 @@ func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { a.Value = annotation.GetValue() formatted = append(formatted, a) } - fmt.Println("formatted annotations: ", formatted) + //fmt.Println("formatted annotations: ", formatted) return formatted } @@ -195,8 +195,6 @@ func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]B if endpoint != nil { b.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) b.ServiceName = endpoint.GetServiceName() - - fmt.Println("Binary Annotation host and service name: ", b.Host, b.ServiceName) } else { b.Host, b.ServiceName = "", "" } @@ -238,6 +236,7 @@ func (l *LineProtocolConverter) Record(t Trace) error { "annotation_value": a.Value, "endpoint_host": a.Host, } + log.Println("adding data") l.acc.AddFields("zipkin", fields, tags, s.Timestamp) } @@ -259,7 +258,7 @@ func (l *LineProtocolConverter) Record(t Trace) error { "key": b.Key, "type": b.Type, } - + log.Printf("adding data") l.acc.AddFields("zipkin", fields, tags, s.Timestamp) } } From bd08579dbf0e73d09315b0252ca0bed957d20df3 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 14:17:54 -0700 Subject: [PATCH 045/154] Add additional log statement to zipkin server test --- plugins/inputs/zipkin/new_server_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/inputs/zipkin/new_server_test.go b/plugins/inputs/zipkin/new_server_test.go index 33189c3fa4ba1..1218dc1c6754d 100644 --- a/plugins/inputs/zipkin/new_server_test.go +++ b/plugins/inputs/zipkin/new_server_test.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io/ioutil" + "log" "net/http" "net/http/httptest" "reflect" @@ -28,6 +29,7 @@ func (m *MockTracer) Error(err error) { } func TestZipkinServer(t *testing.T) { + log.Println("testing server unmarshalling") dat, err := ioutil.ReadFile("testdata/threespans.dat") if err != nil { t.Fatalf("Could not find file %s\n", "test/threespans.dat") From e42c14b683af7ac01ec4aefd84c27f98b51ca701 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Tue, 11 Jul 2017 14:21:09 -0700 Subject: [PATCH 046/154] Add unit test framework to zipkin plugin tests Signed-off-by: Adam Perlin --- plugins/inputs/zipkin/plugin_test.go | 97 ++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index caa65974d9493..8e2dfe7b0b73a 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -13,9 +13,58 @@ import ( "github.com/influxdata/telegraf/testutil" ) -var tests []func(*testing.T) +//expectedTags[0] expectedValues[0] + +func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { + log.Println("running!") + postTestData(t, u.datafile) + log.Println("LENGTH:", len(u.expected)) + //acc.Wait(len(u.expected)) + for _, data := range u.expected { + for key, value := range data.expectedValues { + switch value.(type) { + case int64: + assertContainsTaggedInt64(t, acc, u.measurement, key, value.(int64), data.expectedTags) + break + case time.Duration: + assertContainsTaggedDuration(t, acc, u.measurement, key, value.(time.Duration), data.expectedTags) + break + case time.Time: + if key == "time" { + assertTimeIs(t, acc, u.measurement, value.(time.Time), data.expectedTags) + } else { + assertContainsTaggedTime(t, acc, u.measurement, key, value.(time.Time), data.expectedTags) + } + break + default: + t.Fatalf("Invalid type for field %v\n", reflect.TypeOf(value)) + break + } + } + } +} + +func TestZipkin(t *testing.T) { + log.Println("testing zipkin...") + var acc testutil.Accumulator + z := &Zipkin{ + Path: "/api/v1/test", + Port: 9411, + } + err := z.Start(&acc) + if err != nil { + t.Fatal("Failed to start zipkin server") + } + defer z.Stop() + //t.Fatal("failed!!!") + + for _, test := range tests { + test.Run(t, &acc) + } +} func testBasicSpans(t *testing.T) { + log.Println("testing basic spans...") var acc testutil.Accumulator z := &Zipkin{ Path: "/api/v1/test", @@ -118,16 +167,6 @@ func testBasicSpans(t *testing.T) { log.Println("end") log.Println("TIMESTAMP: ", acc.Metrics[5].Time) assertTimeIs(t, &acc, "zipkin", time.Unix(1498688360, 851318*int64(time.Microsecond)), tags) - -} - -func TestZipkinPlugin(t *testing.T) { - tests = append(tests, testBasicSpans) - - // iterate through all tests to run each test in sequence - for _, test := range tests { - t.Run("Trivial Test", test) - } } func assertTimeIs(t *testing.T, acc *testutil.Accumulator, @@ -225,6 +264,42 @@ func assertContainsTaggedInt64( t.Fatal(msg) } +func assertContainsTaggedTime( + t *testing.T, + acc *testutil.Accumulator, + measurement string, + field string, + expectedValue time.Time, + tags map[string]string, +) { + log.Println("going through tagged ") + var actualValue interface{} + log.Println(acc.Metrics) + for _, pt := range acc.Metrics { + log.Println("looping, point is : ", pt) + log.Println("point tags are : ", pt.Tags) + if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { + log.Println("found measurement") + for fieldname, value := range pt.Fields { + fmt.Println("looping through fields") + if fieldname == field { + fmt.Println("found field: ", field) + actualValue = value + fmt.Println("Value: ", value) + if value == expectedValue { + return + } + t.Errorf("Expected value %v\n got value %v\n", expectedValue, value) + } + } + } + } + msg := fmt.Sprintf( + "Could not find measurement \"%s\" with requested tags within %s, Actual: %d", + measurement, field, actualValue) + t.Fatal(msg) +} + func postTestData(t *testing.T, datafile string) { dat, err := ioutil.ReadFile(datafile) if err != nil { From 7887367484e12b628d8324be9b30ce5016f5adf9 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 14:22:51 -0700 Subject: [PATCH 047/154] Add separate file for unit test data to zipkin plugin Signed-off-by: Ayrdrie Palmer --- plugins/inputs/zipkin/unit_tests.go | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 plugins/inputs/zipkin/unit_tests.go diff --git a/plugins/inputs/zipkin/unit_tests.go b/plugins/inputs/zipkin/unit_tests.go new file mode 100644 index 0000000000000..c791b31518cf8 --- /dev/null +++ b/plugins/inputs/zipkin/unit_tests.go @@ -0,0 +1,114 @@ +package zipkin + +import "time" + +type UnitTest struct { + expected []TestData + measurement string + datafile string +} + +type TestData struct { + expectedTags map[string]string + expectedValues map[string]interface{} +} + +var tests = []UnitTest{ + UnitTest{ + measurement: "zipkin", + datafile: "testdata/threespans.dat", + expected: []TestData{ + { + expectedTags: map[string]string{ + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + expectedValues: map[string]interface{}{ + "duration": time.Duration(53106) * time.Microsecond, + }, + }, + { + expectedTags: map[string]string{ + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + expectedValues: map[string]interface{}{ + "duration": time.Duration(50410) * time.Microsecond, + }, + }, + { + expectedTags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "Starting child #0", + "endpoint_host": "2130706433:0", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1498688360851325), + }, + }, + { + expectedTags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "Starting child #1", + "endpoint_host": "2130706433:0", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1498688360904545), + }, + }, + { + expectedTags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "A Log", + "endpoint_host": "2130706433:0", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1498688360954992), + }, + }, + { + expectedTags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + expectedValues: map[string]interface{}{ + "duration": time.Duration(103680) * time.Microsecond, + "time": time.Unix(1498688360, 851318*int64(time.Microsecond)), + }, + }, + }, + }, +} From 6003833d22fa933c4e7d003c9b16f7cea97d413f Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 14:34:01 -0700 Subject: [PATCH 048/154] Remove new_server_test.go from zipkin plugin --- plugins/inputs/zipkin/new_server_test.go | 148 ----------------------- 1 file changed, 148 deletions(-) delete mode 100644 plugins/inputs/zipkin/new_server_test.go diff --git a/plugins/inputs/zipkin/new_server_test.go b/plugins/inputs/zipkin/new_server_test.go deleted file mode 100644 index 1218dc1c6754d..0000000000000 --- a/plugins/inputs/zipkin/new_server_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package zipkin - -import ( - "bytes" - "fmt" - "io/ioutil" - "log" - "net/http" - "net/http/httptest" - "reflect" - "strconv" - "testing" - "time" -) - -type MockTracer struct { - Data Trace - Err error -} - -func (m *MockTracer) Record(t Trace) error { - fmt.Println("Adding trace ", t) - m.Data = t - return nil -} - -func (m *MockTracer) Error(err error) { - m.Err = err -} - -func TestZipkinServer(t *testing.T) { - log.Println("testing server unmarshalling") - dat, err := ioutil.ReadFile("testdata/threespans.dat") - if err != nil { - t.Fatalf("Could not find file %s\n", "test/threespans.dat") - } - - s := NewServer("/api/v1/spans") - mockTracer := &MockTracer{} - s.tracer = mockTracer - w := httptest.NewRecorder() - r := httptest.NewRequest( - "POST", - "http://server.local/api/v1/spans", - ioutil.NopCloser( - bytes.NewReader(dat))) - handler := s.SpanHandler - handler(w, r) - if w.Code != http.StatusNoContent { - t.Errorf("MainHandler did not return StatusNoContent %d", w.Code) - } - - got := mockTracer.Data - - d := int64(53106) - d1 := int64(50410) - d2 := int64(103680) - parentID1 := int64(22964302721410078) - want := Trace{ - Span{ - Name: "Child", - ID: "8090652509916334619", - TraceID: "2505404965370368069", - ParentID: strconv.FormatInt(parentID1, 10), - Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)), - Duration: time.Duration(d) * time.Microsecond, - //note: []Annotation(nil) is different than - // []Annotation{} - Annotations: []Annotation(nil), - BinaryAnnotations: []BinaryAnnotation{ - BinaryAnnotation{ - Key: "lc", - Value: "trivial", - Host: "2130706433:0", - ServiceName: "trivial", - Type: "STRING", - }, - }, - }, - Span{ - Name: "Child", - ID: "103618986556047333", - TraceID: "2505404965370368069", - ParentID: strconv.FormatInt(parentID1, 10), - Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)), - Duration: time.Duration(d1) * time.Microsecond, - Annotations: []Annotation(nil), - BinaryAnnotations: []BinaryAnnotation{ - BinaryAnnotation{ - Key: "lc", - Value: "trivial", - Host: "2130706433:0", - ServiceName: "trivial", - Type: "STRING", - }, - }, - }, - Span{ - Name: "Parent", - ID: "22964302721410078", - TraceID: "2505404965370368069", - ParentID: "22964302721410078", - Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)), - Duration: time.Duration(d2) * time.Microsecond, - Annotations: []Annotation{ - Annotation{ - Timestamp: time.Unix(1498688360851325, 0), - Value: "Starting child #0", - Host: "2130706433:0", - ServiceName: "trivial", - }, - Annotation{ - Timestamp: time.Unix(1498688360904545, 0), - Value: "Starting child #1", - Host: "2130706433:0", - ServiceName: "trivial", - }, - Annotation{ - Timestamp: time.Unix(1498688360954992, 0), - Value: "A Log", - Host: "2130706433:0", - ServiceName: "trivial", - }, - }, - BinaryAnnotations: []BinaryAnnotation{ - BinaryAnnotation{ - Key: "lc", - Value: "trivial", - Host: "2130706433:0", - ServiceName: "trivial", - Type: "STRING", - }, - }, - }, - } - - /* if !reflect.DeepEqual(got, want) { - t.Fatal("Got != want, Fields weren't unmarshalled correctly") - }*/ - - for i, s := range got { - if !reflect.DeepEqual(s, want[i]) { - fmt.Printf("index %d wasn't equal", i) - fmt.Println(s, want[i]) - t.Fatal("Got != want, Fields weren't unmarshalled correctly") - } - } -} From 581a8af1e5acf54b63e888b6a2f90d09b73e5e20 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 14:35:19 -0700 Subject: [PATCH 049/154] Add dependecy to server.go in zipkin plugin --- plugins/inputs/zipkin/server.go | 257 +++++++++----------------------- 1 file changed, 74 insertions(+), 183 deletions(-) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index 436befbf416a9..c3502e5cf082a 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -1,221 +1,112 @@ package zipkin import ( - "context" "encoding/json" "fmt" "io/ioutil" "log" "net/http" - "os" - "strconv" - "time" + "sync" - //TODO: must add below deps to Godeps file "github.com/apache/thrift/lib/go/thrift" + "github.com/gorilla/mux" "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) -const version = "1.0" -const port = "9411" - -// Zipkin writes to this route by default -//const DefaultRoute = "/api/v1/spans" - -// Provides a shutdown timout in order to gracefully shutdown our http server -const DefaultShutdownTimeout = 5 * time.Second +// Server is an implementation of tracer which is a helper for running an +// http server which accepts zipkin requests +type Server struct { + Path string + tracer Tracer + waitGroup *sync.WaitGroup +} -type ZipkinServer struct { - errorChan chan error - dataChan chan SpanData - Port string - //Done chan struct{} - //listener net.Listener - HTTPServer *http.Server - logger *log.Logger +// Register allows server to implement the Service interface. Server's register metod +// registers its handler on mux, and sets the servers tracer with tracer +func (s *Server) Register(router *mux.Router, tracer Tracer) error { + // TODO: potentially move router into Server if appropriate + router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") + s.tracer = tracer + return nil } -// SpanData is an alias for a slice of references to zipkincore.Span -// created for better abstraction of internal package types -type SpanData []*zipkincore.Span +// SpanHandler is the handler Server uses for handling zipkin POST requests +func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { + log.Printf("Received request from: %s", r.URL.String()) + log.Printf("Raw request data is: %#+v", r) + defer r.Body.Close() -// NewHTTPServer creates a new Zipkin http server given a port and a set of -// channels -func NewHTTPServer(port int, e chan error, d chan SpanData) *ZipkinServer { - logger := log.New(os.Stdout, "", 0) - return &ZipkinServer{ - errorChan: e, - dataChan: d, - Port: strconv.Itoa(port), - logger: logger, + body, err := ioutil.ReadAll(r.Body) + log.Printf("body=%s\n", string(body)) + if err != nil { + e := fmt.Errorf("Encountered error: %s", err) + log.Println(e) + s.tracer.Error(e) + //TODO: Change http status that is sent back to client + w.WriteHeader(http.StatusInternalServerError) + return } -} -// Version adds a version header to response -// Delete later -func Version(h http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("X-Proxy-Version", version) - h.ServeHTTP(w, r) + buffer := thrift.NewTMemoryBuffer() + if _, err = buffer.Write(body); err != nil { + log.Println(err) + s.tracer.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return } - return http.HandlerFunc(fn) -} -// Logger is middleware that logs the request -// delete later, re-implement in a better way -// inspired by the httptrace package -func Logger(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - log.Printf("Received request from url %s\n", r.URL.String()) - next.ServeHTTP(w, r) + transport := thrift.NewTBinaryProtocolTransport(buffer) + _, size, err := transport.ReadListBegin() + if err != nil { + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return } - return http.HandlerFunc(fn) -} - -// MainHandler returns a closure with access to a *ZipkinServer pointer -// for use as an http ZipkinServer handler -func (s *ZipkinServer) MainHandler() http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - s.logger.Printf("Received request from: %s", r.URL.String()) - s.logger.Printf("Raw request data is: %#+v", r) - defer r.Body.Close() - body, err := ioutil.ReadAll(r.Body) - if err != nil { - e := fmt.Errorf("Encoutered error: %s", err) - s.logger.Println(e) - s.errorChan <- e - w.WriteHeader(http.StatusNoContent) - return - } - buffer := thrift.NewTMemoryBuffer() - if _, err = buffer.Write(body); err != nil { - s.logger.Println(err) - s.errorChan <- err - w.WriteHeader(http.StatusNoContent) - return - } - transport := thrift.NewTBinaryProtocolTransport(buffer) - _, size, err := transport.ReadListBegin() - if err != nil { - s.logger.Printf("%s", err) - s.errorChan <- err - w.WriteHeader(http.StatusNoContent) - return - } - var spans []*zipkincore.Span - for i := 0; i < size; i++ { - zs := &zipkincore.Span{} - if err = zs.Read(transport); err != nil { - s.logger.Printf("%s", err) - s.errorChan <- err - w.WriteHeader(http.StatusNoContent) - return - } - spans = append(spans, zs) - } - err = transport.ReadListEnd() - if err != nil { - s.logger.Printf("%s", err) - s.errorChan <- err - w.WriteHeader(http.StatusNoContent) + var spans []*zipkincore.Span + for i := 0; i < size; i++ { + zs := &zipkincore.Span{} + if err = zs.Read(transport); err != nil { + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusInternalServerError) return } - out, _ := json.MarshalIndent(spans, "", " ") - s.logger.Println(string(out)) - s.dataChan <- SpanData(spans) - w.WriteHeader(http.StatusNoContent) + spans = append(spans, zs) } - return http.HandlerFunc(fn) -} - -// Serve creates an internal http.Server and calls its ListenAndServe method to -// start serving. It uses the MainHandler as the route handler. -func (s *ZipkinServer) Serve() { - mux := http.NewServeMux() - mux.Handle(DefaultRoute, s.MainHandler()) - - s.HTTPServer = &http.Server{ - Addr: ":" + s.Port, - Handler: mux, - } - - s.logger.Printf("Starting zipkin HTTP server on %s\n", s.Port) - s.logger.Fatal(s.HTTPServer.ListenAndServe()) -} - -// Shutdown gracefully shuts down the internal http server -func (s *ZipkinServer) Shutdown() { - ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) - defer cancel() - - s.HTTPServer.Shutdown(ctx) -} - -// HandleZipkinRequests starts a zipkin http server on the port specified -// wthin the *Server it is called on. It receives data from zipkin, and sends -// it back to the caller over the channels the caller constructed the *Server -// with -/*func (s *Server) HandleZipkinRequests() { - log.Printf("Starting zipkin HTTP server on %s\n", s.Port) - mux := http.NewServeMux() - // The func MainHandler returns has been closure-ified - mux.Handle("/api/v1/spans", s.MainHandler()) - - listener, err := net.Listen("tcp", ":"+s.Port) + err = transport.ReadListEnd() if err != nil { - //Print error out for debugging purposes - e := fmt.Errorf("Error listening on port 9411 %v", err) - //Send error through channel to caller - s.errorChan <- e + log.Printf("%s", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusInternalServerError) return } - s.addListener(listener) - //TODO: put a sync group around this ListenForStop() - // create wait group; add to wait group (wg.Add(1)) - // pass in to ListenForStop() - //wg.Add() - //go func(){ - go s.ListenForStop() - //wg.Done() - //}() - - // TODO: don't need to use graceful anymore in go 1.8 (there is graceful Server - // shutdown) - httpServer := &graceful.Server{Server: new(http.Server)} - httpServer.SetKeepAlivesEnabled(true) - httpServer.TCPKeepAlive = 5 * time.Second - httpServer.Handler = Version(Logger(mux)) - - log.Fatal(httpServer.Serve(listener)) - -}*/ - -/*func (s *Server) addListener(l net.Listener) { - s.listener = l -}*/ + //marshal json for debugging purposes + out, _ := json.MarshalIndent(spans, "", " ") + log.Println(string(out)) -// ListenForStop selects over the Server.Done channel, and stops the -// server's internal net.Listener when a singnal is received. -/*func (s *Server) ListenForStop() { - if s.listener == nil { - log.Fatal("Listen called without listener instance") + trace, err := UnmarshalZipkinResponse(spans) + if err != nil { + log.Println("Error: ", err) + s.tracer.Error(err) + w.WriteHeader(http.StatusInternalServerError) return } - select { - case <-s.Done: - log.Println("closing up listener...") - s.listener.Close() + + if err = s.tracer.Record(trace); err != nil { + log.Println(err) + w.WriteHeader(http.StatusInternalServerError) return } -}*/ + w.WriteHeader(http.StatusNoContent) +} -// CloseAllChannels closes the Server's communication channels on the server's end. -func (s *ZipkinServer) CloseAllChannels() { - log.Printf("Closing all communication channels...\n") - close(s.dataChan) - close(s.errorChan) - //close(s.Done) +//NewServer returns a new server instance given path to handle +func NewServer(path string) *Server { + return &Server{ + Path: path, + } } From 38c85f61be5fa55ceee76a9559a8215e6c98dac2 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 14:36:05 -0700 Subject: [PATCH 050/154] Remove server_new.go --- plugins/inputs/zipkin/server_new.go | 112 ---------------------------- 1 file changed, 112 deletions(-) delete mode 100644 plugins/inputs/zipkin/server_new.go diff --git a/plugins/inputs/zipkin/server_new.go b/plugins/inputs/zipkin/server_new.go deleted file mode 100644 index c3502e5cf082a..0000000000000 --- a/plugins/inputs/zipkin/server_new.go +++ /dev/null @@ -1,112 +0,0 @@ -package zipkin - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "sync" - - "github.com/apache/thrift/lib/go/thrift" - "github.com/gorilla/mux" - "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" -) - -// Server is an implementation of tracer which is a helper for running an -// http server which accepts zipkin requests -type Server struct { - Path string - tracer Tracer - waitGroup *sync.WaitGroup -} - -// Register allows server to implement the Service interface. Server's register metod -// registers its handler on mux, and sets the servers tracer with tracer -func (s *Server) Register(router *mux.Router, tracer Tracer) error { - // TODO: potentially move router into Server if appropriate - router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") - s.tracer = tracer - return nil -} - -// SpanHandler is the handler Server uses for handling zipkin POST requests -func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { - log.Printf("Received request from: %s", r.URL.String()) - log.Printf("Raw request data is: %#+v", r) - defer r.Body.Close() - - body, err := ioutil.ReadAll(r.Body) - log.Printf("body=%s\n", string(body)) - if err != nil { - e := fmt.Errorf("Encountered error: %s", err) - log.Println(e) - s.tracer.Error(e) - //TODO: Change http status that is sent back to client - w.WriteHeader(http.StatusInternalServerError) - return - } - - buffer := thrift.NewTMemoryBuffer() - if _, err = buffer.Write(body); err != nil { - log.Println(err) - s.tracer.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - transport := thrift.NewTBinaryProtocolTransport(buffer) - _, size, err := transport.ReadListBegin() - if err != nil { - log.Printf("%s", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - var spans []*zipkincore.Span - for i := 0; i < size; i++ { - zs := &zipkincore.Span{} - if err = zs.Read(transport); err != nil { - log.Printf("%s", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - spans = append(spans, zs) - } - - err = transport.ReadListEnd() - if err != nil { - log.Printf("%s", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - //marshal json for debugging purposes - out, _ := json.MarshalIndent(spans, "", " ") - log.Println(string(out)) - - trace, err := UnmarshalZipkinResponse(spans) - if err != nil { - log.Println("Error: ", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - if err = s.tracer.Record(trace); err != nil { - log.Println(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - w.WriteHeader(http.StatusNoContent) -} - -//NewServer returns a new server instance given path to handle -func NewServer(path string) *Server { - return &Server{ - Path: path, - } -} From 3e2662534911fd29d1867a6fb1eb4c22e8ceebb7 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 14:37:08 -0700 Subject: [PATCH 051/154] Update server_test.go to contain new zipkin server unit test --- plugins/inputs/zipkin/server_test.go | 164 +++++++++++++++++++++------ 1 file changed, 130 insertions(+), 34 deletions(-) diff --git a/plugins/inputs/zipkin/server_test.go b/plugins/inputs/zipkin/server_test.go index eba9214e2262e..1218dc1c6754d 100644 --- a/plugins/inputs/zipkin/server_test.go +++ b/plugins/inputs/zipkin/server_test.go @@ -1,52 +1,148 @@ package zipkin -/*func TestMainHandler(t *testing.T) { - dat, err := ioutil.ReadFile("test/threespans.dat") +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "reflect" + "strconv" + "testing" + "time" +) + +type MockTracer struct { + Data Trace + Err error +} + +func (m *MockTracer) Record(t Trace) error { + fmt.Println("Adding trace ", t) + m.Data = t + return nil +} + +func (m *MockTracer) Error(err error) { + m.Err = err +} + +func TestZipkinServer(t *testing.T) { + log.Println("testing server unmarshalling") + dat, err := ioutil.ReadFile("testdata/threespans.dat") if err != nil { - t.Fatal("Could not find threespans.dat") + t.Fatalf("Could not find file %s\n", "test/threespans.dat") } - e := make(chan error, 1) - d := make(chan SpanData, 1) - //f := make(chan struct{}, 1) - s := NewHTTPServer(9411, e, d) + + s := NewServer("/api/v1/spans") + mockTracer := &MockTracer{} + s.tracer = mockTracer w := httptest.NewRecorder() r := httptest.NewRequest( "POST", "http://server.local/api/v1/spans", ioutil.NopCloser( bytes.NewReader(dat))) - handler := s.MainHandler() - handler.ServeHTTP(w, r) + handler := s.SpanHandler + handler(w, r) if w.Code != http.StatusNoContent { t.Errorf("MainHandler did not return StatusNoContent %d", w.Code) } - spans := <-d - if len(spans) != 3 { - t.Fatalf("Expected 3 spans received len(spans) %d", len(spans)) - } - if spans[0].ID != 8090652509916334619 { - t.Errorf("Expected 8090652509916334619 but received ID %d ", spans[0].ID) - } - if spans[0].TraceID != 2505404965370368069 { - t.Errorf("Expected 2505404965370368069 but received TraceID %d ", spans[0].TraceID) - } - if spans[0].Name != "Child" { - t.Errorf("Expected Child but recieved name %s", spans[0].Name) - } - if *(spans[0].ParentID) != 22964302721410078 { - t.Errorf("Expected 22964302721410078 but recieved parent id %d", spans[0].ParentID) - } - serviceName := spans[0].GetBinaryAnnotations()[0].GetHost().GetServiceName() - if serviceName != "trivial" { - t.Errorf("Expected trivial but recieved service name %s", serviceName) - } + got := mockTracer.Data - if spans[0].GetTimestamp() != 1498688360851331 { - t.Errorf("Expected timestamp %d but recieved timestamp %d", 1498688360851331, spans[0].GetTimestamp()) + d := int64(53106) + d1 := int64(50410) + d2 := int64(103680) + parentID1 := int64(22964302721410078) + want := Trace{ + Span{ + Name: "Child", + ID: "8090652509916334619", + TraceID: "2505404965370368069", + ParentID: strconv.FormatInt(parentID1, 10), + Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)), + Duration: time.Duration(d) * time.Microsecond, + //note: []Annotation(nil) is different than + // []Annotation{} + Annotations: []Annotation(nil), + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "trivial", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, + Span{ + Name: "Child", + ID: "103618986556047333", + TraceID: "2505404965370368069", + ParentID: strconv.FormatInt(parentID1, 10), + Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)), + Duration: time.Duration(d1) * time.Microsecond, + Annotations: []Annotation(nil), + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "trivial", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, + Span{ + Name: "Parent", + ID: "22964302721410078", + TraceID: "2505404965370368069", + ParentID: "22964302721410078", + Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)), + Duration: time.Duration(d2) * time.Microsecond, + Annotations: []Annotation{ + Annotation{ + Timestamp: time.Unix(1498688360851325, 0), + Value: "Starting child #0", + Host: "2130706433:0", + ServiceName: "trivial", + }, + Annotation{ + Timestamp: time.Unix(1498688360904545, 0), + Value: "Starting child #1", + Host: "2130706433:0", + ServiceName: "trivial", + }, + Annotation{ + Timestamp: time.Unix(1498688360954992, 0), + Value: "A Log", + Host: "2130706433:0", + ServiceName: "trivial", + }, + }, + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "trivial", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, } - if spans[0].GetDuration() != 53106 { - t.Errorf("Expected duration %d but recieved duration %d", 53106, spans[0].GetDuration()) + /* if !reflect.DeepEqual(got, want) { + t.Fatal("Got != want, Fields weren't unmarshalled correctly") + }*/ + + for i, s := range got { + if !reflect.DeepEqual(s, want[i]) { + fmt.Printf("index %d wasn't equal", i) + fmt.Println(s, want[i]) + t.Fatal("Got != want, Fields weren't unmarshalled correctly") + } } -}*/ +} From 64348cf5c50abcfb3514fe7e21d3aae5dad1217a Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 14:38:13 -0700 Subject: [PATCH 052/154] Add missing constant to zipkin.go --- plugins/inputs/zipkin/zipkin.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 3fac0be226698..ec5b3b2aa863f 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -23,6 +23,8 @@ const ( // DefaultRoute is the default route zipkin uses, and zipkin implementations // expect. DefaultRoute = "/api/v1/spans" + + DefaultShutdownTimeout = 5 ) // TODO: connect methods lexically; method implementations should go right under From b6aa6fb2c3c01e4b920e98449c38eef4a69f8f40 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 18:29:53 -0700 Subject: [PATCH 053/154] Add ability to wait for variable number of points to be written in zipkin plugin test --- plugins/inputs/zipkin/plugin_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index 8e2dfe7b0b73a..af8471502a7f7 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -19,7 +19,12 @@ func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { log.Println("running!") postTestData(t, u.datafile) log.Println("LENGTH:", len(u.expected)) - //acc.Wait(len(u.expected)) + if u.waitPoints == 0 { + acc.Wait(len(u.expected)) + } else { + acc.Wait(u.waitPoints) + } + for _, data := range u.expected { for key, value := range data.expectedValues { switch value.(type) { @@ -56,11 +61,15 @@ func TestZipkin(t *testing.T) { t.Fatal("Failed to start zipkin server") } defer z.Stop() - //t.Fatal("failed!!!") - for _, test := range tests { + /*for _, test := range tests { + log.Println("testing next file...") + log.Println("running test %v\n", test) test.Run(t, &acc) - } + }*/ + + //t.Fatal("ERROR!") + tests[1].Run(t, &acc) } func testBasicSpans(t *testing.T) { @@ -181,7 +190,7 @@ func assertTimeIs(t *testing.T, acc *testutil.Accumulator, return } - t.Errorf("Expected value %d\n got value %d\n", expectedValue, actualValue) + t.Errorf("Expected value %v\n got value %v\n", expectedValue, actualValue) } } @@ -242,6 +251,7 @@ func assertContainsTaggedInt64( for _, pt := range acc.Metrics { log.Println("looping, point is : ", pt) log.Println("point tags are : ", pt.Tags) + log.Println("tags are: ", tags) if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { log.Println("found measurement") for fieldname, value := range pt.Fields { From 1b417c4b04a32d410120a30d78073bf163b73942 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 18:30:28 -0700 Subject: [PATCH 054/154] Add additional unit test case to zipkin plugin, based on go openzipkin example app data --- plugins/inputs/zipkin/unit_tests.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/plugins/inputs/zipkin/unit_tests.go b/plugins/inputs/zipkin/unit_tests.go index c791b31518cf8..a797dbad97535 100644 --- a/plugins/inputs/zipkin/unit_tests.go +++ b/plugins/inputs/zipkin/unit_tests.go @@ -6,6 +6,7 @@ type UnitTest struct { expected []TestData measurement string datafile string + waitPoints int } type TestData struct { @@ -111,4 +112,26 @@ var tests = []UnitTest{ }, }, }, + + // Test data from the cli app + UnitTest{ + measurement: "zipkin", + datafile: "testdata/file.dat", + expected: []TestData{ + { + expectedTags: map[string]string{ + "id": "3383422996321511664", + "parent_id": "4574092882326506380", + "trace_id": "8269862291023777619243463817635710260", + "name": "Concat", + "service_name": "cli", + "annotation_value": "cs", + "endpoint_host": "0:0", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1499817952283903), + }, + }, + }, + }, } From 92464a3a2132614c92e9c0dc971ed232afb9e0c9 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 18:31:16 -0700 Subject: [PATCH 055/154] Remove debug file writing in zipkin server --- plugins/inputs/zipkin/server.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index c3502e5cf082a..267bcca27ff00 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -37,12 +37,19 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) + // err = ioutil.WriteFile(fmt.Sprintf("plugins/inputs/zipkin/testdata/file.dat"), body, 0644) + /* f, err := os.OpenFile("plugins/inputs/zipkin/testdata/file.dat", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) + defer f.Close() + _, err = f.Write(body) + if err != nil { + log.Printf("Could not write to data file") + }*/ + log.Printf("body=%s\n", string(body)) if err != nil { e := fmt.Errorf("Encountered error: %s", err) log.Println(e) s.tracer.Error(e) - //TODO: Change http status that is sent back to client w.WriteHeader(http.StatusInternalServerError) return } @@ -87,6 +94,12 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { //marshal json for debugging purposes out, _ := json.MarshalIndent(spans, "", " ") log.Println(string(out)) + /*f, err = os.OpenFile("plugins/inputs/zipkin/testdata/json/file.json", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) + defer f.Close() + _, err = f.Write(out) + if err != nil { + log.Printf("Could not write to data file") + }*/ trace, err := UnmarshalZipkinResponse(spans) if err != nil { From 5b2bfb7d42d1272a89588eddd3a3db9dff963945 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 11 Jul 2017 18:32:50 -0700 Subject: [PATCH 056/154] Add scraped binary data file from openzipkin example cli app --- plugins/inputs/zipkin/testdata/file.dat | Bin 0 -> 3937 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 plugins/inputs/zipkin/testdata/file.dat diff --git a/plugins/inputs/zipkin/testdata/file.dat b/plugins/inputs/zipkin/testdata/file.dat new file mode 100644 index 0000000000000000000000000000000000000000..585d6b111c420b397af9c2a9fa694c89116cbbf2 GIT binary patch literal 3937 zcmdT{&1=(O7=PEaUHUPn$XHd79F(D8yB3*aGG&y`O*VAvWEa0;`eH-VBumn6_2S8! zNB;qD%I@D*n`=JigNQXxiKnb^iK=0!S(P`c;LGL& zE5jTM`KD2{*SQdYwE{FW3H(o*en+c2O8%iL2bpd zRmbup*!6Luy443$iAVw*X=-2?Dt&Qkes#OEW?OmPd^%Kb6_vUw;kAVZ)>@1qNcS(j3<8rGGD zCg8vk($r@` z6C~dTBf+Ju{uGzy76SLkv2}_w-4|z${rgk4eJ~tf`ULhz=3AvExoD?kSurdm0I$7k zrm&8ABOz{%6l;M8m<0Q>SfMD0R4NTxU_cxQ@{?oa1L+S1MdRJY+JbU9l|yI(zfpch})h{I1+xs?l|BdV#03sPPy;3 zIrXMddo9oG(5mgV+{ShHag=tOBeuP7SKaa+WOwyG7rGQ+dG&MOV|y)lBHLX*owl_^ zw_h+k)+fI_x?V55_ou?{mCP^ZJ~ZB4nWNGf^7=*>^NkBZ;AKm*?_&PhCY3K-9F^Xj fvh9PAkom_XgyUINQ`TDN3 Date: Tue, 11 Jul 2017 18:34:44 -0700 Subject: [PATCH 057/154] Add json representation of binary data from openzipkin cli app to visualize test data --- plugins/inputs/zipkin/testdata/json/file.json | 405 ++++++++++++++++++ 1 file changed, 405 insertions(+) create mode 100644 plugins/inputs/zipkin/testdata/json/file.json diff --git a/plugins/inputs/zipkin/testdata/json/file.json b/plugins/inputs/zipkin/testdata/json/file.json new file mode 100644 index 0000000000000..7c1ad01a5af08 --- /dev/null +++ b/plugins/inputs/zipkin/testdata/json/file.json @@ -0,0 +1,405 @@ +[ + { + "trace_id": 243463817635710260, + "name": "Concat", + "id": 3383422996321511664, + "parent_id": 4574092882326506380, + "annotations": [ + { + "timestamp": 1499817952283903, + "value": "cs", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "timestamp": 1499817952286792, + "value": "cr", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + } + ], + "binary_annotations": [ + { + "key": "http.path", + "value": "L2NvbmNhdC8=", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "http.url", + "value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "peer.hostname", + "value": "bG9jYWxob3N0", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "span.kind", + "value": "Y2xpZW50", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "http.method", + "value": "R0VU", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "http.host", + "value": "bG9jYWxob3N0OjYxMDAx", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + } + ], + "timestamp": 1499817952283903, + "duration": 2888, + "trace_id_high": 8269862291023777619 + }, + { + "trace_id": 243463817635710260, + "name": "Sum", + "id": 6036416808826525494, + "parent_id": 4574092882326506380, + "annotations": [ + { + "timestamp": 1499817952286828, + "value": "cs", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "timestamp": 1499817952333847, + "value": "cr", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + } + ], + "binary_annotations": [ + { + "key": "span.kind", + "value": "Y2xpZW50", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "http.method", + "value": "R0VU", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "http.host", + "value": "bG9jYWxob3N0OjYxMDAx", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "http.path", + "value": "L3N1bS8=", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "http.url", + "value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9zdW0v", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "key": "peer.hostname", + "value": "bG9jYWxob3N0", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + } + ], + "timestamp": 1499817952286828, + "duration": 47019, + "trace_id_high": 8269862291023777619 + }, + { + "trace_id": 243463817635710260, + "name": "Run", + "id": 4574092882326506380, + "annotations": [ + { + "timestamp": 1499817952283897, + "value": "Call Concat", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + }, + { + "timestamp": 1499817952286824, + "value": "Call Sum", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + } + ], + "binary_annotations": [ + { + "key": "lc", + "value": "Y2xp", + "annotation_type": "STRING", + "host": { + "ipv4": 0, + "port": 0, + "service_name": "cli" + } + } + ], + "timestamp": 1499817952283881, + "duration": 50014, + "trace_id_high": 8269862291023777619 + } +][ + { + "trace_id": 243463817635710260, + "name": "myComplexQuery", + "id": 4254041670140233539, + "parent_id": 8633460035494236932, + "annotations": [ + { + "timestamp": 1499817952307418, + "value": "cs", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "timestamp": 1499817952331909, + "value": "cr", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + } + ], + "binary_annotations": [ + { + "key": "sa", + "value": "UG9zdGdyZVNRTA==", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": 5432, + "service_name": "PostgreSQL", + "ipv6": "AAAAAAAAAAAAAAAAAAAAAQ==" + } + }, + { + "key": "query", + "value": "U0VMRUNUIHJlY2lwZXMgRlJPTSBjb29rYm9vayBXSEVSRSB0b3BpYyA9ICd3b3JsZCBkb21pbmF0aW9uJw==", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "key": "span.kind", + "value": "cmVzb3VyY2U=", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "key": "peer.service", + "value": "UG9zdGdyZVNRTA==", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "key": "peer.hostname", + "value": "bG9jYWxob3N0", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "key": "peer.port", + "value": "AAAVOA==", + "annotation_type": "I32", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + } + ], + "timestamp": 1499817952307418, + "duration": 24491, + "trace_id_high": 8269862291023777619 + }, + { + "trace_id": 243463817635710260, + "name": "Sum", + "id": 8633460035494236932, + "parent_id": 6036416808826525494, + "annotations": [ + { + "timestamp": 1499817952287147, + "value": "sr", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "timestamp": 1499817952333348, + "value": "ss", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "timestamp": 1499817952296675, + "value": "MyEventAnnotation", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + } + ], + "binary_annotations": [ + { + "key": "span.kind", + "value": "c2VydmVy", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "key": "serverSide", + "value": "aGVyZQ==", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "key": "service", + "value": "c3ZjMg==", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "key": "key1", + "value": "dmFsdWUx", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + }, + { + "key": "key2", + "value": "AAAAAgAAAAA=", + "annotation_type": "I32", + "host": { + "ipv4": 2130706433, + "port": -4534, + "service_name": "svc2" + } + } + ], + "trace_id_high": 8269862291023777619 + } +] From 7bab3df79dd8b5c839a39ffc2ef9a19fdfdc54bf Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 10:53:11 -0700 Subject: [PATCH 058/154] Rename cli microservice data file --- .../testdata/{file.dat => cli_microservice.dat} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/testdata/{file.dat => cli_microservice.dat} (100%) diff --git a/plugins/inputs/zipkin/testdata/file.dat b/plugins/inputs/zipkin/testdata/cli_microservice.dat similarity index 100% rename from plugins/inputs/zipkin/testdata/file.dat rename to plugins/inputs/zipkin/testdata/cli_microservice.dat From e51b1a0c0dee58b0bb29ebb9106227a3c5c9bb95 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 11:29:46 -0700 Subject: [PATCH 059/154] Rename cli microservice json file --- .../zipkin/testdata/json/{file.json => cli_microservice.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/testdata/json/{file.json => cli_microservice.json} (100%) diff --git a/plugins/inputs/zipkin/testdata/json/file.json b/plugins/inputs/zipkin/testdata/json/cli_microservice.json similarity index 100% rename from plugins/inputs/zipkin/testdata/json/file.json rename to plugins/inputs/zipkin/testdata/json/cli_microservice.json From 716b2949fb11f2e18b4c18a2e51648a7b97f0327 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 12 Jul 2017 14:27:07 -0700 Subject: [PATCH 060/154] Add in loop to run all tests, add additional logs Signed-off-by: Adam Perlin --- plugins/inputs/zipkin/plugin_test.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index af8471502a7f7..259f40016fea3 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -13,8 +13,6 @@ import ( "github.com/influxdata/telegraf/testutil" ) -//expectedTags[0] expectedValues[0] - func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { log.Println("running!") postTestData(t, u.datafile) @@ -62,14 +60,11 @@ func TestZipkin(t *testing.T) { } defer z.Stop() - /*for _, test := range tests { - log.Println("testing next file...") - log.Println("running test %v\n", test) + for _, test := range tests { test.Run(t, &acc) - }*/ + } //t.Fatal("ERROR!") - tests[1].Run(t, &acc) } func testBasicSpans(t *testing.T) { @@ -251,11 +246,13 @@ func assertContainsTaggedInt64( for _, pt := range acc.Metrics { log.Println("looping, point is : ", pt) log.Println("point tags are : ", pt.Tags) + log.Println("point fields are:", pt.Fields) log.Println("tags are: ", tags) if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { log.Println("found measurement") for fieldname, value := range pt.Fields { - fmt.Println("looping through fields") + fmt.Println("looping through fields, fieldname is: ", fieldname) + fmt.Println("user input field is: ", field) if fieldname == field { fmt.Println("found field: ", field) actualValue = value @@ -321,6 +318,8 @@ func postTestData(t *testing.T, datafile string) { if err != nil { t.Fatal("bad http request") } + + req.Header.Set("Content-Type", "application/x-thrift") client := &http.Client{} _, err = client.Do(req) if err != nil { From 2250eb7484f1f7e02f25cb46a20078d4aeb3713f Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 14:29:23 -0700 Subject: [PATCH 061/154] Add additional debugging information to server logs in zipkin plugin --- plugins/inputs/zipkin/server.go | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index 267bcca27ff00..16e13333fe794 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -37,17 +37,17 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) - // err = ioutil.WriteFile(fmt.Sprintf("plugins/inputs/zipkin/testdata/file.dat"), body, 0644) - /* f, err := os.OpenFile("plugins/inputs/zipkin/testdata/file.dat", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) - defer f.Close() - _, err = f.Write(body) - if err != nil { - log.Printf("Could not write to data file") - }*/ + + /*f, err := os.OpenFile("plugins/inputs/zipkin/testdata/file.dat", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) + defer f.Close() + _, err = f.Write(body) + if err != nil { + log.Printf("Could not write to data file") + }*/ log.Printf("body=%s\n", string(body)) if err != nil { - e := fmt.Errorf("Encountered error: %s", err) + e := fmt.Errorf("Encountered error reading: %s", err) log.Println(e) s.tracer.Error(e) w.WriteHeader(http.StatusInternalServerError) @@ -56,7 +56,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { buffer := thrift.NewTMemoryBuffer() if _, err = buffer.Write(body); err != nil { - log.Println(err) + log.Println("Error in buffer write: ", err) s.tracer.Error(err) w.WriteHeader(http.StatusInternalServerError) return @@ -65,7 +65,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { transport := thrift.NewTBinaryProtocolTransport(buffer) _, size, err := transport.ReadListBegin() if err != nil { - log.Printf("%s", err) + log.Printf("Error in ReadListBegin: %s", err) s.tracer.Error(err) w.WriteHeader(http.StatusInternalServerError) return @@ -75,7 +75,7 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { for i := 0; i < size; i++ { zs := &zipkincore.Span{} if err = zs.Read(transport); err != nil { - log.Printf("%s", err) + log.Printf("Error reading into zipkin struct: %s", err) s.tracer.Error(err) w.WriteHeader(http.StatusInternalServerError) return @@ -94,12 +94,13 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { //marshal json for debugging purposes out, _ := json.MarshalIndent(spans, "", " ") log.Println(string(out)) - /*f, err = os.OpenFile("plugins/inputs/zipkin/testdata/json/file.json", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) - defer f.Close() - _, err = f.Write(out) - if err != nil { - log.Printf("Could not write to data file") - }*/ + + /* f, err = os.OpenFile("plugins/inputs/zipkin/testdata/json/file.json", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) + defer f.Close() + _, err = f.Write(out) + if err != nil { + log.Printf("Could not write to data file") + }*/ trace, err := UnmarshalZipkinResponse(spans) if err != nil { From ec4d600be978ec38b9f69d02f3da828811e4d2e3 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 14:35:35 -0700 Subject: [PATCH 062/154] Add additional unit test data to zipkin plugin unit test from distributed trace repo Signed-off-by: Ayrdrie Palmer --- plugins/inputs/zipkin/unit_tests.go | 42 +++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/unit_tests.go b/plugins/inputs/zipkin/unit_tests.go index a797dbad97535..825b6b6f18617 100644 --- a/plugins/inputs/zipkin/unit_tests.go +++ b/plugins/inputs/zipkin/unit_tests.go @@ -113,10 +113,11 @@ var tests = []UnitTest{ }, }, - // Test data from the cli app + // Test data from zipkin cli app: + //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services UnitTest{ measurement: "zipkin", - datafile: "testdata/file.dat", + datafile: "testdata/cli_microservice.dat", expected: []TestData{ { expectedTags: map[string]string{ @@ -134,4 +135,41 @@ var tests = []UnitTest{ }, }, }, + + // Test data from distributed trace repo sample json + // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json + UnitTest{ + measurement: "zipkin", + datafile: "testdata/distributed_trace_sample.dat", + expected: []TestData{ + { + expectedTags: map[string]string{ + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", + "annotation_value": "cs", + "endpoint_host": "0:9410", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1433330263415871), + }, + }, + { + expectedTags: map[string]string{ + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", + "annotation_value": "cr", + "endpoint_host": "0:9410", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1433330263415872), + }, + }, + }, + }, } From ae105e8e74ec5c0addf75ffe104173a31dd66fcb Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 12 Jul 2017 14:36:55 -0700 Subject: [PATCH 063/154] Remove outdated comments from zipkin plugin Signed-off-by: Adam Perlin --- plugins/inputs/zipkin/zipkin.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index ec5b3b2aa863f..10aac90583703 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -138,17 +138,17 @@ func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { s.Timestamp = time.Unix(0, span.GetTimestamp()*int64(time.Microsecond)) } - //duration, parent id, and trace id high are all optional fields. - // below, we check to see if any of these fields are non-zero, and if they are, - // we set the repsective fields in our Span structure to the address of - // these values. Otherwise, we just leave them as nil - duration := time.Duration(span.GetDuration()) // fmt.Println("Duration: ", duration) s.Duration = duration * time.Microsecond parentID := span.GetParentID() // fmt.Println("Parent ID: ", parentID) + + // A parent ID of 0 means that this is a parent span. In this case, + // we set the parent ID of the span to be its own id, so it points to + // itself. + if parentID == 0 { s.ParentID = s.ID } else { From 74db3eec94b58294164592700bfb01c990d2085d Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 14:37:52 -0700 Subject: [PATCH 064/154] Change formatting in example data json file in zipkin plugin --- plugins/inputs/zipkin/testdata/json/cli_microservice.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/testdata/json/cli_microservice.json b/plugins/inputs/zipkin/testdata/json/cli_microservice.json index 7c1ad01a5af08..af9446b4a9e81 100644 --- a/plugins/inputs/zipkin/testdata/json/cli_microservice.json +++ b/plugins/inputs/zipkin/testdata/json/cli_microservice.json @@ -221,7 +221,9 @@ "duration": 50014, "trace_id_high": 8269862291023777619 } -][ +] + +[ { "trace_id": 243463817635710260, "name": "myComplexQuery", From e2d4463081b975e41bc584c8ae44004e6d801254 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 12 Jul 2017 14:40:14 -0700 Subject: [PATCH 065/154] Add additional binary unit test data with json representation to zipkin plugin tests Signed-off-by: Adam Perlin --- .../testdata/distributed_trace_sample.dat | Bin 0 -> 191 bytes .../json/distributed_trace_sample.json | 30 ++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 plugins/inputs/zipkin/testdata/distributed_trace_sample.dat create mode 100644 plugins/inputs/zipkin/testdata/json/distributed_trace_sample.json diff --git a/plugins/inputs/zipkin/testdata/distributed_trace_sample.dat b/plugins/inputs/zipkin/testdata/distributed_trace_sample.dat new file mode 100644 index 0000000000000000000000000000000000000000..9ce376f11738e8b9ee3a27b08246aa0ea86601a8 GIT binary patch literal 191 zcmd;KU|?Y6VqlER&^{#da!vy`12a&BBR4TKPcNl3g^Pg&Cd1Fb#sidM0xD);6`yTt z6l2fLzyuOWF6Loi<^T!<0UHC8${~m$BI)_MRhb3ZnR&V;sl_G9Ihm<>B@7G*{SGkw SMWpEGXW#(2186lTBLe__wj%KW literal 0 HcmV?d00001 diff --git a/plugins/inputs/zipkin/testdata/json/distributed_trace_sample.json b/plugins/inputs/zipkin/testdata/json/distributed_trace_sample.json new file mode 100644 index 0000000000000..a92856e060d19 --- /dev/null +++ b/plugins/inputs/zipkin/testdata/json/distributed_trace_sample.json @@ -0,0 +1,30 @@ +{ + "trace_id": 6802735349851856000, + "name": "main.dud", + "id": 6802735349851856000, + "parent_id": null, + "annotations": [ + { + "timestamp": 1433330263415871, + "value": "cs", + "host": { + "ipv4": 0, + "port": 9410, + "service_name": "go-zipkin-testclient" + }, + "duration": null + }, + { + "timestamp": 1433330263415872, + "value": "cr", + "host": { + "ipv4": 0, + "port": 9410, + "service_name": "go-zipkin-testclient" + }, + "duration": null + } + ], + "binary_annotations": [], + "debug": true +} From e55bce32430771d9a1232f0e648e375ef393db99 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 14:41:56 -0700 Subject: [PATCH 066/154] Add go script to convert json to zipkin thrift protocol --- plugins/inputs/zipkin/cmd/thrift_serialize.go | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 plugins/inputs/zipkin/cmd/thrift_serialize.go diff --git a/plugins/inputs/zipkin/cmd/thrift_serialize.go b/plugins/inputs/zipkin/cmd/thrift_serialize.go new file mode 100644 index 0000000000000..9f77da1a89f27 --- /dev/null +++ b/plugins/inputs/zipkin/cmd/thrift_serialize.go @@ -0,0 +1,114 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + + "github.com/apache/thrift/lib/go/thrift" + "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" +) + +var ( + filename *string + outFileName *string +) + +/*func init() { + filename = flag.String("input", "", usage) + outFileName = flag.String("output", "", usage) +} + +const usage = `./json_serialize -input -output output`*/ + +func main() { + + flag.Parse() + b, err := ioutil.ReadFile("../testdata/json/distributed_trace_sample.json") + if err != nil { + log.Fatalf("error: %v\n", err) + } + + var spans []*zipkincore.Span + span, err := serializeJSON(b) + spans = append(spans, span) + + if err != nil { + log.Fatalf("error: %v\n", err) + } + + fmt.Println(spans) + + buf := thrift.NewTMemoryBuffer() + transport := thrift.NewTBinaryProtocolTransport(buf) + + if err = transport.WriteListBegin(thrift.STRUCT, len(spans)); err != nil { + panic(err) + } + + for _, s := range spans { + err = s.Write(transport) + if err != nil { + panic(err) + } + } + + if err = transport.WriteListEnd(); err != nil { + panic(err) + } + + log.Println(buf.Buffer.String()) + + ioutil.WriteFile("../testdata/out.dat", buf.Buffer.Bytes(), 0644) + + b, err = ioutil.ReadFile("../testdata/out.dat") + + //log.Println("read bytes: ", b) + if err != nil { + log.Printf("%v\n", err) + } + deserializeThrift(b) +} + +func serializeJSON(b []byte) (*zipkincore.Span, error) { + var span *zipkincore.Span + err := json.Unmarshal(b, &span) + return span, err +} + +func deserializeThrift(b []byte) { + buffer := thrift.NewTMemoryBuffer() + if _, err := buffer.Write(b); err != nil { + log.Println("Error in buffer write: ", err) + return + } + + transport := thrift.NewTBinaryProtocolTransport(buffer) + _, size, err := transport.ReadListBegin() + if err != nil { + log.Printf("Error in ReadListBegin: %s", err) + return + } + + var spans []*zipkincore.Span + for i := 0; i < size; i++ { + zs := &zipkincore.Span{} + if err = zs.Read(transport); err != nil { + log.Printf("Error reading into zipkin struct: %s", err) + return + } + spans = append(spans, zs) + } + + err = transport.ReadListEnd() + if err != nil { + log.Printf("%s", err) + return + } + + //marshal json for debugging purposes + out, _ := json.MarshalIndent(spans, "", " ") + log.Println(string(out)) +} From 970569c6979659c37fe33cd674d60c99ca558c0c Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 14:44:41 -0700 Subject: [PATCH 067/154] Add json representation of testdata/threespans.dat to zipkin plugin --- .../zipkin/testdata/json/threespans.json | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 plugins/inputs/zipkin/testdata/json/threespans.json diff --git a/plugins/inputs/zipkin/testdata/json/threespans.json b/plugins/inputs/zipkin/testdata/json/threespans.json new file mode 100644 index 0000000000000..5809977dfa498 --- /dev/null +++ b/plugins/inputs/zipkin/testdata/json/threespans.json @@ -0,0 +1,92 @@ +[ + { + "trace_id": 2505404965370368069, + "name": "Child", + "id": 8090652509916334619, + "parent_id": 22964302721410078, + "annotations": [], + "binary_annotations": [ + { + "key": "lc", + "value": "dHJpdmlhbA==", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": 0, + "service_name": "trivial" + } + } + ], + "timestamp": 1498688360851331, + "duration": 53106 + }, + { + "trace_id": 2505404965370368069, + "name": "Child", + "id": 103618986556047333, + "parent_id": 22964302721410078, + "annotations": [], + "binary_annotations": [ + { + "key": "lc", + "value": "dHJpdmlhbA==", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": 0, + "service_name": "trivial" + } + } + ], + "timestamp": 1498688360904552, + "duration": 50410 + }, + { + "trace_id": 2505404965370368069, + "name": "Parent", + "id": 22964302721410078, + "annotations": [ + { + "timestamp": 1498688360851325, + "value": "Starting child #0", + "host": { + "ipv4": 2130706433, + "port": 0, + "service_name": "trivial" + } + }, + { + "timestamp": 1498688360904545, + "value": "Starting child #1", + "host": { + "ipv4": 2130706433, + "port": 0, + "service_name": "trivial" + } + }, + { + "timestamp": 1498688360954992, + "value": "A Log", + "host": { + "ipv4": 2130706433, + "port": 0, + "service_name": "trivial" + } + } + ], + "binary_annotations": [ + { + "key": "lc", + "value": "dHJpdmlhbA==", + "annotation_type": "STRING", + "host": { + "ipv4": 2130706433, + "port": 0, + "service_name": "trivial" + } + } + ], + "timestamp": 1498688360851318, + "duration": 103680 + } +] From 76fba59a167a9de360b10ad207748a74dd27f5aa Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 14:47:21 -0700 Subject: [PATCH 068/154] Remove copy of first unit test for threespans.dat in zipkin plugin Signed-off-by: Ayrdrie Palmer --- plugins/inputs/zipkin/plugin_test.go | 106 --------------------------- 1 file changed, 106 deletions(-) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index 259f40016fea3..1140c16124905 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -67,112 +67,6 @@ func TestZipkin(t *testing.T) { //t.Fatal("ERROR!") } -func testBasicSpans(t *testing.T) { - log.Println("testing basic spans...") - var acc testutil.Accumulator - z := &Zipkin{ - Path: "/api/v1/test", - Port: 9411, - } - err := z.Start(&acc) - if err != nil { - t.Fatal("Failed to start zipkin server") - } - defer z.Stop() - - postTestData(t, "testdata/threespans.dat") - acc.Wait(6) - - if len(acc.Errors) != 0 { - for _, e := range acc.Errors { - fmt.Println(e) - } - t.Fatal("Errors were added during request") - } - - // Actual testing: - - // The tags we will be querying by: - - // Test for the first span in the trace: - tags := map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "2130706433:0", - "key": "lc", - "type": "STRING", - } - - // assertContainsTaggedDuration asserts that the specified field which corresponds to `tags` has - // the specified value. In this case, we are testing that the measurement zipkin with tags `tags` has a - // field called `duration` with the value 53106 microseconds - - assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(53106)*time.Microsecond, tags) - - // Test for the second span in the trace: - - //Update tags in order to perform our next test - - tags = map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "2130706433:0", - "key": "lc", - "type": "STRING", - } - - //Similar test as above, but with different tags - assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(50410)*time.Microsecond, tags) - - //test for the third span in the trace (with three annotations) - - tags = map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "Starting child #0", - "endpoint_host": "2130706433:0", - } - - // test for existence of annotation specific fields - assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360851325, tags) - - // test for existence of annotation specific fields - tags["annotation_value"] = "Starting child #1" - assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360904545, tags) - - //test for existence of annotation specific fields - tags["annotation_value"] = "A Log" - assertContainsTaggedInt64(t, &acc, "zipkin", "annotation_timestamp", 1498688360954992, tags) - - tags = map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "2130706433:0", - "key": "lc", - "type": "STRING", - } - // test for existence of span time stamp in third span, using binary annotation specific values. - assertContainsTaggedDuration(t, &acc, "zipkin", "duration", time.Duration(103680)*time.Microsecond, tags) - log.Println("end") - log.Println("TIMESTAMP: ", acc.Metrics[5].Time) - assertTimeIs(t, &acc, "zipkin", time.Unix(1498688360, 851318*int64(time.Microsecond)), tags) -} - func assertTimeIs(t *testing.T, acc *testutil.Accumulator, measurement string, expectedValue time.Time, From ce097268b00ca6e8d60953f6eb4fb842eb274f79 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 12 Jul 2017 14:56:33 -0700 Subject: [PATCH 069/154] Add clarifying comments to unit test structs in zipkin plugin Signed-off-by: Adam Perlin --- plugins/inputs/zipkin/unit_tests.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plugins/inputs/zipkin/unit_tests.go b/plugins/inputs/zipkin/unit_tests.go index 825b6b6f18617..e0bcf5be07430 100644 --- a/plugins/inputs/zipkin/unit_tests.go +++ b/plugins/inputs/zipkin/unit_tests.go @@ -2,6 +2,8 @@ package zipkin import "time" +// UnitTest represents contains expected test values and a data file to be +// written to the zipkin http server. type UnitTest struct { expected []TestData measurement string @@ -9,16 +11,23 @@ type UnitTest struct { waitPoints int } +// TestData contains the expected tags and values that the telegraf plugin +// should output type TestData struct { expectedTags map[string]string expectedValues map[string]interface{} } +// Store all unit tests in an array to allow for iteration over all tests var tests = []UnitTest{ UnitTest{ measurement: "zipkin", datafile: "testdata/threespans.dat", expected: []TestData{ + // zipkin data points are stored in InfluxDB tagged partly //annotation specific + //values, and partly on span specific values, + // so there are many repeated tags. Fields have very similar tags, which is why + // tags are relatively redundant in these tests. { expectedTags: map[string]string{ "id": "8090652509916334619", From 4f3c91d026402ae51456dcc42bafb95e6e29ba2b Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 12 Jul 2017 17:41:48 -0600 Subject: [PATCH 070/154] Remove unsued meta annotation struct add comments to remove linter errors --- plugins/inputs/zipkin/zipkin.go | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 10aac90583703..e1e6d6f309e1e 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -24,6 +24,8 @@ const ( // expect. DefaultRoute = "/api/v1/spans" + // DefaultShutdownTimeout is the max amount of time telegraf will wait + // for the plugin to shutdown DefaultShutdownTimeout = 5 ) @@ -54,16 +56,6 @@ type BinaryAnnotation struct { Type string } -func (b BinaryAnnotation) ToMeta() MetaAnnotation { - return MetaAnnotation{ - Key: b.Key, - Value: b.Value, - Host: b.Host, - ServiceName: b.ServiceName, - Type: b.Type, - } -} - // Annotation represents an ordinary zipkin annotation. It contains the data fields // which will become fields/tags in influxdb type Annotation struct { @@ -73,27 +65,6 @@ type Annotation struct { ServiceName string } -func (a Annotation) ToMeta() MetaAnnotation { - return MetaAnnotation{ - Key: a.Value, - Value: "NONE", - Type: "NONE", - Timestamp: a.Timestamp, - Host: a.Host, - ServiceName: a.ServiceName, - } -} - -type MetaAnnotation struct { - Key string - Value string - Type string - Timestamp time.Time - Host string - HostIPV6 string - ServiceName string -} - //Span represents a specific zipkin span. It holds the majority of the same // data as a zipkin span sent via the thrift protocol, but is presented in a // format which is more straightforward for storage purposes. From d82c8a381458c03f1dee8cd2620e51e105be2602 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 12 Jul 2017 18:19:31 -0600 Subject: [PATCH 071/154] Move thrift unmarshalling logic into separate function in zipkin plugin Signed-off-by: Adam Perlin --- plugins/inputs/zipkin/server.go | 67 ++++++++++++++++----------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/server.go index 16e13333fe794..3b75d59b0200b 100644 --- a/plugins/inputs/zipkin/server.go +++ b/plugins/inputs/zipkin/server.go @@ -24,12 +24,42 @@ type Server struct { // Register allows server to implement the Service interface. Server's register metod // registers its handler on mux, and sets the servers tracer with tracer func (s *Server) Register(router *mux.Router, tracer Tracer) error { - // TODO: potentially move router into Server if appropriate router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") s.tracer = tracer return nil } +func unmarshalThrift(body []byte) ([]*zipkincore.Span, error) { + buffer := thrift.NewTMemoryBuffer() + if _, err := buffer.Write(body); err != nil { + log.Println("Error in buffer write: ", err) + return nil, err + } + + transport := thrift.NewTBinaryProtocolTransport(buffer) + _, size, err := transport.ReadListBegin() + if err != nil { + log.Printf("Error in ReadListBegin: %s", err) + return nil, err + } + var spans []*zipkincore.Span + for i := 0; i < size; i++ { + zs := &zipkincore.Span{} + if err = zs.Read(transport); err != nil { + log.Printf("Error reading into zipkin struct: %s", err) + return nil, err + } + spans = append(spans, zs) + } + + err = transport.ReadListEnd() + if err != nil { + log.Printf("%s", err) + return nil, err + } + return spans, nil +} + // SpanHandler is the handler Server uses for handling zipkin POST requests func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { log.Printf("Received request from: %s", r.URL.String()) @@ -45,7 +75,6 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { log.Printf("Could not write to data file") }*/ - log.Printf("body=%s\n", string(body)) if err != nil { e := fmt.Errorf("Encountered error reading: %s", err) log.Println(e) @@ -53,39 +82,9 @@ func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) return } - - buffer := thrift.NewTMemoryBuffer() - if _, err = buffer.Write(body); err != nil { - log.Println("Error in buffer write: ", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - transport := thrift.NewTBinaryProtocolTransport(buffer) - _, size, err := transport.ReadListBegin() - if err != nil { - log.Printf("Error in ReadListBegin: %s", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - var spans []*zipkincore.Span - for i := 0; i < size; i++ { - zs := &zipkincore.Span{} - if err = zs.Read(transport); err != nil { - log.Printf("Error reading into zipkin struct: %s", err) - s.tracer.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - spans = append(spans, zs) - } - - err = transport.ReadListEnd() + spans, err := unmarshalThrift(body) if err != nil { - log.Printf("%s", err) + log.Println("Error: ", err) s.tracer.Error(err) w.WriteHeader(http.StatusInternalServerError) return From 150996d7808d13331846377913f4fb14528d1f5c Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 18:22:19 -0600 Subject: [PATCH 072/154] Add go doc comments to zipkin.go Signed-off-by: Ayrdrie Palmer --- plugins/inputs/zipkin/zipkin.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index e1e6d6f309e1e..e8d9306cdc41d 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -29,10 +29,6 @@ const ( DefaultShutdownTimeout = 5 ) -// TODO: connect methods lexically; method implementations should go right under -// struct definition. Maybe change order of structs, organize where structs are -// declared based on when their type is used - // Tracer represents a type which can record zipkin trace data as well as // any accompanying errors, and process that data. type Tracer interface { @@ -155,9 +151,6 @@ func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { return formatted } -func (s Span) MergeAnnotations() { -} - // UnmarshalBinaryAnnotations is very similar to UnmarshalAnnotations, but it // Unmarshalls zipkincore.BinaryAnnotations instead of the normal zipkincore.Annotation func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryAnnotation, error) { @@ -182,10 +175,16 @@ func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]B return formatted, nil } +// LineProtocolConverter implements the Tracer interface; it is a +// type meant to encapsulate the storage of zipkin tracing data in +// telegraf as line protocol. type LineProtocolConverter struct { acc telegraf.Accumulator } +// Record is LineProtocolConverter's implementation of the Record method of +// the Tracer iterface; it takes a span as input, and adds it to an internal +// telegraf.Accumulator. func (l *LineProtocolConverter) Record(t Trace) error { log.Printf("received trace: %#+v\n", t) //log.Printf("...But converter implementation is not yet done. Here's some example data") @@ -244,6 +243,8 @@ func (l *LineProtocolConverter) Error(err error) { l.acc.AddError(err) } +// NewLineProtocolConverter returns an instance of LineProtocolConverter that +// will add to the given telegraf.Accumulator func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { return &LineProtocolConverter{ acc: acc, From ee32c864ab22637a96b31f044afedf4a072d4586 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 12 Jul 2017 21:19:46 -0700 Subject: [PATCH 073/154] Remove unnecessary variable in zipkin.go, update sample config file --- plugins/inputs/zipkin/zipkin.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index e8d9306cdc41d..21b6442754e9a 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -165,9 +165,8 @@ func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]B b.Host, b.ServiceName = "", "" } - val := annotation.GetValue() b.Key = annotation.GetKey() - b.Value = string(val) + b.Value = string(annotation.GetValue()) b.Type = annotation.GetAnnotationType().String() formatted = append(formatted, b) } @@ -187,7 +186,6 @@ type LineProtocolConverter struct { // telegraf.Accumulator. func (l *LineProtocolConverter) Record(t Trace) error { log.Printf("received trace: %#+v\n", t) - //log.Printf("...But converter implementation is not yet done. Here's some example data") log.Printf("Writing to telegraf...\n") for _, s := range t { for _, a := range s.Annotations { @@ -236,7 +234,6 @@ func (l *LineProtocolConverter) Record(t Trace) error { } return nil - } func (l *LineProtocolConverter) Error(err error) { @@ -254,7 +251,7 @@ func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { const sampleConfig = ` ## # path = /path/your/zipkin/impl/posts/to - # port = + # port = ` // Zipkin is a telegraf configuration structure for the zipkin input plugin, From f4f5882566525ec20329dd956b6a942658aeeb74 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 13 Jul 2017 11:06:12 -0600 Subject: [PATCH 074/154] Update source code with go format --- plugins/inputs/all/all.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index cdfe1fcce002c..84c320fed5d22 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -89,6 +89,6 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/webhooks" _ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters" _ "github.com/influxdata/telegraf/plugins/inputs/zfs" + _ "github.com/influxdata/telegraf/plugins/inputs/zipkin" _ "github.com/influxdata/telegraf/plugins/inputs/zookeeper" - _ "github.com/influxdata/telegraf/plugins/inputs/zipkin" ) From c6a8671cd178a13a14f8c2b9a243f0a85ea99752 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 13 Jul 2017 13:47:45 -0600 Subject: [PATCH 075/154] Rename zipkin server.go to handler.go --- plugins/inputs/zipkin/{server.go => handler.go} | 0 plugins/inputs/zipkin/{server_test.go => handler_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/{server.go => handler.go} (100%) rename plugins/inputs/zipkin/{server_test.go => handler_test.go} (100%) diff --git a/plugins/inputs/zipkin/server.go b/plugins/inputs/zipkin/handler.go similarity index 100% rename from plugins/inputs/zipkin/server.go rename to plugins/inputs/zipkin/handler.go diff --git a/plugins/inputs/zipkin/server_test.go b/plugins/inputs/zipkin/handler_test.go similarity index 100% rename from plugins/inputs/zipkin/server_test.go rename to plugins/inputs/zipkin/handler_test.go From ac56b04aa66914fac1a3c5003e51607c348eaedc Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 13 Jul 2017 15:23:14 -0600 Subject: [PATCH 076/154] Refactor zipkin plugin --- plugins/inputs/zipkin/convert.go | 102 ++++++++++++++++++++ plugins/inputs/zipkin/handler.go | 130 ++++++++++---------------- plugins/inputs/zipkin/handler_test.go | 16 ++-- plugins/inputs/zipkin/zipkin.go | 126 +++---------------------- 4 files changed, 174 insertions(+), 200 deletions(-) create mode 100644 plugins/inputs/zipkin/convert.go diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go new file mode 100644 index 0000000000000..a6c796167e8eb --- /dev/null +++ b/plugins/inputs/zipkin/convert.go @@ -0,0 +1,102 @@ +package zipkin + +import ( + "strconv" + "time" + + "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" +) + +// NewTrace converts a slice of []*zipkincore.Spans into a new Trace +func NewTrace(spans []*zipkincore.Span) (Trace, error) { + var trace Trace + for _, span := range spans { + s := Span{} + s.ID = strconv.FormatInt(span.GetID(), 10) + s.TraceID = strconv.FormatInt(span.GetTraceID(), 10) + if span.GetTraceIDHigh() != 0 { + s.TraceID = strconv.FormatInt(span.GetTraceIDHigh(), 10) + s.TraceID + } + + s.Annotations = NewAnnotations(span.GetAnnotations()) + + var err error + s.BinaryAnnotations, err = NewBinaryAnnotations(span.GetBinaryAnnotations()) + if err != nil { + return nil, err + } + s.Name = span.GetName() + if span.GetTimestamp() == 0 { + s.Timestamp = time.Now() + } else { + s.Timestamp = time.Unix(0, span.GetTimestamp()*int64(time.Microsecond)) + } + + duration := time.Duration(span.GetDuration()) + // fmt.Println("Duration: ", duration) + s.Duration = duration * time.Microsecond + + parentID := span.GetParentID() + // fmt.Println("Parent ID: ", parentID) + + // A parent ID of 0 means that this is a parent span. In this case, + // we set the parent ID of the span to be its own id, so it points to + // itself. + + if parentID == 0 { + s.ParentID = s.ID + } else { + s.ParentID = strconv.FormatInt(parentID, 10) + } + + // fmt.Println("ID:", s.ID) + trace = append(trace, s) + } + + return trace, nil +} + +// NewAnnotations converts a slice of *zipkincore.Annotation into a slice +// of new Annotations +func NewAnnotations(annotations []*zipkincore.Annotation) []Annotation { + var formatted []Annotation + for _, annotation := range annotations { + a := Annotation{} + endpoint := annotation.GetHost() + if endpoint != nil { + a.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) + a.ServiceName = endpoint.GetServiceName() + } else { + a.Host, a.ServiceName = "", "" + } + + a.Timestamp = time.Unix(annotation.GetTimestamp(), 0) + a.Value = annotation.GetValue() + formatted = append(formatted, a) + } + //fmt.Println("formatted annotations: ", formatted) + return formatted +} + +// NewBinaryAnnotations is very similar to NewAnnotations, but it +// converts zipkincore.BinaryAnnotations instead of the normal zipkincore.Annotation +func NewBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryAnnotation, error) { + var formatted []BinaryAnnotation + for _, annotation := range annotations { + b := BinaryAnnotation{} + endpoint := annotation.GetHost() + if endpoint != nil { + b.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) + b.ServiceName = endpoint.GetServiceName() + } else { + b.Host, b.ServiceName = "", "" + } + + b.Key = annotation.GetKey() + b.Value = string(annotation.GetValue()) + b.Type = annotation.GetAnnotationType().String() + formatted = append(formatted, b) + } + + return formatted, nil +} diff --git a/plugins/inputs/zipkin/handler.go b/plugins/inputs/zipkin/handler.go index 3b75d59b0200b..6ba20d12c03b0 100644 --- a/plugins/inputs/zipkin/handler.go +++ b/plugins/inputs/zipkin/handler.go @@ -1,10 +1,7 @@ package zipkin import ( - "encoding/json" - "fmt" "io/ioutil" - "log" "net/http" "sync" @@ -13,113 +10,86 @@ import ( "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) -// Server is an implementation of tracer which is a helper for running an -// http server which accepts zipkin requests -type Server struct { +// SpanHandler is an implementation of a Handler which accepts zipkin thrift +// span data and sends it to the recorder +type SpanHandler struct { Path string - tracer Tracer + recorder Recorder waitGroup *sync.WaitGroup } -// Register allows server to implement the Service interface. Server's register metod -// registers its handler on mux, and sets the servers tracer with tracer -func (s *Server) Register(router *mux.Router, tracer Tracer) error { - router.HandleFunc(s.Path, s.SpanHandler).Methods("POST") - s.tracer = tracer - return nil -} - -func unmarshalThrift(body []byte) ([]*zipkincore.Span, error) { - buffer := thrift.NewTMemoryBuffer() - if _, err := buffer.Write(body); err != nil { - log.Println("Error in buffer write: ", err) - return nil, err - } - - transport := thrift.NewTBinaryProtocolTransport(buffer) - _, size, err := transport.ReadListBegin() - if err != nil { - log.Printf("Error in ReadListBegin: %s", err) - return nil, err - } - var spans []*zipkincore.Span - for i := 0; i < size; i++ { - zs := &zipkincore.Span{} - if err = zs.Read(transport); err != nil { - log.Printf("Error reading into zipkin struct: %s", err) - return nil, err - } - spans = append(spans, zs) +// NewSpanHandler returns a new server instance given path to handle +func NewSpanHandler(path string) *SpanHandler { + return &SpanHandler{ + Path: path, } +} - err = transport.ReadListEnd() - if err != nil { - log.Printf("%s", err) - return nil, err - } - return spans, nil +// Register implements the Service interface. Register accepts zipkin thrift data +// POSTed to the path of the mux router +func (s *SpanHandler) Register(router *mux.Router, recorder Recorder) error { + router.HandleFunc(s.Path, s.Spans).Methods("POST") + s.recorder = recorder + return nil } -// SpanHandler is the handler Server uses for handling zipkin POST requests -func (s *Server) SpanHandler(w http.ResponseWriter, r *http.Request) { - log.Printf("Received request from: %s", r.URL.String()) - log.Printf("Raw request data is: %#+v", r) +// Spans handles zipkin thrift spans +func (s *SpanHandler) Spans(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() - body, err := ioutil.ReadAll(r.Body) - - /*f, err := os.OpenFile("plugins/inputs/zipkin/testdata/file.dat", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) - defer f.Close() - _, err = f.Write(body) if err != nil { - log.Printf("Could not write to data file") - }*/ - - if err != nil { - e := fmt.Errorf("Encountered error reading: %s", err) - log.Println(e) - s.tracer.Error(e) + s.recorder.Error(err) w.WriteHeader(http.StatusInternalServerError) return } + spans, err := unmarshalThrift(body) if err != nil { - log.Println("Error: ", err) - s.tracer.Error(err) + s.recorder.Error(err) w.WriteHeader(http.StatusInternalServerError) return } - //marshal json for debugging purposes - out, _ := json.MarshalIndent(spans, "", " ") - log.Println(string(out)) - - /* f, err = os.OpenFile("plugins/inputs/zipkin/testdata/json/file.json", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) - defer f.Close() - _, err = f.Write(out) - if err != nil { - log.Printf("Could not write to data file") - }*/ - - trace, err := UnmarshalZipkinResponse(spans) + trace, err := NewTrace(spans) if err != nil { - log.Println("Error: ", err) - s.tracer.Error(err) + s.recorder.Error(err) w.WriteHeader(http.StatusInternalServerError) return } - if err = s.tracer.Record(trace); err != nil { - log.Println(err) + if err = s.recorder.Record(trace); err != nil { + s.recorder.Error(err) w.WriteHeader(http.StatusInternalServerError) return } + w.WriteHeader(http.StatusNoContent) } -//NewServer returns a new server instance given path to handle -func NewServer(path string) *Server { - return &Server{ - Path: path, +func unmarshalThrift(body []byte) ([]*zipkincore.Span, error) { + buffer := thrift.NewTMemoryBuffer() + if _, err := buffer.Write(body); err != nil { + return nil, err + } + + transport := thrift.NewTBinaryProtocolTransport(buffer) + _, size, err := transport.ReadListBegin() + if err != nil { + return nil, err + } + + spans := make([]*zipkincore.Span, size) + for i := 0; i < size; i++ { + zs := &zipkincore.Span{} + if err = zs.Read(transport); err != nil { + return nil, err + } + spans[i] = zs } + + if err = transport.ReadListEnd(); err != nil { + return nil, err + } + + return spans, nil } diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 1218dc1c6754d..452a1fa5d969f 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -13,18 +13,18 @@ import ( "time" ) -type MockTracer struct { +type MockRecorder struct { Data Trace Err error } -func (m *MockTracer) Record(t Trace) error { +func (m *MockRecorder) Record(t Trace) error { fmt.Println("Adding trace ", t) m.Data = t return nil } -func (m *MockTracer) Error(err error) { +func (m *MockRecorder) Error(err error) { m.Err = err } @@ -35,22 +35,22 @@ func TestZipkinServer(t *testing.T) { t.Fatalf("Could not find file %s\n", "test/threespans.dat") } - s := NewServer("/api/v1/spans") - mockTracer := &MockTracer{} - s.tracer = mockTracer + s := NewSpanHandler("/api/v1/spans") + mockRecorder := &MockRecorder{} + s.recorder = mockRecorder w := httptest.NewRecorder() r := httptest.NewRequest( "POST", "http://server.local/api/v1/spans", ioutil.NopCloser( bytes.NewReader(dat))) - handler := s.SpanHandler + handler := s.Spans handler(w, r) if w.Code != http.StatusNoContent { t.Errorf("MainHandler did not return StatusNoContent %d", w.Code) } - got := mockTracer.Data + got := mockRecorder.Data d := int64(53106) d1 := int64(50410) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 21b6442754e9a..e2dec8b7f0573 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -12,7 +12,6 @@ import ( "github.com/gorilla/mux" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" - "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) const ( @@ -29,17 +28,17 @@ const ( DefaultShutdownTimeout = 5 ) -// Tracer represents a type which can record zipkin trace data as well as +// Recorder represents a type which can record zipkin trace data as well as // any accompanying errors, and process that data. -type Tracer interface { +type Recorder interface { Record(Trace) error Error(error) } -// Service represents a type which can register itself with a router for -// http routing, and a Tracer for trace data collection. -type Service interface { - Register(router *mux.Router, tracer Tracer) error +// Handler represents a type which can register itself with a router for +// http routing, and a Recorder for trace data collection. +type Handler interface { + Register(router *mux.Router, recorder Recorder) error } // BinaryAnnotation represents a zipkin binary annotation. It contains @@ -78,103 +77,7 @@ type Span struct { // Trace is an array (or a series) of spans type Trace []Span -//UnmarshalZipkinResponse is a helper method for unmarhsalling a slice of []*zipkincore.Spans -// into a Trace (technically a []Span) -func UnmarshalZipkinResponse(spans []*zipkincore.Span) (Trace, error) { - var trace Trace - for _, span := range spans { - - s := Span{} - s.ID = strconv.FormatInt(span.GetID(), 10) - s.TraceID = strconv.FormatInt(span.GetTraceID(), 10) - if span.GetTraceIDHigh() != 0 { - s.TraceID = strconv.FormatInt(span.GetTraceIDHigh(), 10) + s.TraceID - } - - s.Annotations = UnmarshalAnnotations(span.GetAnnotations()) - - var err error - s.BinaryAnnotations, err = UnmarshalBinaryAnnotations(span.GetBinaryAnnotations()) - if err != nil { - return nil, err - } - s.Name = span.GetName() - if span.GetTimestamp() == 0 { - s.Timestamp = time.Now() - } else { - s.Timestamp = time.Unix(0, span.GetTimestamp()*int64(time.Microsecond)) - } - - duration := time.Duration(span.GetDuration()) - // fmt.Println("Duration: ", duration) - s.Duration = duration * time.Microsecond - - parentID := span.GetParentID() - // fmt.Println("Parent ID: ", parentID) - - // A parent ID of 0 means that this is a parent span. In this case, - // we set the parent ID of the span to be its own id, so it points to - // itself. - - if parentID == 0 { - s.ParentID = s.ID - } else { - s.ParentID = strconv.FormatInt(parentID, 10) - } - - // fmt.Println("ID:", s.ID) - trace = append(trace, s) - } - - return trace, nil -} - -// UnmarshalAnnotations is a helper method for unmarshalling a slice of -// *zipkincore.Annotation into a slice of Annotations -func UnmarshalAnnotations(annotations []*zipkincore.Annotation) []Annotation { - var formatted []Annotation - for _, annotation := range annotations { - a := Annotation{} - endpoint := annotation.GetHost() - if endpoint != nil { - a.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) - a.ServiceName = endpoint.GetServiceName() - } else { - a.Host, a.ServiceName = "", "" - } - - a.Timestamp = time.Unix(annotation.GetTimestamp(), 0) - a.Value = annotation.GetValue() - formatted = append(formatted, a) - } - //fmt.Println("formatted annotations: ", formatted) - return formatted -} - -// UnmarshalBinaryAnnotations is very similar to UnmarshalAnnotations, but it -// Unmarshalls zipkincore.BinaryAnnotations instead of the normal zipkincore.Annotation -func UnmarshalBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryAnnotation, error) { - var formatted []BinaryAnnotation - for _, annotation := range annotations { - b := BinaryAnnotation{} - endpoint := annotation.GetHost() - if endpoint != nil { - b.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) - b.ServiceName = endpoint.GetServiceName() - } else { - b.Host, b.ServiceName = "", "" - } - - b.Key = annotation.GetKey() - b.Value = string(annotation.GetValue()) - b.Type = annotation.GetAnnotationType().String() - formatted = append(formatted, b) - } - - return formatted, nil -} - -// LineProtocolConverter implements the Tracer interface; it is a +// LineProtocolConverter implements the Recorder interface; it is a // type meant to encapsulate the storage of zipkin tracing data in // telegraf as line protocol. type LineProtocolConverter struct { @@ -182,7 +85,7 @@ type LineProtocolConverter struct { } // Record is LineProtocolConverter's implementation of the Record method of -// the Tracer iterface; it takes a span as input, and adds it to an internal +// the Recorder interface; it takes a trace as input, and adds it to an internal // telegraf.Accumulator. func (l *LineProtocolConverter) Record(t Trace) error { log.Printf("received trace: %#+v\n", t) @@ -261,7 +164,7 @@ type Zipkin struct { ServiceAddress string Port int Path string - tracing Service + handler Handler server *http.Server waitGroup *sync.WaitGroup } @@ -284,9 +187,8 @@ func (z *Zipkin) Gather(acc telegraf.Accumulator) error { return nil } // passing in a telegraf.Accumulator such that data can be collected. func (z *Zipkin) Start(acc telegraf.Accumulator) error { log.Println("starting...") - if z.tracing == nil { - t := NewServer(z.Path) - z.tracing = t + if z.handler == nil { + z.handler = NewSpanHandler(z.Path) } var wg sync.WaitGroup @@ -314,14 +216,14 @@ func (z *Zipkin) Stop() { // Listen creates an http server on the zipkin instance it is called with, and // serves http until it is stopped by Zipkin's (*Zipkin).Stop() method. func (z *Zipkin) Listen(acc telegraf.Accumulator) { - r := mux.NewRouter() + router := mux.NewRouter() converter := NewLineProtocolConverter(acc) - z.tracing.Register(r, converter) + z.handler.Register(router, converter) if z.server == nil { z.server = &http.Server{ Addr: ":" + strconv.Itoa(z.Port), - Handler: r, + Handler: router, } } if err := z.server.ListenAndServe(); err != nil { From 09a5628c3bb6fadfdd4cadf883b15d0bf790d649 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 13 Jul 2017 16:32:54 -0600 Subject: [PATCH 077/154] WIP: Refactor tests in zipkin plugin --- plugins/inputs/zipkin/convert.go | 63 +++++++++++++++++++++++ plugins/inputs/zipkin/handler_test.go | 50 ++++++++---------- plugins/inputs/zipkin/plugin_test.go | 6 +-- plugins/inputs/zipkin/zipkin.go | 74 --------------------------- 4 files changed, 86 insertions(+), 107 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index a6c796167e8eb..2c4fb369d0786 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -4,9 +4,72 @@ import ( "strconv" "time" + "github.com/influxdata/telegraf" "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) +// LineProtocolConverter implements the Recorder interface; it is a +// type meant to encapsulate the storage of zipkin tracing data in +// telegraf as line protocol. +type LineProtocolConverter struct { + acc telegraf.Accumulator +} + +// NewLineProtocolConverter returns an instance of LineProtocolConverter that +// will add to the given telegraf.Accumulator +func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { + return &LineProtocolConverter{ + acc: acc, + } +} + +// Record is LineProtocolConverter's implementation of the Record method of +// the Recorder interface; it takes a trace as input, and adds it to an internal +// telegraf.Accumulator. +func (l *LineProtocolConverter) Record(t Trace) error { + for _, s := range t { + fields := map[string]interface{}{ + "duration": s.Duration, + } + + tags := map[string]string{ + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + } + + for _, a := range s.Annotations { + fields["annotation_timestamp"] = a.Timestamp.Unix() + tags["service_name"] = a.ServiceName + tags["annotation_value"] = a.Value + tags["endpoint_host"] = a.Host + l.acc.AddFields("zipkin", fields, tags, s.Timestamp) + } + + for _, b := range s.BinaryAnnotations { + tags := map[string]string{ + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + "service_name": b.ServiceName, + "annotation_value": b.Value, + "endpoint_host": b.Host, + "key": b.Key, + "type": b.Type, + } + l.acc.AddFields("zipkin", fields, tags, s.Timestamp) + } + } + + return nil +} + +func (l *LineProtocolConverter) Error(err error) { + l.acc.AddError(err) +} + // NewTrace converts a slice of []*zipkincore.Spans into a new Trace func NewTrace(spans []*zipkincore.Span) (Trace, error) { var trace Trace diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 452a1fa5d969f..9d9d815cb3f7d 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io/ioutil" - "log" "net/http" "net/http/httptest" "reflect" @@ -28,45 +27,40 @@ func (m *MockRecorder) Error(err error) { m.Err = err } -func TestZipkinServer(t *testing.T) { - log.Println("testing server unmarshalling") +func TestSpanHandler(t *testing.T) { dat, err := ioutil.ReadFile("testdata/threespans.dat") if err != nil { - t.Fatalf("Could not find file %s\n", "test/threespans.dat") + t.Fatalf("Could not find file %s\n", "testdata/threespans.dat") } - s := NewSpanHandler("/api/v1/spans") - mockRecorder := &MockRecorder{} - s.recorder = mockRecorder w := httptest.NewRecorder() r := httptest.NewRequest( "POST", "http://server.local/api/v1/spans", ioutil.NopCloser( bytes.NewReader(dat))) - handler := s.Spans - handler(w, r) + + handler := NewSpanHandler("/api/v1/spans") + mockRecorder := &MockRecorder{} + handler.recorder = mockRecorder + + handler.Spans(w, r) if w.Code != http.StatusNoContent { t.Errorf("MainHandler did not return StatusNoContent %d", w.Code) } got := mockRecorder.Data - d := int64(53106) - d1 := int64(50410) - d2 := int64(103680) - parentID1 := int64(22964302721410078) + parentID := strconv.FormatInt(22964302721410078, 10) want := Trace{ Span{ - Name: "Child", - ID: "8090652509916334619", - TraceID: "2505404965370368069", - ParentID: strconv.FormatInt(parentID1, 10), - Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)), - Duration: time.Duration(d) * time.Microsecond, - //note: []Annotation(nil) is different than - // []Annotation{} - Annotations: []Annotation(nil), + Name: "Child", + ID: "8090652509916334619", + TraceID: "2505404965370368069", + ParentID: parentID, + Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)), + Duration: time.Duration(53106) * time.Microsecond, + Annotations: nil, BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ Key: "lc", @@ -81,10 +75,10 @@ func TestZipkinServer(t *testing.T) { Name: "Child", ID: "103618986556047333", TraceID: "2505404965370368069", - ParentID: strconv.FormatInt(parentID1, 10), + ParentID: parentID, Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)), - Duration: time.Duration(d1) * time.Microsecond, - Annotations: []Annotation(nil), + Duration: time.Duration(50410) * time.Microsecond, + Annotations: nil, BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ Key: "lc", @@ -101,7 +95,7 @@ func TestZipkinServer(t *testing.T) { TraceID: "2505404965370368069", ParentID: "22964302721410078", Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)), - Duration: time.Duration(d2) * time.Microsecond, + Duration: time.Duration(103680) * time.Microsecond, Annotations: []Annotation{ Annotation{ Timestamp: time.Unix(1498688360851325, 0), @@ -134,10 +128,6 @@ func TestZipkinServer(t *testing.T) { }, } - /* if !reflect.DeepEqual(got, want) { - t.Fatal("Got != want, Fields weren't unmarshalled correctly") - }*/ - for i, s := range got { if !reflect.DeepEqual(s, want[i]) { fmt.Printf("index %d wasn't equal", i) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index 1140c16124905..02547fb82d996 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -121,7 +121,7 @@ func assertContainsTaggedDuration( } } msg := fmt.Sprintf( - "Could not find measurement \"%s\" with requested tags within %s, Actual: %d", + "assertContainsTaggedDuration: Could not find measurement \"%s\" with requested tags within %s, Actual: %d", measurement, field, actualValue) t.Fatal(msg) } @@ -160,7 +160,7 @@ func assertContainsTaggedInt64( } } msg := fmt.Sprintf( - "Could not find measurement \"%s\" with requested tags within %s, Actual: %d", + "assertContainsTaggedInt64: Could not find measurement \"%s\" with requested tags within %s, Actual: %d", measurement, field, actualValue) t.Fatal(msg) } @@ -196,7 +196,7 @@ func assertContainsTaggedTime( } } msg := fmt.Sprintf( - "Could not find measurement \"%s\" with requested tags within %s, Actual: %d", + "assertContainsTaggedTime: Could not find measurement \"%s\" with requested tags within %s, Actual: %d", measurement, field, actualValue) t.Fatal(msg) } diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index e2dec8b7f0573..a94a9308865a7 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -77,80 +77,6 @@ type Span struct { // Trace is an array (or a series) of spans type Trace []Span -// LineProtocolConverter implements the Recorder interface; it is a -// type meant to encapsulate the storage of zipkin tracing data in -// telegraf as line protocol. -type LineProtocolConverter struct { - acc telegraf.Accumulator -} - -// Record is LineProtocolConverter's implementation of the Record method of -// the Recorder interface; it takes a trace as input, and adds it to an internal -// telegraf.Accumulator. -func (l *LineProtocolConverter) Record(t Trace) error { - log.Printf("received trace: %#+v\n", t) - log.Printf("Writing to telegraf...\n") - for _, s := range t { - for _, a := range s.Annotations { - fields := map[string]interface{}{ - // Maybe we don't need "annotation_timestamp"? - "annotation_timestamp": a.Timestamp.Unix(), - "duration": s.Duration, - } - - log.Printf("Duration is: %d", s.Duration) - - tags := map[string]string{ - "id": s.ID, - "parent_id": s.ParentID, - "trace_id": s.TraceID, - "name": s.Name, - "service_name": a.ServiceName, - "annotation_value": a.Value, - "endpoint_host": a.Host, - } - log.Println("adding data") - l.acc.AddFields("zipkin", fields, tags, s.Timestamp) - } - - for _, b := range s.BinaryAnnotations { - fields := map[string]interface{}{ - "duration": s.Duration, - } - - log.Printf("Duration is: %d", s.Duration) - - tags := map[string]string{ - "id": s.ID, - "parent_id": s.ParentID, - "trace_id": s.TraceID, - "name": s.Name, - "service_name": b.ServiceName, - "annotation_value": b.Value, - "endpoint_host": b.Host, - "key": b.Key, - "type": b.Type, - } - log.Printf("adding data") - l.acc.AddFields("zipkin", fields, tags, s.Timestamp) - } - } - - return nil -} - -func (l *LineProtocolConverter) Error(err error) { - l.acc.AddError(err) -} - -// NewLineProtocolConverter returns an instance of LineProtocolConverter that -// will add to the given telegraf.Accumulator -func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { - return &LineProtocolConverter{ - acc: acc, - } -} - const sampleConfig = ` ## # path = /path/your/zipkin/impl/posts/to From 6ddf4d8d024a0778460500e67b2829472b61b122 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 14 Jul 2017 17:02:43 -0600 Subject: [PATCH 078/154] Add convert table test to zipkin plugin --- plugins/inputs/zipkin/convert.go | 34 ++++--- plugins/inputs/zipkin/convert_test.go | 138 ++++++++++++++++++++++++++ plugins/inputs/zipkin/handler_test.go | 1 - plugins/inputs/zipkin/plugin_test.go | 29 ++---- 4 files changed, 164 insertions(+), 38 deletions(-) create mode 100644 plugins/inputs/zipkin/convert_test.go diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 2c4fb369d0786..2f5ee48add725 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -28,26 +28,30 @@ func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { // telegraf.Accumulator. func (l *LineProtocolConverter) Record(t Trace) error { for _, s := range t { - fields := map[string]interface{}{ - "duration": s.Duration, - } - - tags := map[string]string{ - "id": s.ID, - "parent_id": s.ParentID, - "trace_id": s.TraceID, - "name": s.Name, - } - for _, a := range s.Annotations { - fields["annotation_timestamp"] = a.Timestamp.Unix() - tags["service_name"] = a.ServiceName - tags["annotation_value"] = a.Value - tags["endpoint_host"] = a.Host + fields := map[string]interface{}{ + // TODO: Maybe we don't need "annotation_timestamp"? + "annotation_timestamp": a.Timestamp.Unix(), + "duration": s.Duration, + } + + tags := map[string]string{ + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + "service_name": a.ServiceName, + "annotation_value": a.Value, + "endpoint_host": a.Host, + } l.acc.AddFields("zipkin", fields, tags, s.Timestamp) } for _, b := range s.BinaryAnnotations { + fields := map[string]interface{}{ + "duration": s.Duration, + } + tags := map[string]string{ "id": s.ID, "parent_id": s.ParentID, diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go new file mode 100644 index 0000000000000..19668157b2810 --- /dev/null +++ b/plugins/inputs/zipkin/convert_test.go @@ -0,0 +1,138 @@ +package zipkin + +import ( + "reflect" + "testing" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/testutil" +) + +func TestLineProtocolConverter_Record(t *testing.T) { + mockAcc := testutil.Accumulator{} + type fields struct { + acc telegraf.Accumulator + } + type args struct { + t Trace + } + tests := []struct { + name string + fields fields + args args + wantErr bool + want []testutil.Metric + }{ + { + name: "threespan", + fields: fields{ + acc: &mockAcc, + }, + args: args{ + t: Trace{ + Span{ + ID: "8090652509916334619", + TraceID: "2505404965370368069", + Name: "Child", + ParentID: "22964302721410078", + Timestamp: time.Unix(0, 1498688360851331000), + Duration: time.Duration(53106) * time.Microsecond, + Annotations: []Annotation{}, + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "dHJpdmlhbA==", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, + Span{ + ID: "103618986556047333", + TraceID: "2505404965370368069", + Name: "Child", + ParentID: "22964302721410078", + Timestamp: time.Unix(0, 1498688360904552000), + Duration: time.Duration(50410) * time.Microsecond, + Annotations: []Annotation{}, + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "dHJpdmlhbA==", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, + }, + }, + want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "dHJpdmlhbA==", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(53106) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851331000), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "dHJpdmlhbA==", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(50410) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360904552000), + }, + }, + wantErr: false, + }, + + /*{ + name: "changeMe2", + fields: fields{}, + args: args{}, + wantErr: false, + },*/ + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockAcc.ClearMetrics() + l := &LineProtocolConverter{ + acc: tt.fields.acc, + } + if err := l.Record(tt.args.t); (err != nil) != tt.wantErr { + t.Errorf("LineProtocolConverter.Record() error = %v, wantErr %v", err, tt.wantErr) + } + got := []testutil.Metric{} + for _, metric := range mockAcc.Metrics { + got = append(got, *metric) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("LineProtocolConverter.Record() error = \n%#v\n, want \n%#v\n", got, tt.want) + } + }) + } +} diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 9d9d815cb3f7d..602bf03e033f6 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -18,7 +18,6 @@ type MockRecorder struct { } func (m *MockRecorder) Record(t Trace) error { - fmt.Println("Adding trace ", t) m.Data = t return nil } diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index 02547fb82d996..7b7346fc555dc 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -14,9 +14,7 @@ import ( ) func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { - log.Println("running!") postTestData(t, u.datafile) - log.Println("LENGTH:", len(u.expected)) if u.waitPoints == 0 { acc.Wait(len(u.expected)) } else { @@ -27,7 +25,7 @@ func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { for key, value := range data.expectedValues { switch value.(type) { case int64: - assertContainsTaggedInt64(t, acc, u.measurement, key, value.(int64), data.expectedTags) + //assertContainsTaggedInt64(t, acc, u.measurement, key, value.(int64), data.expectedTags) break case time.Duration: assertContainsTaggedDuration(t, acc, u.measurement, key, value.(time.Duration), data.expectedTags) @@ -48,7 +46,6 @@ func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { } func TestZipkin(t *testing.T) { - log.Println("testing zipkin...") var acc testutil.Accumulator z := &Zipkin{ Path: "/api/v1/test", @@ -98,20 +95,12 @@ func assertContainsTaggedDuration( expectedValue time.Duration, tags map[string]string, ) { - log.Println("going through tagged ") var actualValue interface{} - log.Println(acc.Metrics) for _, pt := range acc.Metrics { - log.Println("looping, point is : ", pt) - log.Println("point tags are : ", pt.Tags) if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { - log.Println("found measurement") for fieldname, value := range pt.Fields { - fmt.Println("looping through fields") if fieldname == field { - fmt.Println("found field: ", field) actualValue = value - fmt.Println("Value: ", value) if value == expectedValue { return } @@ -155,13 +144,17 @@ func assertContainsTaggedInt64( return } t.Errorf("Expected value %v\n got value %v\n", expectedValue, value) + } else { + t.Errorf("Fieldname != field %s", fieldname) } } + } else if !reflect.DeepEqual(pt.Tags, tags) { + log.Printf("%s\n%s", pt.Tags, tags) } } msg := fmt.Sprintf( - "assertContainsTaggedInt64: Could not find measurement \"%s\" with requested tags within %s, Actual: %d", - measurement, field, actualValue) + "assertContainsTaggedInt64: Could not find measurement \"%s\" with requested tags within %s, Actual: %d ,Expected: %d", + measurement, field, actualValue, expectedValue) t.Fatal(msg) } @@ -173,20 +166,12 @@ func assertContainsTaggedTime( expectedValue time.Time, tags map[string]string, ) { - log.Println("going through tagged ") var actualValue interface{} - log.Println(acc.Metrics) for _, pt := range acc.Metrics { - log.Println("looping, point is : ", pt) - log.Println("point tags are : ", pt.Tags) if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { - log.Println("found measurement") for fieldname, value := range pt.Fields { - fmt.Println("looping through fields") if fieldname == field { - fmt.Println("found field: ", field) actualValue = value - fmt.Println("Value: ", value) if value == expectedValue { return } From 070ec0458d9a100b4aa1321cf856ee7ee6a140b2 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Mon, 17 Jul 2017 15:20:44 -0600 Subject: [PATCH 079/154] Fix timestamp units to microseconds for zipkin plugin --- plugins/inputs/zipkin/convert.go | 14 ++++++++------ plugins/inputs/zipkin/handler_test.go | 6 +++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 2f5ee48add725..14bd14c8ec951 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -93,30 +93,27 @@ func NewTrace(spans []*zipkincore.Span) (Trace, error) { return nil, err } s.Name = span.GetName() + //TODO: find out what zipkin does with a timestamp of zero if span.GetTimestamp() == 0 { s.Timestamp = time.Now() } else { - s.Timestamp = time.Unix(0, span.GetTimestamp()*int64(time.Microsecond)) + s.Timestamp = microToTime(span.GetTimestamp()) } duration := time.Duration(span.GetDuration()) - // fmt.Println("Duration: ", duration) s.Duration = duration * time.Microsecond parentID := span.GetParentID() - // fmt.Println("Parent ID: ", parentID) // A parent ID of 0 means that this is a parent span. In this case, // we set the parent ID of the span to be its own id, so it points to // itself. - if parentID == 0 { s.ParentID = s.ID } else { s.ParentID = strconv.FormatInt(parentID, 10) } - // fmt.Println("ID:", s.ID) trace = append(trace, s) } @@ -131,13 +128,14 @@ func NewAnnotations(annotations []*zipkincore.Annotation) []Annotation { a := Annotation{} endpoint := annotation.GetHost() if endpoint != nil { + //TODO: Fix Ipv4 hostname to bit shifted a.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) a.ServiceName = endpoint.GetServiceName() } else { a.Host, a.ServiceName = "", "" } - a.Timestamp = time.Unix(annotation.GetTimestamp(), 0) + a.Timestamp = microToTime(annotation.GetTimestamp()) a.Value = annotation.GetValue() formatted = append(formatted, a) } @@ -167,3 +165,7 @@ func NewBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryA return formatted, nil } + +func microToTime(micro int64) time.Time { + return time.Unix(0, micro*int64(time.Microsecond)) +} diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 602bf03e033f6..55217894f66d3 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -97,19 +97,19 @@ func TestSpanHandler(t *testing.T) { Duration: time.Duration(103680) * time.Microsecond, Annotations: []Annotation{ Annotation{ - Timestamp: time.Unix(1498688360851325, 0), + Timestamp: time.Unix(0, 1498688360851325*int64(time.Microsecond)), Value: "Starting child #0", Host: "2130706433:0", ServiceName: "trivial", }, Annotation{ - Timestamp: time.Unix(1498688360904545, 0), + Timestamp: time.Unix(0, 1498688360904545*int64(time.Microsecond)), Value: "Starting child #1", Host: "2130706433:0", ServiceName: "trivial", }, Annotation{ - Timestamp: time.Unix(1498688360954992, 0), + Timestamp: time.Unix(0, 1498688360954992*int64(time.Microsecond)), Value: "A Log", Host: "2130706433:0", ServiceName: "trivial", From 9d378948f99dd81677e738b7349036eb1c7d9860 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Mon, 17 Jul 2017 15:21:32 -0600 Subject: [PATCH 080/154] Add more data for testing zipkin plugin --- plugins/inputs/zipkin/convert_test.go | 237 +++++++++++++++++++++++++- 1 file changed, 234 insertions(+), 3 deletions(-) diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index 19668157b2810..ea91bfbf0b726 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -67,6 +67,43 @@ func TestLineProtocolConverter_Record(t *testing.T) { }, }, }, + Span{ + ID: "22964302721410078", + TraceID: "2505404965370368069", + Name: "Parent", + ParentID: "22964302721410078", + Timestamp: time.Unix(0, 1498688360851318000), + Duration: time.Duration(103680) * time.Microsecond, + Annotations: []Annotation{ + Annotation{ + Timestamp: time.Unix(0, 1498688360851325000), + Value: "Starting child #0", + Host: "2130706433:0", + ServiceName: "trivial", + }, + Annotation{ + Timestamp: time.Unix(0, 1498688360904545000), + Value: "Starting child #1", + Host: "2130706433:0", + ServiceName: "trivial", + }, + Annotation{ + Timestamp: time.Unix(0, 1498688360954992000), + Value: "A Log", + Host: "2130706433:0", + ServiceName: "trivial", + }, + }, + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "lc", + Value: "dHJpdmlhbA==", + Host: "2130706433:0", + ServiceName: "trivial", + Type: "STRING", + }, + }, + }, }, }, want: []testutil.Metric{ @@ -106,16 +143,210 @@ func TestLineProtocolConverter_Record(t *testing.T) { }, Time: time.Unix(0, 1498688360904552000), }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "service_name": "trivial", + "annotation_value": "Starting child #0", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1498688360), + "duration": time.Duration(103680) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851318000), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "service_name": "trivial", + "annotation_value": "Starting child #1", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1498688360), + "duration": time.Duration(103680) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851318000), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "A Log", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1498688360), + "duration": time.Duration(103680) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851318000), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "trace_id": "2505404965370368069", + "service_name": "trivial", + "annotation_value": "dHJpdmlhbA==", + "key": "lc", + "type": "STRING", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "name": "Parent", + "endpoint_host": "2130706433:0", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(103680) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851318000), + }, }, wantErr: false, }, + // Test data from zipkin cli app: + //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services /*{ - name: "changeMe2", - fields: fields{}, - args: args{}, + name: "cli", + fields: fields{ + acc: &mockAcc, + }, + args: args{ + t: Trace{ + Span{ + ID: "3383422996321511664", + TraceID: "243463817635710260", + Name: "Concat", + ParentID: "4574092882326506380", + Timestamp: time.Unix(0, 1499817952283903000), + Duration: time.Duration(2888) * time.Microsecond, + Annotations: []Annotation{ + Annotaitons{ + Timestamp: time.Unix(0, 1499817952283903000), + Value: "cs", + Host: "0:0", + ServiceName: "cli", + }, + Annotaitons{ + Timestamp: time.Unix(0, 1499817952286792000), + Value: "cr", + Host: "0:0", + ServiceName: "cli", + }, + }, + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "http.path", + Value: "L2NvbmNhdC8=", + Host: "0:0", + ServiceName: "cli", + Type: "STRING", + }, + BinaryAnnotation{ + Key: "http.url", + Value: "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", + Host: "0:0", + ServiceName: "cli", + Type: "STRING", + }, + BinaryAnnotation{ + Key: "peer.hostname", + Value: "bG9jYWxob3N0", + Host: "0:0", + ServiceName: "cli", + Type: "STRING", + }, + BinaryAnnotation{ + Key: "span.kind", + Value: "Y2xpZW50", + Host: "0:0", + ServiceName: "cli", + Type: "STRING", + }, + BinaryAnnotation{ + Key: "http.method", + Value: "R0VU", + Host: "0:0", + ServiceName: "cli", + Type: "STRING", + }, + BinaryAnnotation{ + Key: "http.host", + Value: "bG9jYWxob3N0OjYxMDAx", + Host: "0:0", + ServiceName: "cli", + Type: "STRING", + }, + }, + }, + want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "3383422996321511664", + "parent_id": "4574092882326506380", + //TODO: Can trace_id be this long? + "trace_id": "8269862291023777619243463817635710260", + "name": "Concat", + "service_name": "cli", + //TODO: Do we need annotation_timestamp? + "annotation_value": "cs", + //TODO: Multiple binary_annotations + //"endpoint_host": "0:0", + //"key": "lc", + //"type": "STRING", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(2888) * time.Microsecond, + }, + Time: time.Unix(0, 1499817952283903000), + }, wantErr: false, },*/ + + //// Test data from distributed trace repo sample json + // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json + /*{ + name: "distributed_trace_sample" + fields: fields{ + acc: &mockAcc, + }, + args: args{ + t: Trace{ + Span{ + ID: "6802735349851856000", + TraceID: "6802735349851856000", + Name:"main.dud", + //TODO: ParentID nil or itself + ParentID: + //TODO: No Timestamp? or duraiton? + Timestamp: + Duration: + Annotation: []Annotation{ + Annotation{ + Timestamp: "1433330263415871", + Value: "cs", + Host: "0:9410", + ServiceName: "go-zipkin-testclient", + } + } + BinaryAnnotation: []BinaryAnnotation{}, + }, + }, + }, + },*/ } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 743a18d73db75ce6fcef2bf4648addb77911a8c2 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Tue, 18 Jul 2017 15:14:37 -0600 Subject: [PATCH 081/154] Refactor formatting for zipkin plugin --- plugins/inputs/zipkin/convert.go | 39 ++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 14bd14c8ec951..4e0485e865172 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -79,11 +79,8 @@ func NewTrace(spans []*zipkincore.Span) (Trace, error) { var trace Trace for _, span := range spans { s := Span{} - s.ID = strconv.FormatInt(span.GetID(), 10) - s.TraceID = strconv.FormatInt(span.GetTraceID(), 10) - if span.GetTraceIDHigh() != 0 { - s.TraceID = strconv.FormatInt(span.GetTraceIDHigh(), 10) + s.TraceID - } + s.ID = formatID(span.GetID()) + s.TraceID = formatTraceID(span.GetTraceIDHigh(), span.GetTraceID()) s.Annotations = NewAnnotations(span.GetAnnotations()) @@ -93,12 +90,7 @@ func NewTrace(spans []*zipkincore.Span) (Trace, error) { return nil, err } s.Name = span.GetName() - //TODO: find out what zipkin does with a timestamp of zero - if span.GetTimestamp() == 0 { - s.Timestamp = time.Now() - } else { - s.Timestamp = microToTime(span.GetTimestamp()) - } + s.Timestamp = guessTimestamp(span) duration := time.Duration(span.GetDuration()) s.Duration = duration * time.Microsecond @@ -111,7 +103,7 @@ func NewTrace(spans []*zipkincore.Span) (Trace, error) { if parentID == 0 { s.ParentID = s.ID } else { - s.ParentID = strconv.FormatInt(parentID, 10) + s.ParentID = formatID(parentID) } trace = append(trace, s) @@ -169,3 +161,26 @@ func NewBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryA func microToTime(micro int64) time.Time { return time.Unix(0, micro*int64(time.Microsecond)) } + +func formatID(id int64) string { + return strconv.FormatInt(id, 10) +} + +func formatTraceID(high, low int64) string { + return formatID(high) + ":" + formatID(low) +} + +func guessTimestamp(span *zipkincore.Span) time.Time { + if span.GetTimestamp() != 0 { + return microToTime(span.GetTimestamp()) + } + + minTimestamp := time.Now() + for _, annotation := range span.Annotations { + ts := microToTime(annotation.GetTimestamp()) + if !ts.IsZero() && ts.Before(minTimestamp) { + minTimestamp = ts + } + } + return minTimestamp +} From e0d4c38099a7a907021cf39b195a0031fe590079 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Tue, 18 Jul 2017 15:15:55 -0600 Subject: [PATCH 082/154] Fix bugs in test data for zipkin plugin --- plugins/inputs/zipkin/convert_test.go | 122 ++++++++++++-------------- plugins/inputs/zipkin/handler_test.go | 8 +- plugins/inputs/zipkin/unit_tests.go | 18 ++-- 3 files changed, 69 insertions(+), 79 deletions(-) diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index ea91bfbf0b726..68c1692b1e20e 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -239,21 +239,8 @@ func TestLineProtocolConverter_Record(t *testing.T) { Host: "0:0", ServiceName: "cli", }, - Annotaitons{ - Timestamp: time.Unix(0, 1499817952286792000), - Value: "cr", - Host: "0:0", - ServiceName: "cli", - }, }, BinaryAnnotations: []BinaryAnnotation{ - BinaryAnnotation{ - Key: "http.path", - Value: "L2NvbmNhdC8=", - Host: "0:0", - ServiceName: "cli", - Type: "STRING", - }, BinaryAnnotation{ Key: "http.url", Value: "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", @@ -261,34 +248,6 @@ func TestLineProtocolConverter_Record(t *testing.T) { ServiceName: "cli", Type: "STRING", }, - BinaryAnnotation{ - Key: "peer.hostname", - Value: "bG9jYWxob3N0", - Host: "0:0", - ServiceName: "cli", - Type: "STRING", - }, - BinaryAnnotation{ - Key: "span.kind", - Value: "Y2xpZW50", - Host: "0:0", - ServiceName: "cli", - Type: "STRING", - }, - BinaryAnnotation{ - Key: "http.method", - Value: "R0VU", - Host: "0:0", - ServiceName: "cli", - Type: "STRING", - }, - BinaryAnnotation{ - Key: "http.host", - Value: "bG9jYWxob3N0OjYxMDAx", - Host: "0:0", - ServiceName: "cli", - Type: "STRING", - }, }, }, want: []testutil.Metric{ @@ -297,16 +256,30 @@ func TestLineProtocolConverter_Record(t *testing.T) { Tags: map[string]string{ "id": "3383422996321511664", "parent_id": "4574092882326506380", - //TODO: Can trace_id be this long? - "trace_id": "8269862291023777619243463817635710260", + "trace_id": "8269862291023777619:243463817635710260", "name": "Concat", "service_name": "cli", - //TODO: Do we need annotation_timestamp? "annotation_value": "cs", - //TODO: Multiple binary_annotations - //"endpoint_host": "0:0", - //"key": "lc", - //"type": "STRING", + "endpoint_host": "0:0", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(149981795), + "duration": time.Duration(2888) * time.Microsecond, + }, + Time: time.Unix(0, 1499817952283903000), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "trace_id": "2505404965370368069", + "service_name": "cli", + "annotation_value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", + "key": "http.url", + "type": "STRING", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "name": "Concat", + "endpoint_host": "0:0", }, Fields: map[string]interface{}{ "duration": time.Duration(2888) * time.Microsecond, @@ -318,35 +291,52 @@ func TestLineProtocolConverter_Record(t *testing.T) { //// Test data from distributed trace repo sample json // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json - /*{ - name: "distributed_trace_sample" + { + name: "distributed_trace_sample", fields: fields{ acc: &mockAcc, }, args: args{ t: Trace{ Span{ - ID: "6802735349851856000", - TraceID: "6802735349851856000", - Name:"main.dud", - //TODO: ParentID nil or itself - ParentID: - //TODO: No Timestamp? or duraiton? - Timestamp: - Duration: - Annotation: []Annotation{ + ID: "6802735349851856000", + TraceID: "0:6802735349851856000", + Name: "main.dud", + ParentID: "6802735349851856000", + Timestamp: time.Unix(1, 0), + Duration: 1, + Annotations: []Annotation{ Annotation{ - Timestamp: "1433330263415871", - Value: "cs", - Host: "0:9410", + Timestamp: time.Unix(0, 1433330263415871000), + Value: "cs", + Host: "0:9410", ServiceName: "go-zipkin-testclient", - } - } - BinaryAnnotation: []BinaryAnnotation{}, + }, + }, + BinaryAnnotations: []BinaryAnnotation{}, }, }, }, - },*/ + want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "annotation_value": "cs", + "endpoint_host": "0:9410", + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1433330263), + "duration": time.Duration(1) * time.Nanosecond, + }, + Time: time.Unix(1, 0), + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 55217894f66d3..2ccc899aa3b6a 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -55,7 +55,7 @@ func TestSpanHandler(t *testing.T) { Span{ Name: "Child", ID: "8090652509916334619", - TraceID: "2505404965370368069", + TraceID: "0:2505404965370368069", ParentID: parentID, Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)), Duration: time.Duration(53106) * time.Microsecond, @@ -73,7 +73,7 @@ func TestSpanHandler(t *testing.T) { Span{ Name: "Child", ID: "103618986556047333", - TraceID: "2505404965370368069", + TraceID: "0:2505404965370368069", ParentID: parentID, Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)), Duration: time.Duration(50410) * time.Microsecond, @@ -91,7 +91,7 @@ func TestSpanHandler(t *testing.T) { Span{ Name: "Parent", ID: "22964302721410078", - TraceID: "2505404965370368069", + TraceID: "0:2505404965370368069", ParentID: "22964302721410078", Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)), Duration: time.Duration(103680) * time.Microsecond, @@ -131,7 +131,7 @@ func TestSpanHandler(t *testing.T) { if !reflect.DeepEqual(s, want[i]) { fmt.Printf("index %d wasn't equal", i) fmt.Println(s, want[i]) - t.Fatal("Got != want, Fields weren't unmarshalled correctly") + t.Fatalf("Got %#v\n != want %#v\n, Fields weren't unmarshalled correctly", s, want[i]) } } } diff --git a/plugins/inputs/zipkin/unit_tests.go b/plugins/inputs/zipkin/unit_tests.go index e0bcf5be07430..01ebfdb85ecb8 100644 --- a/plugins/inputs/zipkin/unit_tests.go +++ b/plugins/inputs/zipkin/unit_tests.go @@ -32,7 +32,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "8090652509916334619", "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", + "trace_id": "0:2505404965370368069", "name": "Child", "service_name": "trivial", "annotation_value": "trivial", @@ -48,7 +48,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "103618986556047333", "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", + "trace_id": "0:2505404965370368069", "name": "Child", "service_name": "trivial", "annotation_value": "trivial", @@ -64,7 +64,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "22964302721410078", "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", + "trace_id": "0:2505404965370368069", "name": "Parent", "service_name": "trivial", "annotation_value": "Starting child #0", @@ -78,7 +78,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "22964302721410078", "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", + "trace_id": "0:2505404965370368069", "name": "Parent", "service_name": "trivial", "annotation_value": "Starting child #1", @@ -92,7 +92,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "22964302721410078", "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", + "trace_id": "0:2505404965370368069", "name": "Parent", "service_name": "trivial", "annotation_value": "A Log", @@ -106,7 +106,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "22964302721410078", "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", + "trace_id": "0:2505404965370368069", "name": "Parent", "service_name": "trivial", "annotation_value": "trivial", @@ -132,7 +132,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "3383422996321511664", "parent_id": "4574092882326506380", - "trace_id": "8269862291023777619243463817635710260", + "trace_id": "0:8269862291023777619243463817635710260", "name": "Concat", "service_name": "cli", "annotation_value": "cs", @@ -155,7 +155,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "6802735349851856000", "parent_id": "6802735349851856000", - "trace_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", "name": "main.dud", "service_name": "go-zipkin-testclient", "annotation_value": "cs", @@ -169,7 +169,7 @@ var tests = []UnitTest{ expectedTags: map[string]string{ "id": "6802735349851856000", "parent_id": "6802735349851856000", - "trace_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", "name": "main.dud", "service_name": "go-zipkin-testclient", "annotation_value": "cr", From aa430af8da1a5508868e329fb99a5ddd9209bfeb Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 19 Jul 2017 00:34:39 -0600 Subject: [PATCH 083/154] Fix test logic bugs in zipkin plugin --- plugins/inputs/zipkin/handler.go | 7 +------ plugins/inputs/zipkin/handler_test.go | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/zipkin/handler.go b/plugins/inputs/zipkin/handler.go index 6ba20d12c03b0..e12313c3c5d54 100644 --- a/plugins/inputs/zipkin/handler.go +++ b/plugins/inputs/zipkin/handler.go @@ -50,12 +50,7 @@ func (s *SpanHandler) Spans(w http.ResponseWriter, r *http.Request) { return } - trace, err := NewTrace(spans) - if err != nil { - s.recorder.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } + trace := NewTrace(spans) if err = s.recorder.Record(trace); err != nil { s.recorder.Error(err) diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 2ccc899aa3b6a..4ca4549c8ec49 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -59,7 +59,7 @@ func TestSpanHandler(t *testing.T) { ParentID: parentID, Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)), Duration: time.Duration(53106) * time.Microsecond, - Annotations: nil, + Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ Key: "lc", @@ -77,7 +77,7 @@ func TestSpanHandler(t *testing.T) { ParentID: parentID, Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)), Duration: time.Duration(50410) * time.Microsecond, - Annotations: nil, + Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ Key: "lc", From c8e549436876975ceda50e550770ad9c663105ab Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 19 Jul 2017 14:07:27 -0600 Subject: [PATCH 084/154] Refactor zipkin plugin --- plugins/inputs/zipkin/convert.go | 155 +++++++++++++++++-------------- 1 file changed, 83 insertions(+), 72 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 4e0485e865172..bd77ca00f7a1f 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -75,87 +75,53 @@ func (l *LineProtocolConverter) Error(err error) { } // NewTrace converts a slice of []*zipkincore.Spans into a new Trace -func NewTrace(spans []*zipkincore.Span) (Trace, error) { - var trace Trace - for _, span := range spans { - s := Span{} - s.ID = formatID(span.GetID()) - s.TraceID = formatTraceID(span.GetTraceIDHigh(), span.GetTraceID()) - - s.Annotations = NewAnnotations(span.GetAnnotations()) - - var err error - s.BinaryAnnotations, err = NewBinaryAnnotations(span.GetBinaryAnnotations()) - if err != nil { - return nil, err +func NewTrace(spans []*zipkincore.Span) Trace { + trace := make(Trace, len(spans)) + for i, span := range spans { + trace[i] = Span{ + ID: formatID(span.GetID()), + TraceID: formatTraceID(span.GetTraceIDHigh(), span.GetTraceID()), + Name: span.GetName(), + Timestamp: guessTimestamp(span), + Duration: convertDuration(span), + ParentID: parentID(span), + Annotations: NewAnnotations(span.GetAnnotations()), + BinaryAnnotations: NewBinaryAnnotations(span.GetBinaryAnnotations()), } - s.Name = span.GetName() - s.Timestamp = guessTimestamp(span) - - duration := time.Duration(span.GetDuration()) - s.Duration = duration * time.Microsecond - - parentID := span.GetParentID() - - // A parent ID of 0 means that this is a parent span. In this case, - // we set the parent ID of the span to be its own id, so it points to - // itself. - if parentID == 0 { - s.ParentID = s.ID - } else { - s.ParentID = formatID(parentID) - } - - trace = append(trace, s) } - - return trace, nil + return trace } // NewAnnotations converts a slice of *zipkincore.Annotation into a slice // of new Annotations func NewAnnotations(annotations []*zipkincore.Annotation) []Annotation { - var formatted []Annotation - for _, annotation := range annotations { - a := Annotation{} - endpoint := annotation.GetHost() - if endpoint != nil { - //TODO: Fix Ipv4 hostname to bit shifted - a.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) - a.ServiceName = endpoint.GetServiceName() - } else { - a.Host, a.ServiceName = "", "" + formatted := make([]Annotation, len(annotations)) + for i, annotation := range annotations { + formatted[i] = Annotation{ + Host: host(annotation.GetHost()), + ServiceName: serviceName(annotation.GetHost()), + Timestamp: microToTime(annotation.GetTimestamp()), + Value: annotation.GetValue(), } - - a.Timestamp = microToTime(annotation.GetTimestamp()) - a.Value = annotation.GetValue() - formatted = append(formatted, a) } - //fmt.Println("formatted annotations: ", formatted) + return formatted } // NewBinaryAnnotations is very similar to NewAnnotations, but it // converts zipkincore.BinaryAnnotations instead of the normal zipkincore.Annotation -func NewBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) ([]BinaryAnnotation, error) { - var formatted []BinaryAnnotation - for _, annotation := range annotations { - b := BinaryAnnotation{} - endpoint := annotation.GetHost() - if endpoint != nil { - b.Host = strconv.Itoa(int(endpoint.GetIpv4())) + ":" + strconv.Itoa(int(endpoint.GetPort())) - b.ServiceName = endpoint.GetServiceName() - } else { - b.Host, b.ServiceName = "", "" +func NewBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) []BinaryAnnotation { + formatted := make([]BinaryAnnotation, len(annotations)) + for i, annotation := range annotations { + formatted[i] = BinaryAnnotation{ + Host: host(annotation.GetHost()), + ServiceName: serviceName(annotation.GetHost()), + Key: annotation.GetKey(), + Value: string(annotation.GetValue()), + Type: annotation.GetAnnotationType().String(), } - - b.Key = annotation.GetKey() - b.Value = string(annotation.GetValue()) - b.Type = annotation.GetAnnotationType().String() - formatted = append(formatted, b) } - - return formatted, nil + return formatted } func microToTime(micro int64) time.Time { @@ -170,17 +136,62 @@ func formatTraceID(high, low int64) string { return formatID(high) + ":" + formatID(low) } +func minMax(span *zipkincore.Span) (time.Time, time.Time) { + min := time.Now() + max := time.Unix(0, 0) + for _, annotation := range span.Annotations { + ts := microToTime(annotation.GetTimestamp()) + if !ts.IsZero() && ts.Before(min) { + min = ts + } + if !ts.IsZero() && ts.After(max) { + max = ts + } + } + if max.IsZero() { + max = min + } + return min, max +} + func guessTimestamp(span *zipkincore.Span) time.Time { if span.GetTimestamp() != 0 { return microToTime(span.GetTimestamp()) } + min, _ := minMax(span) + return min +} - minTimestamp := time.Now() - for _, annotation := range span.Annotations { - ts := microToTime(annotation.GetTimestamp()) - if !ts.IsZero() && ts.Before(minTimestamp) { - minTimestamp = ts - } +func convertDuration(span *zipkincore.Span) time.Duration { + duration := time.Duration(span.GetDuration()) * time.Microsecond + if duration != 0 { + return duration + } + min, max := minMax(span) + return max.Sub(min) +} + +func parentID(span *zipkincore.Span) string { + // A parent ID of 0 means that this is a parent span. In this case, + // we set the parent ID of the span to be its own id, so it points to + // itself. + id := span.GetParentID() + if id != 0 { + return formatID(id) + } + return formatID(span.ID) +} + +func host(h *zipkincore.Endpoint) string { + if h == nil { + return "" + } + return strconv.Itoa(int(h.GetIpv4())) + ":" + strconv.Itoa(int(h.GetPort())) +} + +func serviceName(h *zipkincore.Endpoint) string { + if h == nil { + return "" } - return minTimestamp + return h.GetServiceName() } From 6916c329d214ba38f83dde93b7ef70d2f4ff7de6 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 19 Jul 2017 18:12:27 -0600 Subject: [PATCH 085/154] WIP: Create tests for zipkin plugin --- plugins/inputs/zipkin/convert.go | 6 +- plugins/inputs/zipkin/convert_test.go | 201 +++++++++++++++++++++++--- 2 files changed, 185 insertions(+), 22 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index bd77ca00f7a1f..8ef90dbbd7c1e 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -125,7 +125,7 @@ func NewBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) []BinaryAn } func microToTime(micro int64) time.Time { - return time.Unix(0, micro*int64(time.Microsecond)) + return time.Unix(0, micro*int64(time.Microsecond)).UTC() } func formatID(id int64) string { @@ -137,8 +137,8 @@ func formatTraceID(high, low int64) string { } func minMax(span *zipkincore.Span) (time.Time, time.Time) { - min := time.Now() - max := time.Unix(0, 0) + min := time.Now().UTC() + max := time.Time{}.UTC() for _, annotation := range span.Annotations { ts := microToTime(annotation.GetTimestamp()) if !ts.IsZero() && ts.Before(min) { diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index 68c1692b1e20e..85c8dd8e1b0d7 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -7,6 +7,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" + "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) func TestLineProtocolConverter_Record(t *testing.T) { @@ -36,7 +37,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { TraceID: "2505404965370368069", Name: "Child", ParentID: "22964302721410078", - Timestamp: time.Unix(0, 1498688360851331000), + Timestamp: time.Unix(0, 1498688360851331000).UTC(), Duration: time.Duration(53106) * time.Microsecond, Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ @@ -54,7 +55,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { TraceID: "2505404965370368069", Name: "Child", ParentID: "22964302721410078", - Timestamp: time.Unix(0, 1498688360904552000), + Timestamp: time.Unix(0, 1498688360904552000).UTC(), Duration: time.Duration(50410) * time.Microsecond, Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ @@ -72,23 +73,23 @@ func TestLineProtocolConverter_Record(t *testing.T) { TraceID: "2505404965370368069", Name: "Parent", ParentID: "22964302721410078", - Timestamp: time.Unix(0, 1498688360851318000), + Timestamp: time.Unix(0, 1498688360851318000).UTC(), Duration: time.Duration(103680) * time.Microsecond, Annotations: []Annotation{ Annotation{ - Timestamp: time.Unix(0, 1498688360851325000), + Timestamp: time.Unix(0, 1498688360851325000).UTC(), Value: "Starting child #0", Host: "2130706433:0", ServiceName: "trivial", }, Annotation{ - Timestamp: time.Unix(0, 1498688360904545000), + Timestamp: time.Unix(0, 1498688360904545000).UTC(), Value: "Starting child #1", Host: "2130706433:0", ServiceName: "trivial", }, Annotation{ - Timestamp: time.Unix(0, 1498688360954992000), + Timestamp: time.Unix(0, 1498688360954992000).UTC(), Value: "A Log", Host: "2130706433:0", ServiceName: "trivial", @@ -123,7 +124,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { Fields: map[string]interface{}{ "duration": time.Duration(53106) * time.Microsecond, }, - Time: time.Unix(0, 1498688360851331000), + Time: time.Unix(0, 1498688360851331000).UTC(), }, testutil.Metric{ Measurement: "zipkin", @@ -141,7 +142,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { Fields: map[string]interface{}{ "duration": time.Duration(50410) * time.Microsecond, }, - Time: time.Unix(0, 1498688360904552000), + Time: time.Unix(0, 1498688360904552000).UTC(), }, testutil.Metric{ Measurement: "zipkin", @@ -158,7 +159,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "annotation_timestamp": int64(1498688360), "duration": time.Duration(103680) * time.Microsecond, }, - Time: time.Unix(0, 1498688360851318000), + Time: time.Unix(0, 1498688360851318000).UTC(), }, testutil.Metric{ Measurement: "zipkin", @@ -175,7 +176,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "annotation_timestamp": int64(1498688360), "duration": time.Duration(103680) * time.Microsecond, }, - Time: time.Unix(0, 1498688360851318000), + Time: time.Unix(0, 1498688360851318000).UTC(), }, testutil.Metric{ Measurement: "zipkin", @@ -192,7 +193,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "annotation_timestamp": int64(1498688360), "duration": time.Duration(103680) * time.Microsecond, }, - Time: time.Unix(0, 1498688360851318000), + Time: time.Unix(0, 1498688360851318000).UTC(), }, testutil.Metric{ Measurement: "zipkin", @@ -210,7 +211,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { Fields: map[string]interface{}{ "duration": time.Duration(103680) * time.Microsecond, }, - Time: time.Unix(0, 1498688360851318000), + Time: time.Unix(0, 1498688360851318000).UTC(), }, }, wantErr: false, @@ -230,11 +231,11 @@ func TestLineProtocolConverter_Record(t *testing.T) { TraceID: "243463817635710260", Name: "Concat", ParentID: "4574092882326506380", - Timestamp: time.Unix(0, 1499817952283903000), + Timestamp: time.Unix(0, 1499817952283903000).UTC(), Duration: time.Duration(2888) * time.Microsecond, Annotations: []Annotation{ Annotaitons{ - Timestamp: time.Unix(0, 1499817952283903000), + Timestamp: time.Unix(0, 1499817952283903000).UTC(), Value: "cs", Host: "0:0", ServiceName: "cli", @@ -266,7 +267,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "annotation_timestamp": int64(149981795), "duration": time.Duration(2888) * time.Microsecond, }, - Time: time.Unix(0, 1499817952283903000), + Time: time.Unix(0, 1499817952283903000).UTC(), }, testutil.Metric{ Measurement: "zipkin", @@ -284,7 +285,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { Fields: map[string]interface{}{ "duration": time.Duration(2888) * time.Microsecond, }, - Time: time.Unix(0, 1499817952283903000), + Time: time.Unix(0, 1499817952283903000).UTC(), }, wantErr: false, },*/ @@ -303,11 +304,11 @@ func TestLineProtocolConverter_Record(t *testing.T) { TraceID: "0:6802735349851856000", Name: "main.dud", ParentID: "6802735349851856000", - Timestamp: time.Unix(1, 0), + Timestamp: time.Unix(1, 0).UTC(), Duration: 1, Annotations: []Annotation{ Annotation{ - Timestamp: time.Unix(0, 1433330263415871000), + Timestamp: time.Unix(0, 1433330263415871000).UTC(), Value: "cs", Host: "0:9410", ServiceName: "go-zipkin-testclient", @@ -333,7 +334,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "annotation_timestamp": int64(1433330263), "duration": time.Duration(1) * time.Nanosecond, }, - Time: time.Unix(1, 0), + Time: time.Unix(1, 0).UTC(), }, }, }, @@ -357,3 +358,165 @@ func TestLineProtocolConverter_Record(t *testing.T) { }) } } + +func Test_microToTime(t *testing.T) { + type args struct { + micro int64 + } + tests := []struct { + name string + args args + want time.Time + }{ + { + name: "given zero micro seconds expected unix time zero", + args: args{ + micro: 0, + }, + want: time.Unix(0, 0).UTC(), + }, + { + name: "given a million micro seconds expected unix time one", + args: args{ + micro: 1000000, + }, + want: time.Unix(1, 0).UTC(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := microToTime(tt.args.micro); !reflect.DeepEqual(got, tt.want) { + t.Errorf("microToTime() = %v, want %v", got, tt.want) + } + }) + } +} + +func newAnnotation(micro int64) *zipkincore.Annotation { + return &zipkincore.Annotation{ + Timestamp: micro, + } +} + +func Test_minMax(t *testing.T) { + type args struct { + span *zipkincore.Span + } + tests := []struct { + name string + args args + wantMin time.Time + wantMax time.Time + }{ + { + name: "Single annotation", + args: args{ + span: &zipkincore.Span{ + Annotations: []*zipkincore.Annotation{ + newAnnotation(1000000), + }, + }, + }, + wantMin: time.Unix(1, 0).UTC(), + wantMax: time.Unix(1, 0).UTC(), + }, + { + name: "Three annotations", + args: args{ + span: &zipkincore.Span{ + Annotations: []*zipkincore.Annotation{ + newAnnotation(1000000), + newAnnotation(2000000), + newAnnotation(3000000), + }, + }, + }, + wantMin: time.Unix(1, 0).UTC(), + wantMax: time.Unix(3, 0).UTC(), + }, + /*{ + name: "zero time annotation", + args: args{ + span: &zipkincore.Span{ + Annotations: []*zipkincore.Annotation{ + newAnnotation(0), + }, + }, + }, + wantMin: time.Unix(0, 0).UTC(), + wantMax: time.Unix(1, 0).UTC(), + },*/ + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := minMax(tt.args.span) + if !reflect.DeepEqual(got, tt.wantMin) { + t.Errorf("minMax() got = %v, want %v", got, tt.wantMin) + } + if !reflect.DeepEqual(got1, tt.wantMax) { + t.Errorf("minMax() got1 = %v, want %v", got1, tt.wantMax) + } + }) + } +} + +func Test_host(t *testing.T) { + type args struct { + h *zipkincore.Endpoint + } + tests := []struct { + name string + args args + want string + }{ + { + name: "No host", + args: args{ + h: &zipkincore.Endpoint{ + Ipv4: 1234, + Port: 8888, + }, + }, + want: "1234:8888", + }, + { + name: "Host", + args: args{ + h: nil, + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := host(tt.args.h); got != tt.want { + t.Errorf("host() = %v, want %v", got, tt.want) + } + }) + } +} + +/* +func Test_serviceName(t *testing.T) { + type args struct { + h *zipkincore.Endpoint + } + tests := []struct { + name string + args args + want string + }{ + name: "no serviceName" + args: args{ + h: &zipkincore.Endpoint{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := serviceName(tt.args.h); got != tt.want { + t.Errorf("serviceName() = %v, want %v", got, tt.want) + } + }) + } +} +*/ From 72d099730d448b48576654e8206f36c9a9b5d4f1 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 19 Jul 2017 18:17:01 -0600 Subject: [PATCH 086/154] Temporarily comment out un-used test files in zipkin plugin --- plugins/inputs/zipkin/plugin_test.go | 2 ++ plugins/inputs/zipkin/unit_tests.go | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index 7b7346fc555dc..1a187989f1922 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -1,5 +1,6 @@ package zipkin +/* import ( "bytes" "fmt" @@ -205,3 +206,4 @@ func postTestData(t *testing.T, datafile string) { t.Fatal("http request failed") } } +*/ diff --git a/plugins/inputs/zipkin/unit_tests.go b/plugins/inputs/zipkin/unit_tests.go index 01ebfdb85ecb8..cc2795f4f09a9 100644 --- a/plugins/inputs/zipkin/unit_tests.go +++ b/plugins/inputs/zipkin/unit_tests.go @@ -1,6 +1,6 @@ package zipkin -import "time" +/*import "time" // UnitTest represents contains expected test values and a data file to be // written to the zipkin http server. @@ -182,3 +182,4 @@ var tests = []UnitTest{ }, }, } +*/ From 554df49c0a2798ae3005d162a922a1714d658436 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 20 Jul 2017 10:45:58 -0600 Subject: [PATCH 087/154] Update test to expect UTC times in zipkin plugin --- plugins/inputs/zipkin/handler_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 4ca4549c8ec49..e8822ef9c1c08 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -57,7 +57,7 @@ func TestSpanHandler(t *testing.T) { ID: "8090652509916334619", TraceID: "0:2505404965370368069", ParentID: parentID, - Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)), + Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)).UTC(), Duration: time.Duration(53106) * time.Microsecond, Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ @@ -75,7 +75,7 @@ func TestSpanHandler(t *testing.T) { ID: "103618986556047333", TraceID: "0:2505404965370368069", ParentID: parentID, - Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)), + Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)).UTC(), Duration: time.Duration(50410) * time.Microsecond, Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ @@ -93,23 +93,23 @@ func TestSpanHandler(t *testing.T) { ID: "22964302721410078", TraceID: "0:2505404965370368069", ParentID: "22964302721410078", - Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)), + Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)).UTC(), Duration: time.Duration(103680) * time.Microsecond, Annotations: []Annotation{ Annotation{ - Timestamp: time.Unix(0, 1498688360851325*int64(time.Microsecond)), + Timestamp: time.Unix(0, 1498688360851325*int64(time.Microsecond)).UTC(), Value: "Starting child #0", Host: "2130706433:0", ServiceName: "trivial", }, Annotation{ - Timestamp: time.Unix(0, 1498688360904545*int64(time.Microsecond)), + Timestamp: time.Unix(0, 1498688360904545*int64(time.Microsecond)).UTC(), Value: "Starting child #1", Host: "2130706433:0", ServiceName: "trivial", }, Annotation{ - Timestamp: time.Unix(0, 1498688360954992*int64(time.Microsecond)), + Timestamp: time.Unix(0, 1498688360954992*int64(time.Microsecond)).UTC(), Value: "A Log", Host: "2130706433:0", ServiceName: "trivial", From 017a682302b6f10d0c455f7ae61f4ce0481c277a Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 20 Jul 2017 10:47:36 -0600 Subject: [PATCH 088/154] add tests for minMax in zipkin plugin --- plugins/inputs/zipkin/convert.go | 5 ++- plugins/inputs/zipkin/convert_test.go | 59 ++++++++++++++++++++------- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 8ef90dbbd7c1e..ead9da17f5d5e 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -8,6 +8,9 @@ import ( "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) +//now is a moackable time for now +var now = time.Now + // LineProtocolConverter implements the Recorder interface; it is a // type meant to encapsulate the storage of zipkin tracing data in // telegraf as line protocol. @@ -137,7 +140,7 @@ func formatTraceID(high, low int64) string { } func minMax(span *zipkincore.Span) (time.Time, time.Time) { - min := time.Now().UTC() + min := now().UTC() max := time.Time{}.UTC() for _, annotation := range span.Annotations { ts := microToTime(annotation.GetTimestamp()) diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index 85c8dd8e1b0d7..bd2e95bfe4155 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -405,6 +405,7 @@ func Test_minMax(t *testing.T) { tests := []struct { name string args args + now func() time.Time wantMin time.Time wantMax time.Time }{ @@ -434,21 +435,40 @@ func Test_minMax(t *testing.T) { wantMin: time.Unix(1, 0).UTC(), wantMax: time.Unix(3, 0).UTC(), }, - /*{ - name: "zero time annotation", + { + name: "Annotations are in the future", args: args{ span: &zipkincore.Span{ Annotations: []*zipkincore.Annotation{ - newAnnotation(0), + newAnnotation(3000000), }, }, }, - wantMin: time.Unix(0, 0).UTC(), - wantMax: time.Unix(1, 0).UTC(), - },*/ + wantMin: time.Unix(2, 0).UTC(), + wantMax: time.Unix(3, 0).UTC(), + now: func() time.Time { + return time.Unix(2, 0).UTC() + }, + }, + { + name: "No Annotations", + args: args{ + span: &zipkincore.Span{ + Annotations: []*zipkincore.Annotation{}, + }, + }, + wantMin: time.Unix(2, 0).UTC(), + wantMax: time.Unix(2, 0).UTC(), + now: func() time.Time { + return time.Unix(2, 0).UTC() + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.now != nil { + now = tt.now + } got, got1 := minMax(tt.args.span) if !reflect.DeepEqual(got, tt.wantMin) { t.Errorf("minMax() got = %v, want %v", got, tt.wantMin) @@ -456,6 +476,7 @@ func Test_minMax(t *testing.T) { if !reflect.DeepEqual(got1, tt.wantMax) { t.Errorf("minMax() got1 = %v, want %v", got1, tt.wantMax) } + now = time.Now }) } } @@ -470,7 +491,7 @@ func Test_host(t *testing.T) { want string }{ { - name: "No host", + name: "Host Found", args: args{ h: &zipkincore.Endpoint{ Ipv4: 1234, @@ -480,7 +501,7 @@ func Test_host(t *testing.T) { want: "1234:8888", }, { - name: "Host", + name: "No Host", args: args{ h: nil, }, @@ -496,7 +517,6 @@ func Test_host(t *testing.T) { } } -/* func Test_serviceName(t *testing.T) { type args struct { h *zipkincore.Endpoint @@ -506,10 +526,22 @@ func Test_serviceName(t *testing.T) { args args want string }{ - name: "no serviceName" - args: args{ - h: &zipkincore.Endpoint{}, - }, + { + name: "Found ServiceName", + args: args{ + h: &zipkincore.Endpoint{ + ServiceName: "zipkin", + }, + }, + want: "zipkin", + }, + { + name: "No ServiceName", + args: args{ + h: nil, + }, + want: "", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -519,4 +551,3 @@ func Test_serviceName(t *testing.T) { }) } } -*/ From ed792f9a42e9d999015c99abd38f9ee70d930401 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 20 Jul 2017 10:51:50 -0600 Subject: [PATCH 089/154] Update zipkin tests to use UTC time --- plugins/inputs/zipkin/plugin_test.go | 2 -- plugins/inputs/zipkin/unit_tests.go | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/plugin_test.go index 1a187989f1922..7b7346fc555dc 100644 --- a/plugins/inputs/zipkin/plugin_test.go +++ b/plugins/inputs/zipkin/plugin_test.go @@ -1,6 +1,5 @@ package zipkin -/* import ( "bytes" "fmt" @@ -206,4 +205,3 @@ func postTestData(t *testing.T, datafile string) { t.Fatal("http request failed") } } -*/ diff --git a/plugins/inputs/zipkin/unit_tests.go b/plugins/inputs/zipkin/unit_tests.go index cc2795f4f09a9..3b138be9a5cdc 100644 --- a/plugins/inputs/zipkin/unit_tests.go +++ b/plugins/inputs/zipkin/unit_tests.go @@ -1,6 +1,6 @@ package zipkin -/*import "time" +import "time" // UnitTest represents contains expected test values and a data file to be // written to the zipkin http server. @@ -116,7 +116,7 @@ var tests = []UnitTest{ }, expectedValues: map[string]interface{}{ "duration": time.Duration(103680) * time.Microsecond, - "time": time.Unix(1498688360, 851318*int64(time.Microsecond)), + "time": time.Unix(1498688360, 851318*int64(time.Microsecond)).UTC(), }, }, }, @@ -182,4 +182,3 @@ var tests = []UnitTest{ }, }, } -*/ From 58d2352cf7417a1ac80b8d79edaf2385bb501b6d Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 20 Jul 2017 10:54:56 -0600 Subject: [PATCH 090/154] Rename plugin test to zipkin test --- plugins/inputs/zipkin/{plugin_test.go => zipkin_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/{plugin_test.go => zipkin_test.go} (100%) diff --git a/plugins/inputs/zipkin/plugin_test.go b/plugins/inputs/zipkin/zipkin_test.go similarity index 100% rename from plugins/inputs/zipkin/plugin_test.go rename to plugins/inputs/zipkin/zipkin_test.go From d4e5e646dae3a31559e077e646f6eaa582e51189 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Thu, 20 Jul 2017 10:57:48 -0600 Subject: [PATCH 091/154] Combine unit_test.go to zipkin_test.go --- plugins/inputs/zipkin/unit_tests.go | 184 --------------------------- plugins/inputs/zipkin/zipkin_test.go | 181 ++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 184 deletions(-) delete mode 100644 plugins/inputs/zipkin/unit_tests.go diff --git a/plugins/inputs/zipkin/unit_tests.go b/plugins/inputs/zipkin/unit_tests.go deleted file mode 100644 index 3b138be9a5cdc..0000000000000 --- a/plugins/inputs/zipkin/unit_tests.go +++ /dev/null @@ -1,184 +0,0 @@ -package zipkin - -import "time" - -// UnitTest represents contains expected test values and a data file to be -// written to the zipkin http server. -type UnitTest struct { - expected []TestData - measurement string - datafile string - waitPoints int -} - -// TestData contains the expected tags and values that the telegraf plugin -// should output -type TestData struct { - expectedTags map[string]string - expectedValues map[string]interface{} -} - -// Store all unit tests in an array to allow for iteration over all tests -var tests = []UnitTest{ - UnitTest{ - measurement: "zipkin", - datafile: "testdata/threespans.dat", - expected: []TestData{ - // zipkin data points are stored in InfluxDB tagged partly //annotation specific - //values, and partly on span specific values, - // so there are many repeated tags. Fields have very similar tags, which is why - // tags are relatively redundant in these tests. - { - expectedTags: map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "2130706433:0", - "key": "lc", - "type": "STRING", - }, - expectedValues: map[string]interface{}{ - "duration": time.Duration(53106) * time.Microsecond, - }, - }, - { - expectedTags: map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "2130706433:0", - "key": "lc", - "type": "STRING", - }, - expectedValues: map[string]interface{}{ - "duration": time.Duration(50410) * time.Microsecond, - }, - }, - { - expectedTags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "Starting child #0", - "endpoint_host": "2130706433:0", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1498688360851325), - }, - }, - { - expectedTags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "Starting child #1", - "endpoint_host": "2130706433:0", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1498688360904545), - }, - }, - { - expectedTags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "A Log", - "endpoint_host": "2130706433:0", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1498688360954992), - }, - }, - { - expectedTags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "2130706433:0", - "key": "lc", - "type": "STRING", - }, - expectedValues: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, - "time": time.Unix(1498688360, 851318*int64(time.Microsecond)).UTC(), - }, - }, - }, - }, - - // Test data from zipkin cli app: - //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services - UnitTest{ - measurement: "zipkin", - datafile: "testdata/cli_microservice.dat", - expected: []TestData{ - { - expectedTags: map[string]string{ - "id": "3383422996321511664", - "parent_id": "4574092882326506380", - "trace_id": "0:8269862291023777619243463817635710260", - "name": "Concat", - "service_name": "cli", - "annotation_value": "cs", - "endpoint_host": "0:0", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1499817952283903), - }, - }, - }, - }, - - // Test data from distributed trace repo sample json - // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json - UnitTest{ - measurement: "zipkin", - datafile: "testdata/distributed_trace_sample.dat", - expected: []TestData{ - { - expectedTags: map[string]string{ - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", - "name": "main.dud", - "service_name": "go-zipkin-testclient", - "annotation_value": "cs", - "endpoint_host": "0:9410", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1433330263415871), - }, - }, - { - expectedTags: map[string]string{ - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", - "name": "main.dud", - "service_name": "go-zipkin-testclient", - "annotation_value": "cr", - "endpoint_host": "0:9410", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1433330263415872), - }, - }, - }, - }, -} diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 7b7346fc555dc..4102133781de5 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -205,3 +205,184 @@ func postTestData(t *testing.T, datafile string) { t.Fatal("http request failed") } } + +// UnitTest represents contains expected test values and a data file to be +// written to the zipkin http server. +type UnitTest struct { + expected []TestData + measurement string + datafile string + waitPoints int +} + +// TestData contains the expected tags and values that the telegraf plugin +// should output +type TestData struct { + expectedTags map[string]string + expectedValues map[string]interface{} +} + +// Store all unit tests in an array to allow for iteration over all tests +var tests = []UnitTest{ + UnitTest{ + measurement: "zipkin", + datafile: "testdata/threespans.dat", + expected: []TestData{ + // zipkin data points are stored in InfluxDB tagged partly //annotation specific + //values, and partly on span specific values, + // so there are many repeated tags. Fields have very similar tags, which is why + // tags are relatively redundant in these tests. + { + expectedTags: map[string]string{ + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + expectedValues: map[string]interface{}{ + "duration": time.Duration(53106) * time.Microsecond, + }, + }, + { + expectedTags: map[string]string{ + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + expectedValues: map[string]interface{}{ + "duration": time.Duration(50410) * time.Microsecond, + }, + }, + { + expectedTags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "Starting child #0", + "endpoint_host": "2130706433:0", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1498688360851325), + }, + }, + { + expectedTags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "Starting child #1", + "endpoint_host": "2130706433:0", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1498688360904545), + }, + }, + { + expectedTags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "A Log", + "endpoint_host": "2130706433:0", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1498688360954992), + }, + }, + { + expectedTags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "trivial", + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + expectedValues: map[string]interface{}{ + "duration": time.Duration(103680) * time.Microsecond, + "time": time.Unix(1498688360, 851318*int64(time.Microsecond)).UTC(), + }, + }, + }, + }, + + // Test data from zipkin cli app: + //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services + UnitTest{ + measurement: "zipkin", + datafile: "testdata/cli_microservice.dat", + expected: []TestData{ + { + expectedTags: map[string]string{ + "id": "3383422996321511664", + "parent_id": "4574092882326506380", + "trace_id": "0:8269862291023777619243463817635710260", + "name": "Concat", + "service_name": "cli", + "annotation_value": "cs", + "endpoint_host": "0:0", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1499817952283903), + }, + }, + }, + }, + + // Test data from distributed trace repo sample json + // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json + UnitTest{ + measurement: "zipkin", + datafile: "testdata/distributed_trace_sample.dat", + expected: []TestData{ + { + expectedTags: map[string]string{ + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", + "annotation_value": "cs", + "endpoint_host": "0:9410", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1433330263415871), + }, + }, + { + expectedTags: map[string]string{ + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", + "annotation_value": "cr", + "endpoint_host": "0:9410", + }, + expectedValues: map[string]interface{}{ + "annotation_timestamp": int64(1433330263415872), + }, + }, + }, + }, +} From f23b9bded75a942805a0c1bc2d98b460badd613f Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Sun, 23 Jul 2017 11:43:17 -0600 Subject: [PATCH 092/154] Add bitshift format to host in zipkin plugin --- plugins/inputs/zipkin/convert.go | 17 ++++++++++++++++- plugins/inputs/zipkin/convert_test.go | 2 +- plugins/inputs/zipkin/handler_test.go | 12 ++++++------ plugins/inputs/zipkin/zipkin_test.go | 18 +++++++++--------- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index ead9da17f5d5e..61909a80deac9 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -1,6 +1,8 @@ package zipkin import ( + "bytes" + "encoding/binary" "strconv" "time" @@ -185,11 +187,24 @@ func parentID(span *zipkincore.Span) string { return formatID(span.ID) } +func ipv4(addr int32) string { + var buf [4]byte + binary.BigEndian.PutUint32(buf[:], uint32(addr)) + var sBuf bytes.Buffer + for i, b := range buf { + if i > 0 { + sBuf.WriteRune('.') + } + sBuf.WriteString(strconv.FormatUint(uint64(b), 10)) + } + return sBuf.String() +} + func host(h *zipkincore.Endpoint) string { if h == nil { return "" } - return strconv.Itoa(int(h.GetIpv4())) + ":" + strconv.Itoa(int(h.GetPort())) + return ipv4(h.GetIpv4()) + ":" + strconv.Itoa(int(h.GetPort())) } func serviceName(h *zipkincore.Endpoint) string { diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index bd2e95bfe4155..cdb365b03d238 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -498,7 +498,7 @@ func Test_host(t *testing.T) { Port: 8888, }, }, - want: "1234:8888", + want: "0.0.4.210:8888", }, { name: "No Host", diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index e8822ef9c1c08..ce71ec716115e 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -64,7 +64,7 @@ func TestSpanHandler(t *testing.T) { BinaryAnnotation{ Key: "lc", Value: "trivial", - Host: "2130706433:0", + Host: "127.0.0.1:0", ServiceName: "trivial", Type: "STRING", }, @@ -82,7 +82,7 @@ func TestSpanHandler(t *testing.T) { BinaryAnnotation{ Key: "lc", Value: "trivial", - Host: "2130706433:0", + Host: "127.0.0.1:0", ServiceName: "trivial", Type: "STRING", }, @@ -99,19 +99,19 @@ func TestSpanHandler(t *testing.T) { Annotation{ Timestamp: time.Unix(0, 1498688360851325*int64(time.Microsecond)).UTC(), Value: "Starting child #0", - Host: "2130706433:0", + Host: "127.0.0.1:0", ServiceName: "trivial", }, Annotation{ Timestamp: time.Unix(0, 1498688360904545*int64(time.Microsecond)).UTC(), Value: "Starting child #1", - Host: "2130706433:0", + Host: "127.0.0.1:0", ServiceName: "trivial", }, Annotation{ Timestamp: time.Unix(0, 1498688360954992*int64(time.Microsecond)).UTC(), Value: "A Log", - Host: "2130706433:0", + Host: "127.0.0.1:0", ServiceName: "trivial", }, }, @@ -119,7 +119,7 @@ func TestSpanHandler(t *testing.T) { BinaryAnnotation{ Key: "lc", Value: "trivial", - Host: "2130706433:0", + Host: "127.0.0.1:0", ServiceName: "trivial", Type: "STRING", }, diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 4102133781de5..6138b97775527 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -240,7 +240,7 @@ var tests = []UnitTest{ "name": "Child", "service_name": "trivial", "annotation_value": "trivial", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", "key": "lc", "type": "STRING", }, @@ -256,7 +256,7 @@ var tests = []UnitTest{ "name": "Child", "service_name": "trivial", "annotation_value": "trivial", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", "key": "lc", "type": "STRING", }, @@ -272,7 +272,7 @@ var tests = []UnitTest{ "name": "Parent", "service_name": "trivial", "annotation_value": "Starting child #0", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", }, expectedValues: map[string]interface{}{ "annotation_timestamp": int64(1498688360851325), @@ -286,7 +286,7 @@ var tests = []UnitTest{ "name": "Parent", "service_name": "trivial", "annotation_value": "Starting child #1", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", }, expectedValues: map[string]interface{}{ "annotation_timestamp": int64(1498688360904545), @@ -300,7 +300,7 @@ var tests = []UnitTest{ "name": "Parent", "service_name": "trivial", "annotation_value": "A Log", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", }, expectedValues: map[string]interface{}{ "annotation_timestamp": int64(1498688360954992), @@ -314,7 +314,7 @@ var tests = []UnitTest{ "name": "Parent", "service_name": "trivial", "annotation_value": "trivial", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", "key": "lc", "type": "STRING", }, @@ -340,7 +340,7 @@ var tests = []UnitTest{ "name": "Concat", "service_name": "cli", "annotation_value": "cs", - "endpoint_host": "0:0", + "endpoint_host": "0.0.0.0:0", }, expectedValues: map[string]interface{}{ "annotation_timestamp": int64(1499817952283903), @@ -363,7 +363,7 @@ var tests = []UnitTest{ "name": "main.dud", "service_name": "go-zipkin-testclient", "annotation_value": "cs", - "endpoint_host": "0:9410", + "endpoint_host": "0.0.0.0:9410", }, expectedValues: map[string]interface{}{ "annotation_timestamp": int64(1433330263415871), @@ -377,7 +377,7 @@ var tests = []UnitTest{ "name": "main.dud", "service_name": "go-zipkin-testclient", "annotation_value": "cr", - "endpoint_host": "0:9410", + "endpoint_host": "0.0.0.0:9410", }, expectedValues: map[string]interface{}{ "annotation_timestamp": int64(1433330263415872), From 9265d808d0b3d534c51df36189a35b33c52afcba Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 13:15:19 -0700 Subject: [PATCH 093/154] Add full integration test for zipkin plugin --- plugins/inputs/zipkin/zipkin_test.go | 313 ++++++++++++++++++++++++++- 1 file changed, 311 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 6138b97775527..4c828bc3d4850 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -4,16 +4,16 @@ import ( "bytes" "fmt" "io/ioutil" - "log" "net/http" "reflect" "testing" "time" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" ) -func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { +/*func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { postTestData(t, u.datafile) if u.waitPoints == 0 { acc.Wait(len(u.expected)) @@ -385,4 +385,313 @@ var tests = []UnitTest{ }, }, }, +}*/ + +func TestZipkinPlugin(t *testing.T) { + mockAcc := testutil.Accumulator{} + type fields struct { + acc telegraf.Accumulator + } + type args struct { + t Trace + } + tests := []struct { + name string + fields fields + thriftDataFile string //pathname to a binary thrift data file which contains test data + wantErr bool + want []testutil.Metric + }{ + { + name: "threespan", + fields: fields{ + acc: &mockAcc, + }, + thriftDataFile: "testdata/threespans.dat", + want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "trivial", //base64: dHJpdmlhbA== + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(53106) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851331000).UTC(), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation_value": "trivial", //base64: dHJpdmlhbA== + "endpoint_host": "2130706433:0", + "key": "lc", + "type": "STRING", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(50410) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360904552000).UTC(), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "service_name": "trivial", + "annotation_value": "Starting child #0", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1498688360), + "duration": time.Duration(103680) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851318000).UTC(), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "service_name": "trivial", + "annotation_value": "Starting child #1", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1498688360), + "duration": time.Duration(103680) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851318000).UTC(), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation_value": "A Log", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1498688360), + "duration": time.Duration(103680) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851318000).UTC(), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "trace_id": "0:2505404965370368069", + "service_name": "trivial", + "annotation_value": "trivial", //base64: dHJpdmlhbA== + "key": "lc", + "type": "STRING", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "name": "Parent", + "endpoint_host": "2130706433:0", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(103680) * time.Microsecond, + }, + Time: time.Unix(0, 1498688360851318000).UTC(), + }, + }, + wantErr: false, + }, + + // Test data from zipkin cli app: + //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services + /*{ + name: "cli", + fields: fields{ + acc: &mockAcc, + }, + args: args{ + t: Trace{ + Span{ + ID: "3383422996321511664", + TraceID: "243463817635710260", + Name: "Concat", + ParentID: "4574092882326506380", + Timestamp: time.Unix(0, 1499817952283903000).UTC(), + Duration: time.Duration(2888) * time.Microsecond, + Annotations: []Annotation{ + Annotaitons{ + Timestamp: time.Unix(0, 1499817952283903000).UTC(), + Value: "cs", + Host: "0:0", + ServiceName: "cli", + }, + }, + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "http.url", + Value: "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", + Host: "0:0", + ServiceName: "cli", + Type: "STRING", + }, + }, + }, + want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "3383422996321511664", + "parent_id": "4574092882326506380", + "trace_id": "8269862291023777619:243463817635710260", + "name": "Concat", + "service_name": "cli", + "annotation_value": "cs", + "endpoint_host": "0:0", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(149981795), + "duration": time.Duration(2888) * time.Microsecond, + }, + Time: time.Unix(0, 1499817952283903000).UTC(), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "trace_id": "2505404965370368069", + "service_name": "cli", + "annotation_value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", + "key": "http.url", + "type": "STRING", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "name": "Concat", + "endpoint_host": "0:0", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(2888) * time.Microsecond, + }, + Time: time.Unix(0, 1499817952283903000).UTC(), + }, + wantErr: false, + },*/ + + //// Test data from distributed trace repo sample json + // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json + { + name: "distributed_trace_sample", + fields: fields{ + acc: &mockAcc, + }, + thriftDataFile: "testdata/distributed_trace_sample.dat", + want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "annotation_value": "cs", + "endpoint_host": "0:9410", + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1433330263), + "duration": time.Duration(1) * time.Microsecond, + }, + //Time: time.Unix(1, 0).UTC(), + Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), + }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "annotation_value": "cr", + "endpoint_host": "0:9410", + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", + }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(1433330263), + "duration": time.Duration(1) * time.Microsecond, + }, + Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), + }, + }, + }, + } + + z := &Zipkin{ + Path: "/api/v1/spans", + Port: 9411, + } + + err := z.Start(&mockAcc) + if err != nil { + t.Fatal("Failed to start zipkin server") + } + + defer z.Stop() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockAcc.ClearMetrics() + if err := postThriftData(tt.thriftDataFile); err != nil { + t.Fatalf("Posting data to http endpoint /api/v1/spans failed. Error: %s\n", err) + } + mockAcc.Wait(len(tt.want)) //Since the server is running concurrently, we need to wait for the number of data points we want to test to be added to the Accumulator. + if len(mockAcc.Errors) > 0 != tt.wantErr { + t.Fatalf("Got unexpected errors. want error = %v, errors = %v\n", tt.wantErr, mockAcc.Errors) + } + + var got []testutil.Metric + for _, m := range mockAcc.Metrics { + got = append(got, *m) + } + + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("Got != Want\n Got: %#v\n, Want: %#v\n", got, tt.want) + } + }) + } +} + +func postThriftData(datafile string) error { + dat, err := ioutil.ReadFile(datafile) + if err != nil { + return fmt.Errorf("could not read from data file %s", datafile) + } + + req, err := http.NewRequest("POST", "http://localhost:9411/api/v1/spans", bytes.NewReader(dat)) + + if err != nil { + return fmt.Errorf("HTTP request creation failed") + } + + req.Header.Set("Content-Type", "application/x-thrift") + client := &http.Client{} + _, err = client.Do(req) + if err != nil { + return fmt.Errorf("HTTP POST request to zipkin endpoint http://localhost:/9411/v1/spans failed") + } + + return nil } From 1754e13e740599500e563cd2158b0b7d64444ad3 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 13:42:06 -0700 Subject: [PATCH 094/154] Update test table in zipkin_test.go for new ipv4 string formatting --- plugins/inputs/zipkin/zipkin_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 4c828bc3d4850..44e1cd74cbafe 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -418,7 +418,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Child", "service_name": "trivial", "annotation_value": "trivial", //base64: dHJpdmlhbA== - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", "key": "lc", "type": "STRING", }, @@ -436,7 +436,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Child", "service_name": "trivial", "annotation_value": "trivial", //base64: dHJpdmlhbA== - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", "key": "lc", "type": "STRING", }, @@ -450,7 +450,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "service_name": "trivial", "annotation_value": "Starting child #0", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", "id": "22964302721410078", "parent_id": "22964302721410078", "trace_id": "0:2505404965370368069", @@ -467,7 +467,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "service_name": "trivial", "annotation_value": "Starting child #1", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", "id": "22964302721410078", "parent_id": "22964302721410078", "trace_id": "0:2505404965370368069", @@ -487,7 +487,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Parent", "service_name": "trivial", "annotation_value": "A Log", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", "id": "22964302721410078", }, Fields: map[string]interface{}{ @@ -507,7 +507,7 @@ func TestZipkinPlugin(t *testing.T) { "id": "22964302721410078", "parent_id": "22964302721410078", "name": "Parent", - "endpoint_host": "2130706433:0", + "endpoint_host": "127.0.0.1:0", }, Fields: map[string]interface{}{ "duration": time.Duration(103680) * time.Microsecond, @@ -604,7 +604,7 @@ func TestZipkinPlugin(t *testing.T) { Measurement: "zipkin", Tags: map[string]string{ "annotation_value": "cs", - "endpoint_host": "0:9410", + "endpoint_host": "0.0.0.0:9410", "id": "6802735349851856000", "parent_id": "6802735349851856000", "trace_id": "0:6802735349851856000", @@ -622,7 +622,7 @@ func TestZipkinPlugin(t *testing.T) { Measurement: "zipkin", Tags: map[string]string{ "annotation_value": "cr", - "endpoint_host": "0:9410", + "endpoint_host": "0.0.0.0:9410", "id": "6802735349851856000", "parent_id": "6802735349851856000", "trace_id": "0:6802735349851856000", From f30e787fdc673db0bdc953669af8a440c3bebeb4 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 13:47:56 -0700 Subject: [PATCH 095/154] Refactor ipv4 string conversion function in zipkin plugin to use standard library --- plugins/inputs/zipkin/convert.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 61909a80deac9..94bcf44159b99 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -1,8 +1,8 @@ package zipkin import ( - "bytes" "encoding/binary" + "net" "strconv" "time" @@ -188,16 +188,9 @@ func parentID(span *zipkincore.Span) string { } func ipv4(addr int32) string { - var buf [4]byte - binary.BigEndian.PutUint32(buf[:], uint32(addr)) - var sBuf bytes.Buffer - for i, b := range buf { - if i > 0 { - sBuf.WriteRune('.') - } - sBuf.WriteString(strconv.FormatUint(uint64(b), 10)) - } - return sBuf.String() + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, uint32(addr)) + return net.IP(buf).String() } func host(h *zipkincore.Endpoint) string { From 64dfd37dba462f1477b58eb73f821f92af3174df Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 13:55:37 -0700 Subject: [PATCH 096/154] Remove old comments from zipkin_test.go --- plugins/inputs/zipkin/zipkin_test.go | 450 --------------------------- 1 file changed, 450 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 44e1cd74cbafe..9afe69793910f 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -13,380 +13,6 @@ import ( "github.com/influxdata/telegraf/testutil" ) -/*func (u UnitTest) Run(t *testing.T, acc *testutil.Accumulator) { - postTestData(t, u.datafile) - if u.waitPoints == 0 { - acc.Wait(len(u.expected)) - } else { - acc.Wait(u.waitPoints) - } - - for _, data := range u.expected { - for key, value := range data.expectedValues { - switch value.(type) { - case int64: - //assertContainsTaggedInt64(t, acc, u.measurement, key, value.(int64), data.expectedTags) - break - case time.Duration: - assertContainsTaggedDuration(t, acc, u.measurement, key, value.(time.Duration), data.expectedTags) - break - case time.Time: - if key == "time" { - assertTimeIs(t, acc, u.measurement, value.(time.Time), data.expectedTags) - } else { - assertContainsTaggedTime(t, acc, u.measurement, key, value.(time.Time), data.expectedTags) - } - break - default: - t.Fatalf("Invalid type for field %v\n", reflect.TypeOf(value)) - break - } - } - } -} - -func TestZipkin(t *testing.T) { - var acc testutil.Accumulator - z := &Zipkin{ - Path: "/api/v1/test", - Port: 9411, - } - err := z.Start(&acc) - if err != nil { - t.Fatal("Failed to start zipkin server") - } - defer z.Stop() - - for _, test := range tests { - test.Run(t, &acc) - } - - //t.Fatal("ERROR!") -} - -func assertTimeIs(t *testing.T, acc *testutil.Accumulator, - measurement string, - expectedValue time.Time, - tags map[string]string) { - var actualValue time.Time - for _, pt := range acc.Metrics { - if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { - actualValue = pt.Time - if actualValue == expectedValue { - return - } - - t.Errorf("Expected value %v\n got value %v\n", expectedValue, actualValue) - - } - } - - msg := fmt.Sprintf( - "Could not find measurement \"%s\" with requested tags and time: %v, Actual: %v", - measurement, expectedValue, actualValue) - t.Fatal(msg) -} - -func assertContainsTaggedDuration( - t *testing.T, - acc *testutil.Accumulator, - measurement string, - field string, - expectedValue time.Duration, - tags map[string]string, -) { - var actualValue interface{} - for _, pt := range acc.Metrics { - if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { - for fieldname, value := range pt.Fields { - if fieldname == field { - actualValue = value - if value == expectedValue { - return - } - t.Errorf("Expected value %d\n got value %d\n", expectedValue, value) - } - } - } - } - msg := fmt.Sprintf( - "assertContainsTaggedDuration: Could not find measurement \"%s\" with requested tags within %s, Actual: %d", - measurement, field, actualValue) - t.Fatal(msg) -} - -func assertContainsTaggedInt64( - t *testing.T, - acc *testutil.Accumulator, - measurement string, - field string, - expectedValue int64, - tags map[string]string, -) { - log.Println("going through tagged ") - var actualValue interface{} - log.Println(acc.Metrics) - for _, pt := range acc.Metrics { - log.Println("looping, point is : ", pt) - log.Println("point tags are : ", pt.Tags) - log.Println("point fields are:", pt.Fields) - log.Println("tags are: ", tags) - if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { - log.Println("found measurement") - for fieldname, value := range pt.Fields { - fmt.Println("looping through fields, fieldname is: ", fieldname) - fmt.Println("user input field is: ", field) - if fieldname == field { - fmt.Println("found field: ", field) - actualValue = value - fmt.Println("Value: ", value) - if value == expectedValue { - return - } - t.Errorf("Expected value %v\n got value %v\n", expectedValue, value) - } else { - t.Errorf("Fieldname != field %s", fieldname) - } - } - } else if !reflect.DeepEqual(pt.Tags, tags) { - log.Printf("%s\n%s", pt.Tags, tags) - } - } - msg := fmt.Sprintf( - "assertContainsTaggedInt64: Could not find measurement \"%s\" with requested tags within %s, Actual: %d ,Expected: %d", - measurement, field, actualValue, expectedValue) - t.Fatal(msg) -} - -func assertContainsTaggedTime( - t *testing.T, - acc *testutil.Accumulator, - measurement string, - field string, - expectedValue time.Time, - tags map[string]string, -) { - var actualValue interface{} - for _, pt := range acc.Metrics { - if pt.Measurement == measurement && reflect.DeepEqual(pt.Tags, tags) { - for fieldname, value := range pt.Fields { - if fieldname == field { - actualValue = value - if value == expectedValue { - return - } - t.Errorf("Expected value %v\n got value %v\n", expectedValue, value) - } - } - } - } - msg := fmt.Sprintf( - "assertContainsTaggedTime: Could not find measurement \"%s\" with requested tags within %s, Actual: %d", - measurement, field, actualValue) - t.Fatal(msg) -} - -func postTestData(t *testing.T, datafile string) { - dat, err := ioutil.ReadFile(datafile) - if err != nil { - t.Fatal("Could not read from data file") - } - - req, err := http.NewRequest("POST", "http://localhost:9411/api/v1/test", bytes.NewReader(dat)) - - if err != nil { - t.Fatal("bad http request") - } - - req.Header.Set("Content-Type", "application/x-thrift") - client := &http.Client{} - _, err = client.Do(req) - if err != nil { - t.Fatal("http request failed") - } -} - -// UnitTest represents contains expected test values and a data file to be -// written to the zipkin http server. -type UnitTest struct { - expected []TestData - measurement string - datafile string - waitPoints int -} - -// TestData contains the expected tags and values that the telegraf plugin -// should output -type TestData struct { - expectedTags map[string]string - expectedValues map[string]interface{} -} - -// Store all unit tests in an array to allow for iteration over all tests -var tests = []UnitTest{ - UnitTest{ - measurement: "zipkin", - datafile: "testdata/threespans.dat", - expected: []TestData{ - // zipkin data points are stored in InfluxDB tagged partly //annotation specific - //values, and partly on span specific values, - // so there are many repeated tags. Fields have very similar tags, which is why - // tags are relatively redundant in these tests. - { - expectedTags: map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "127.0.0.1:0", - "key": "lc", - "type": "STRING", - }, - expectedValues: map[string]interface{}{ - "duration": time.Duration(53106) * time.Microsecond, - }, - }, - { - expectedTags: map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "127.0.0.1:0", - "key": "lc", - "type": "STRING", - }, - expectedValues: map[string]interface{}{ - "duration": time.Duration(50410) * time.Microsecond, - }, - }, - { - expectedTags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "Starting child #0", - "endpoint_host": "127.0.0.1:0", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1498688360851325), - }, - }, - { - expectedTags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "Starting child #1", - "endpoint_host": "127.0.0.1:0", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1498688360904545), - }, - }, - { - expectedTags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "A Log", - "endpoint_host": "127.0.0.1:0", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1498688360954992), - }, - }, - { - expectedTags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "trivial", - "endpoint_host": "127.0.0.1:0", - "key": "lc", - "type": "STRING", - }, - expectedValues: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, - "time": time.Unix(1498688360, 851318*int64(time.Microsecond)).UTC(), - }, - }, - }, - }, - - // Test data from zipkin cli app: - //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services - UnitTest{ - measurement: "zipkin", - datafile: "testdata/cli_microservice.dat", - expected: []TestData{ - { - expectedTags: map[string]string{ - "id": "3383422996321511664", - "parent_id": "4574092882326506380", - "trace_id": "0:8269862291023777619243463817635710260", - "name": "Concat", - "service_name": "cli", - "annotation_value": "cs", - "endpoint_host": "0.0.0.0:0", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1499817952283903), - }, - }, - }, - }, - - // Test data from distributed trace repo sample json - // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json - UnitTest{ - measurement: "zipkin", - datafile: "testdata/distributed_trace_sample.dat", - expected: []TestData{ - { - expectedTags: map[string]string{ - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", - "name": "main.dud", - "service_name": "go-zipkin-testclient", - "annotation_value": "cs", - "endpoint_host": "0.0.0.0:9410", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1433330263415871), - }, - }, - { - expectedTags: map[string]string{ - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", - "name": "main.dud", - "service_name": "go-zipkin-testclient", - "annotation_value": "cr", - "endpoint_host": "0.0.0.0:9410", - }, - expectedValues: map[string]interface{}{ - "annotation_timestamp": int64(1433330263415872), - }, - }, - }, - }, -}*/ - func TestZipkinPlugin(t *testing.T) { mockAcc := testutil.Accumulator{} type fields struct { @@ -517,82 +143,6 @@ func TestZipkinPlugin(t *testing.T) { }, wantErr: false, }, - - // Test data from zipkin cli app: - //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services - /*{ - name: "cli", - fields: fields{ - acc: &mockAcc, - }, - args: args{ - t: Trace{ - Span{ - ID: "3383422996321511664", - TraceID: "243463817635710260", - Name: "Concat", - ParentID: "4574092882326506380", - Timestamp: time.Unix(0, 1499817952283903000).UTC(), - Duration: time.Duration(2888) * time.Microsecond, - Annotations: []Annotation{ - Annotaitons{ - Timestamp: time.Unix(0, 1499817952283903000).UTC(), - Value: "cs", - Host: "0:0", - ServiceName: "cli", - }, - }, - BinaryAnnotations: []BinaryAnnotation{ - BinaryAnnotation{ - Key: "http.url", - Value: "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", - Host: "0:0", - ServiceName: "cli", - Type: "STRING", - }, - }, - }, - want: []testutil.Metric{ - testutil.Metric{ - Measurement: "zipkin", - Tags: map[string]string{ - "id": "3383422996321511664", - "parent_id": "4574092882326506380", - "trace_id": "8269862291023777619:243463817635710260", - "name": "Concat", - "service_name": "cli", - "annotation_value": "cs", - "endpoint_host": "0:0", - }, - Fields: map[string]interface{}{ - "annotation_timestamp": int64(149981795), - "duration": time.Duration(2888) * time.Microsecond, - }, - Time: time.Unix(0, 1499817952283903000).UTC(), - }, - testutil.Metric{ - Measurement: "zipkin", - Tags: map[string]string{ - "trace_id": "2505404965370368069", - "service_name": "cli", - "annotation_value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", - "key": "http.url", - "type": "STRING", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "name": "Concat", - "endpoint_host": "0:0", - }, - Fields: map[string]interface{}{ - "duration": time.Duration(2888) * time.Microsecond, - }, - Time: time.Unix(0, 1499817952283903000).UTC(), - }, - wantErr: false, - },*/ - - //// Test data from distributed trace repo sample json - // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json { name: "distributed_trace_sample", fields: fields{ From 0f48e484fced8efdcf4acabb87bd6afaf6b503c6 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 14:46:48 -0700 Subject: [PATCH 097/154] Comment inaccurate test data in zipkin plugin --- plugins/inputs/zipkin/convert_test.go | 135 +++++++++++++------------- 1 file changed, 69 insertions(+), 66 deletions(-) diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index cdb365b03d238..314fd0f12949d 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -219,77 +219,80 @@ func TestLineProtocolConverter_Record(t *testing.T) { // Test data from zipkin cli app: //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services - /*{ - name: "cli", - fields: fields{ - acc: &mockAcc, - }, - args: args{ - t: Trace{ - Span{ - ID: "3383422996321511664", - TraceID: "243463817635710260", - Name: "Concat", - ParentID: "4574092882326506380", - Timestamp: time.Unix(0, 1499817952283903000).UTC(), - Duration: time.Duration(2888) * time.Microsecond, - Annotations: []Annotation{ - Annotaitons{ - Timestamp: time.Unix(0, 1499817952283903000).UTC(), - Value: "cs", - Host: "0:0", - ServiceName: "cli", + /* { + name: "cli", + fields: fields{ + acc: &mockAcc, + }, + args: args{ + t: Trace{ + Span{ + ID: "3383422996321511664", + TraceID: "243463817635710260", + Name: "Concat", + ParentID: "4574092882326506380", + Timestamp: time.Unix(0, 1499817952283903000).UTC(), + Duration: time.Duration(2888) * time.Microsecond, + Annotations: []Annotation{ + Annotation{ + Timestamp: time.Unix(0, 1499817952283903000).UTC(), + Value: "cs", + Host: "0:0", + ServiceName: "cli", + }, + }, + BinaryAnnotations: []BinaryAnnotation{ + BinaryAnnotation{ + Key: "http.url", + Value: "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", + Host: "0:0", + ServiceName: "cli", + Type: "STRING", + }, + }, }, + }, }, - BinaryAnnotations: []BinaryAnnotation{ - BinaryAnnotation{ - Key: "http.url", - Value: "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", - Host: "0:0", - ServiceName: "cli", - Type: "STRING", + want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "3383422996321511664", + "parent_id": "4574092882326506380", + "trace_id": "8269862291023777619:243463817635710260", + "name": "Concat", + "service_name": "cli", + "annotation_value": "cs", + "endpoint_host": "0:0", }, + Fields: map[string]interface{}{ + "annotation_timestamp": int64(149981795), + "duration": time.Duration(2888) * time.Microsecond, + }, + Time: time.Unix(0, 1499817952283903000).UTC(), }, - }, - want: []testutil.Metric{ - testutil.Metric{ - Measurement: "zipkin", - Tags: map[string]string{ - "id": "3383422996321511664", - "parent_id": "4574092882326506380", - "trace_id": "8269862291023777619:243463817635710260", - "name": "Concat", - "service_name": "cli", - "annotation_value": "cs", - "endpoint_host": "0:0", - }, - Fields: map[string]interface{}{ - "annotation_timestamp": int64(149981795), - "duration": time.Duration(2888) * time.Microsecond, - }, - Time: time.Unix(0, 1499817952283903000).UTC(), - }, - testutil.Metric{ - Measurement: "zipkin", - Tags: map[string]string{ - "trace_id": "2505404965370368069", - "service_name": "cli", - "annotation_value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", - "key": "http.url", - "type": "STRING", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "name": "Concat", - "endpoint_host": "0:0", - }, - Fields: map[string]interface{}{ - "duration": time.Duration(2888) * time.Microsecond, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "trace_id": "2505404965370368069", + "service_name": "cli", + "annotation_value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", + "key": "http.url", + "type": "STRING", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "name": "Concat", + "endpoint_host": "0:0", + }, + Fields: map[string]interface{}{ + "duration": time.Duration(2888) * time.Microsecond, + }, + Time: time.Unix(0, 1499817952283903000).UTC(), + }, + }, + wantErr: false, }, - Time: time.Unix(0, 1499817952283903000).UTC(), - }, - wantErr: false, - },*/ - + */ //// Test data from distributed trace repo sample json // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json { From 5780e5ce41513dec9759ecf2ade55b5d5bdd286b Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 17:39:48 -0700 Subject: [PATCH 098/154] Add main file for stress writes to telegraf to zipkin plugin --- .../inputs/zipkin/cmd/stress_test_write.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 plugins/inputs/zipkin/cmd/stress_test_write.go diff --git a/plugins/inputs/zipkin/cmd/stress_test_write.go b/plugins/inputs/zipkin/cmd/stress_test_write.go new file mode 100644 index 0000000000000..a6e253e632ee2 --- /dev/null +++ b/plugins/inputs/zipkin/cmd/stress_test_write.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "log" + "os" + "time" + + zipkin "github.com/openzipkin/zipkin-go-opentracing" +) + +func main() { + // 1) Create a opentracing.Tracer that sends data to Zipkin + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Need 1 argument\n") + os.Exit(0) + } + collector, err := zipkin.NewHTTPCollector( + fmt.Sprintf("http://%s:9411/api/v1/spans", os.Args[1])) + if err != nil { + log.Fatalf("error: ", err) + } + tracer, err := zipkin.NewTracer( + zipkin.NewRecorder(collector, false, "127.0.0.1:0", "trivial")) + + if err != nil { + log.Fatalf("Error: %v\n", err) + } + + log.Println("Writing 1,000,000 spans to zipkin impl...") + for i := 0; i < 1000000; i++ { + parent := tracer.StartSpan("Parent") + parent.LogEvent(fmt.Sprintf("Trace %d\n", i)) + parent.Finish() + time.Sleep(10 * time.Microsecond) + } + log.Println("Done!") + +} From a073a920c7f747550bb7e11db94255feddcf33c8 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 17:40:10 -0700 Subject: [PATCH 099/154] Add sample config for working with new tsi engine --- .../inputs/zipkin/influx_sample_config.txt | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 plugins/inputs/zipkin/influx_sample_config.txt diff --git a/plugins/inputs/zipkin/influx_sample_config.txt b/plugins/inputs/zipkin/influx_sample_config.txt new file mode 100644 index 0000000000000..76105ea0df548 --- /dev/null +++ b/plugins/inputs/zipkin/influx_sample_config.txt @@ -0,0 +1,123 @@ +reporting-disabled = false +bind-address = "127.0.0.1:8088" + +[meta] + dir = "/Users/adamperlin/.influxdb/meta" + retention-autocreate = true + logging-enabled = true + +[data] + dir = "/Users/adamperlin/.influxdb/data" + index-version = "tsi1" + wal-dir = "/Users/adamperlin/.influxdb/wal" + query-log-enabled = true + cache-max-memory-size = 1073741824 + cache-snapshot-memory-size = 26214400 + cache-snapshot-write-cold-duration = "10m0s" + compact-full-write-cold-duration = "4h0m0s" + max-series-per-database = 1000000 + max-values-per-tag = 100000 + trace-logging-enabled = false + +[coordinator] + write-timeout = "10s" + max-concurrent-queries = 0 + query-timeout = "0s" + log-queries-after = "0s" + max-select-point = 0 + max-select-series = 0 + max-select-buckets = 0 + +[retention] + enabled = true + check-interval = "30m0s" + +[shard-precreation] + enabled = true + check-interval = "10m0s" + advance-period = "30m0s" + +[monitor] + store-enabled = true + store-database = "_internal" + store-interval = "10s" + +[subscriber] + enabled = true + http-timeout = "30s" + insecure-skip-verify = false + ca-certs = "" + write-concurrency = 40 + write-buffer-size = 1000 + +[http] + enabled = true + bind-address = ":8086" + auth-enabled = false + log-enabled = true + write-tracing = false + pprof-enabled = true + https-enabled = false + https-certificate = "/etc/ssl/influxdb.pem" + https-private-key = "" + max-row-limit = 0 + max-connection-limit = 0 + shared-secret = "" + realm = "InfluxDB" + unix-socket-enabled = false + bind-socket = "/var/run/influxdb.sock" + +[[graphite]] + enabled = false + bind-address = ":2003" + database = "graphite" + retention-policy = "" + protocol = "tcp" + batch-size = 5000 + batch-pending = 10 + batch-timeout = "1s" + consistency-level = "one" + separator = "." + udp-read-buffer = 0 + +[[collectd]] + enabled = false + bind-address = ":25826" + database = "collectd" + retention-policy = "" + batch-size = 5000 + batch-pending = 10 + batch-timeout = "10s" + read-buffer = 0 + typesdb = "/usr/share/collectd/types.db" + security-level = "none" + auth-file = "/etc/collectd/auth_file" + +[[opentsdb]] + enabled = false + bind-address = ":4242" + database = "opentsdb" + retention-policy = "" + consistency-level = "one" + tls-enabled = false + certificate = "/etc/ssl/influxdb.pem" + batch-size = 1000 + batch-pending = 5 + batch-timeout = "1s" + log-point-errors = true + +[[udp]] + enabled = false + bind-address = ":8089" + database = "udp" + retention-policy = "" + batch-size = 5000 + batch-pending = 10 + read-buffer = 0 + batch-timeout = "1s" + precision = "" + +[continuous_queries] + log-enabled = true + enabled = true + run-interval = "1s" From b2129101c030bdd7019bfe58b9561520ba91fe92 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 17:40:39 -0700 Subject: [PATCH 100/154] Add draft of setup instructions for working with zipkin data in InfluxDB --- .../zipkin/influx_setup_instructions.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 plugins/inputs/zipkin/influx_setup_instructions.md diff --git a/plugins/inputs/zipkin/influx_setup_instructions.md b/plugins/inputs/zipkin/influx_setup_instructions.md new file mode 100644 index 0000000000000..073f8d3bf9c53 --- /dev/null +++ b/plugins/inputs/zipkin/influx_setup_instructions.md @@ -0,0 +1,32 @@ +# How To Set Up InfluxDB For Work With Zipkin + +## Steps +___Update___ InfluxDB to >= 1.3, in order to use the new tsi engine. + +___Generate___ a config file with the following command: + `influxd config > /path/for/config/file` + +___Add___ the following to your config file, under the `[data]` tab: + +```toml +[data] + dir = "/Users/goller/.influxdb/data" + index-version = "tsi1" + wal-dir = "/Users/goller/.influxdb/wal" + query-log-enabled = true + cache-max-memory-size = 1073741824 + cache-snapshot-memory-size = 26214400 + cache-snapshot-write-cold-duration = "10m0s" + compact-full-write-cold-duration = "4h0m0s" + max-series-per-database = 1000000 + max-values-per-tag = 100000 + trace-logging-enabled = false + ``` + + ___Start___ `influxd` with your new config file: + `$ influxd -config=/path/to/your/config/file` + +___Update___ your retention policy: +```sql +ALTER RETENTION POLICY "retention_policy" ON "telegraf" DURATION 3w SHARD DURATION 30m DEFAULT +``` From 57f1e2fb6fdc0457c26b5a91738aecc0fdd380be Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 17:55:16 -0700 Subject: [PATCH 101/154] Update log message in stress writing script in zipkin plugin --- plugins/inputs/zipkin/cmd/stress_test_write.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/cmd/stress_test_write.go b/plugins/inputs/zipkin/cmd/stress_test_write.go index a6e253e632ee2..0facefbb97038 100644 --- a/plugins/inputs/zipkin/cmd/stress_test_write.go +++ b/plugins/inputs/zipkin/cmd/stress_test_write.go @@ -27,8 +27,8 @@ func main() { log.Fatalf("Error: %v\n", err) } - log.Println("Writing 1,000,000 spans to zipkin impl...") - for i := 0; i < 1000000; i++ { + log.Println("Writing 100 spans to zipkin impl...") + for i := 0; i < 100; i++ { parent := tracer.StartSpan("Parent") parent.LogEvent(fmt.Sprintf("Trace %d\n", i)) parent.Finish() From 210569877454eb3614b6061cbd3285d77ff8ff96 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 24 Jul 2017 17:55:43 -0700 Subject: [PATCH 102/154] Add currently broken write test benchmark --- plugins/inputs/zipkin/write_test.go | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 plugins/inputs/zipkin/write_test.go diff --git a/plugins/inputs/zipkin/write_test.go b/plugins/inputs/zipkin/write_test.go new file mode 100644 index 0000000000000..0a27b93b8ea86 --- /dev/null +++ b/plugins/inputs/zipkin/write_test.go @@ -0,0 +1,34 @@ +package zipkin + +import ( + "fmt" + "log" + "testing" + "time" + + zipkin "github.com/openzipkin/zipkin-go-opentracing" +) + +func BenchmarkWrites(b *testing.B) { + collector, err := zipkin.NewHTTPCollector( + fmt.Sprintf("http://%s:9411/api/v1/spans", "localhost")) + if err != nil { + log.Fatalf("error: ", err) + } + tracer, err := zipkin.NewTracer( + zipkin.NewRecorder(collector, false, "127.0.0.1:0", "trivial")) + + if err != nil { + log.Fatalf("Error: %v\n", err) + } + + log.Println("Writing 1,000,000 spans to zipkin impl...") + for i := 0; i < 20; i++ { + log.Println("going") + parent := tracer.StartSpan("Parent") + parent.LogEvent(fmt.Sprintf("Trace %d\n", i)) + parent.Finish() + time.Sleep(10 * time.Microsecond) + } + log.Println("Done!") +} From c5526f55c66e1300d6c7ac672dfda56df25271a6 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 25 Jul 2017 17:42:01 -0700 Subject: [PATCH 103/154] Update telegraf stress writing script in zipkin plugin --- .../inputs/zipkin/cmd/stress_test_write.go | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/zipkin/cmd/stress_test_write.go b/plugins/inputs/zipkin/cmd/stress_test_write.go index 0facefbb97038..e24d529e968d6 100644 --- a/plugins/inputs/zipkin/cmd/stress_test_write.go +++ b/plugins/inputs/zipkin/cmd/stress_test_write.go @@ -4,7 +4,6 @@ import ( "fmt" "log" "os" - "time" zipkin "github.com/openzipkin/zipkin-go-opentracing" ) @@ -16,10 +15,13 @@ func main() { os.Exit(0) } collector, err := zipkin.NewHTTPCollector( - fmt.Sprintf("http://%s:9411/api/v1/spans", os.Args[1])) + fmt.Sprintf("http://%s:9411/api/v1/spans", os.Args[1]), zipkin.HTTPBatchSize(1), zipkin.HTTPMaxBacklog(100000)) + defer collector.Close() if err != nil { log.Fatalf("error: ", err) } + + //zipkin.HTTPMaxBacklog(1000000)(collector.(*zipkin.HTTPCollector)) tracer, err := zipkin.NewTracer( zipkin.NewRecorder(collector, false, "127.0.0.1:0", "trivial")) @@ -27,13 +29,24 @@ func main() { log.Fatalf("Error: %v\n", err) } - log.Println("Writing 100 spans to zipkin impl...") - for i := 0; i < 100; i++ { + log.Println("Writing 1000000 spans to zipkin impl...") + //var wg sync.WaitGroup + for i := 0; i < 100000; i++ { + + log.Printf("Writing span %d\n", i) + /*go func(i int) { + wg.Add(1) + defer wg.Done() + log.Println("Writing span %d\n", i) + parent := tracer.StartSpan("Parent") + parent.LogEvent(fmt.Sprintf("Trace%d", i)) + parent.Finish() + time.Sleep(2 * time.Second) + }(i)*/ parent := tracer.StartSpan("Parent") - parent.LogEvent(fmt.Sprintf("Trace %d\n", i)) + parent.LogEvent(fmt.Sprintf("Trace%d", i)) parent.Finish() - time.Sleep(10 * time.Microsecond) + // time.Sleep(2 * time.Second) } log.Println("Done!") - } From 665e75f94ace939cbd4ea8cdc8838ad5bc7b7bbb Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 25 Jul 2017 17:42:43 -0700 Subject: [PATCH 104/154] Add debug log to zipkin http lander (temporary) --- plugins/inputs/zipkin/handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/inputs/zipkin/handler.go b/plugins/inputs/zipkin/handler.go index e12313c3c5d54..260b0ea8549e7 100644 --- a/plugins/inputs/zipkin/handler.go +++ b/plugins/inputs/zipkin/handler.go @@ -1,6 +1,7 @@ package zipkin import ( + "fmt" "io/ioutil" "net/http" "sync" @@ -35,6 +36,7 @@ func (s *SpanHandler) Register(router *mux.Router, recorder Recorder) error { // Spans handles zipkin thrift spans func (s *SpanHandler) Spans(w http.ResponseWriter, r *http.Request) { + fmt.Println("Got request from host: ", r.Host) defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) if err != nil { From b1c14902d6ccab2995fddb6a193ac2ebb48f3702 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 25 Jul 2017 17:43:33 -0700 Subject: [PATCH 105/154] Make starting message more descriptive in zipkin.go --- plugins/inputs/zipkin/zipkin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index a94a9308865a7..b4efb5162a958 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -112,7 +112,7 @@ func (z *Zipkin) Gather(acc telegraf.Accumulator) error { return nil } // Start launches a separate goroutine for collecting zipkin client http requests, // passing in a telegraf.Accumulator such that data can be collected. func (z *Zipkin) Start(acc telegraf.Accumulator) error { - log.Println("starting...") + log.Println("starting zipkin plugin...") if z.handler == nil { z.handler = NewSpanHandler(z.Path) } From d31aef571b0b8397502a70c1aaf6e6fe61066cd1 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 26 Jul 2017 15:39:07 -0700 Subject: [PATCH 106/154] Add command line flags to cmd/stress_test_write.go in zipkin plugin --- .../inputs/zipkin/cmd/stress_test_write.go | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/plugins/inputs/zipkin/cmd/stress_test_write.go b/plugins/inputs/zipkin/cmd/stress_test_write.go index e24d529e968d6..fdd14c651f458 100644 --- a/plugins/inputs/zipkin/cmd/stress_test_write.go +++ b/plugins/inputs/zipkin/cmd/stress_test_write.go @@ -1,27 +1,45 @@ package main import ( + "flag" "fmt" "log" - "os" + "time" zipkin "github.com/openzipkin/zipkin-go-opentracing" ) +var ( + BatchSize int + MaxBackLog int + BatchTimeInterval int + SpanCount int + ZipkinServerHost string +) + +const usage = `./stress_test_write -batch_size= -max_backlog= -batch_interval= -span_count -zipkin_host=` + +func init() { + flag.IntVar(&BatchSize, "batch_size", 10000, usage) + flag.IntVar(&MaxBackLog, "max_backlog", 100000, usage) + flag.IntVar(&BatchTimeInterval, "batch_interval", 1, usage) + flag.IntVar(&SpanCount, "span_count", 100000, usage) + flag.StringVar(&ZipkinServerHost, "zipkin_host", "localhost", usage) +} + func main() { - // 1) Create a opentracing.Tracer that sends data to Zipkin - if len(os.Args) != 2 { - fmt.Fprintf(os.Stderr, "Need 1 argument\n") - os.Exit(0) - } + flag.Parse() + var hostname = fmt.Sprintf("http://%s:9411/api/v1/spans", ZipkinServerHost) collector, err := zipkin.NewHTTPCollector( - fmt.Sprintf("http://%s:9411/api/v1/spans", os.Args[1]), zipkin.HTTPBatchSize(1), zipkin.HTTPMaxBacklog(100000)) + hostname, + zipkin.HTTPBatchSize(BatchSize), + zipkin.HTTPMaxBacklog(MaxBackLog), + zipkin.HTTPBatchInterval(time.Duration(BatchTimeInterval)*time.Second)) defer collector.Close() if err != nil { - log.Fatalf("error: ", err) + log.Fatalf("Error intializing zipkin http collector: %v\n", err) } - //zipkin.HTTPMaxBacklog(1000000)(collector.(*zipkin.HTTPCollector)) tracer, err := zipkin.NewTracer( zipkin.NewRecorder(collector, false, "127.0.0.1:0", "trivial")) @@ -29,24 +47,11 @@ func main() { log.Fatalf("Error: %v\n", err) } - log.Println("Writing 1000000 spans to zipkin impl...") - //var wg sync.WaitGroup - for i := 0; i < 100000; i++ { - - log.Printf("Writing span %d\n", i) - /*go func(i int) { - wg.Add(1) - defer wg.Done() - log.Println("Writing span %d\n", i) - parent := tracer.StartSpan("Parent") - parent.LogEvent(fmt.Sprintf("Trace%d", i)) - parent.Finish() - time.Sleep(2 * time.Second) - }(i)*/ + log.Printf("Writing %d spans to zipkin server at %s\n", SpanCount, hostname) + for i := 0; i < SpanCount; i++ { parent := tracer.StartSpan("Parent") parent.LogEvent(fmt.Sprintf("Trace%d", i)) parent.Finish() - // time.Sleep(2 * time.Second) } - log.Println("Done!") + log.Println("Done. Flushing remaining spans...") } From c6bbb7af97de215816918845d0530d887c95ebfb Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 26 Jul 2017 16:47:37 -0600 Subject: [PATCH 107/154] Move thrift_serialize to directory of Zipkin plugin --- .../inputs/zipkin/cmd/{ => thrift_serialize}/thrift_serialize.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/cmd/{ => thrift_serialize}/thrift_serialize.go (100%) diff --git a/plugins/inputs/zipkin/cmd/thrift_serialize.go b/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go similarity index 100% rename from plugins/inputs/zipkin/cmd/thrift_serialize.go rename to plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go From a65cf856943270693b69c35b4e28f2f7efab25d8 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 26 Jul 2017 16:49:20 -0600 Subject: [PATCH 108/154] Move stress_test_write to directory of Zipkin plugin --- .../zipkin/cmd/{ => stress_test_write}/stress_test_write.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/inputs/zipkin/cmd/{ => stress_test_write}/stress_test_write.go (100%) diff --git a/plugins/inputs/zipkin/cmd/stress_test_write.go b/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go similarity index 100% rename from plugins/inputs/zipkin/cmd/stress_test_write.go rename to plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go From 8c40aa97fe5489b92d79d638950caa980954855d Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 26 Jul 2017 16:50:31 -0600 Subject: [PATCH 109/154] Update Godeps for Zipkin stress test --- Godeps | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Godeps b/Godeps index f6de2fe99f5ab..a2d1ffa30f1ca 100644 --- a/Godeps +++ b/Godeps @@ -17,8 +17,10 @@ github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 github.com/eapache/go-xerial-snappy bb955e01b9346ac19dc29eb16586c90ded99a98c github.com/eapache/queue 44cc805cf13205b55f69e14bcb69867d1ae92f98 github.com/eclipse/paho.mqtt.golang d4f545eb108a2d19f9b1a735689dbfb719bc21fb +github.com/go-logfmt/logfmt 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 github.com/go-sql-driver/mysql 2e00b5cd70399450106cec6431c2e2ce3cae5034 github.com/gobwas/glob bea32b9cd2d6f55753d94a28e959b13f0244797a +github.com/gogo/protobuf 7b6c6391c4ff245962047fc1e2c6e08b1cdfa0e8 github.com/golang/protobuf 8ee79997227bf9b34611aee7946ae64735e6fd93 github.com/golang/snappy 7db9049039a047d955fe8c19b83c8ff5abd765c7 github.com/gorilla/mux 392c28fe23e1c45ddba891b0320b3b5df220beea @@ -38,6 +40,8 @@ github.com/nats-io/go-nats ea9585611a4ab58a205b9b125ebd74c389a6b898 github.com/nats-io/nats ea9585611a4ab58a205b9b125ebd74c389a6b898 github.com/nats-io/nuid 289cccf02c178dc782430d534e3c1f5b72af807f github.com/nsqio/go-nsq a53d495e81424aaf7a7665a9d32a97715c40e953 +github.com/opentracing-contrib/go-observer a52f2342449246d5bcc273e65cbdcfa5f7d6c63c +github.com/opentracing/opentracing-go 06f47b42c792fef2796e9681353e1d908c417827 github.com/openzipkin/zipkin-go-opentracing 1cafbdfde94fbf2b373534764e0863aa3bd0bf7b github.com/pierrec/lz4 5c9560bfa9ace2bf86080bf40d46b34ae44604df github.com/pierrec/xxHash 5a004441f897722c627870a981d02b29924215fa From f6e6ee26e56a025f8d35add5115f8e754bd0b841 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 26 Jul 2017 17:04:39 -0600 Subject: [PATCH 110/154] Fix vet errors for Zipkin plugin --- plugins/inputs/zipkin/write_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/write_test.go b/plugins/inputs/zipkin/write_test.go index 0a27b93b8ea86..37437dd59433b 100644 --- a/plugins/inputs/zipkin/write_test.go +++ b/plugins/inputs/zipkin/write_test.go @@ -13,7 +13,7 @@ func BenchmarkWrites(b *testing.B) { collector, err := zipkin.NewHTTPCollector( fmt.Sprintf("http://%s:9411/api/v1/spans", "localhost")) if err != nil { - log.Fatalf("error: ", err) + log.Fatalf("error: %v", err) } tracer, err := zipkin.NewTracer( zipkin.NewRecorder(collector, false, "127.0.0.1:0", "trivial")) From 3f29ecc59a3aaed3282092c9c50e4d5cc7ed6137 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 27 Jul 2017 15:22:38 -0700 Subject: [PATCH 111/154] Add GZIP middleware to zipkin handler --- plugins/inputs/zipkin/handler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/handler.go b/plugins/inputs/zipkin/handler.go index 260b0ea8549e7..e4ff9a0f58c28 100644 --- a/plugins/inputs/zipkin/handler.go +++ b/plugins/inputs/zipkin/handler.go @@ -6,6 +6,7 @@ import ( "net/http" "sync" + "github.com/NYTimes/gziphandler" "github.com/apache/thrift/lib/go/thrift" "github.com/gorilla/mux" "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" @@ -29,7 +30,9 @@ func NewSpanHandler(path string) *SpanHandler { // Register implements the Service interface. Register accepts zipkin thrift data // POSTed to the path of the mux router func (s *SpanHandler) Register(router *mux.Router, recorder Recorder) error { - router.HandleFunc(s.Path, s.Spans).Methods("POST") + handler := gziphandler.GzipHandler(http.HandlerFunc(s.Spans)) + //TODO: add more middleware + router.Handle(s.Path, handler).Methods("POST") s.recorder = recorder return nil } From f571b11ec71df2d3629a3584110452705d1a8b94 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 27 Jul 2017 15:27:27 -0700 Subject: [PATCH 112/154] Add function for cors middleware to zipkin plugin --- plugins/inputs/zipkin/handler.go | 33 +++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/handler.go b/plugins/inputs/zipkin/handler.go index e4ff9a0f58c28..18532ee2bd47e 100644 --- a/plugins/inputs/zipkin/handler.go +++ b/plugins/inputs/zipkin/handler.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "sync" "github.com/NYTimes/gziphandler" @@ -27,11 +28,41 @@ func NewSpanHandler(path string) *SpanHandler { } } +func cors(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if origin := r.Header.Get("Origin"); origin != "" { + w.Header().Set(`Access-Control-Allow-Origin`, origin) + w.Header().Set(`Access-Control-Allow-Methods`, strings.Join([]string{ + `GET`, + `OPTIONS`, + }, ", ")) + + w.Header().Set(`Access-Control-Allow-Headers`, strings.Join([]string{ + `Accept`, + `Accept-Encoding`, + `Content-Length`, + `Content-Type`, + }, ", ")) + + w.Header().Set(`Access-Control-Expose-Headers`, strings.Join([]string{ + `Date`, + }, ", ")) + } + + if r.Method == "OPTIONS" { + return + } + + next.ServeHTTP(w, r) + } +} + // Register implements the Service interface. Register accepts zipkin thrift data // POSTed to the path of the mux router func (s *SpanHandler) Register(router *mux.Router, recorder Recorder) error { handler := gziphandler.GzipHandler(http.HandlerFunc(s.Spans)) - //TODO: add more middleware + //TODO: add more cors middleware + router.Handle(s.Path, handler).Methods("POST") s.recorder = recorder return nil From f21fd421a3ed94b03156335b7ffc901ce7856e31 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 27 Jul 2017 15:27:55 -0700 Subject: [PATCH 113/154] Change RP command in influx setup instructions in zipkin plugin --- plugins/inputs/zipkin/influx_setup_instructions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/influx_setup_instructions.md b/plugins/inputs/zipkin/influx_setup_instructions.md index 073f8d3bf9c53..07658e123ee7e 100644 --- a/plugins/inputs/zipkin/influx_setup_instructions.md +++ b/plugins/inputs/zipkin/influx_setup_instructions.md @@ -28,5 +28,5 @@ ___Add___ the following to your config file, under the `[data]` tab: ___Update___ your retention policy: ```sql -ALTER RETENTION POLICY "retention_policy" ON "telegraf" DURATION 3w SHARD DURATION 30m DEFAULT +ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m ``` From 64fa99296988bdc9e2d2d979ddb5014ec238ce1a Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 27 Jul 2017 15:28:58 -0700 Subject: [PATCH 114/154] Remove write_test.go in zipkin plugin --- plugins/inputs/zipkin/write_test.go | 34 ----------------------------- 1 file changed, 34 deletions(-) delete mode 100644 plugins/inputs/zipkin/write_test.go diff --git a/plugins/inputs/zipkin/write_test.go b/plugins/inputs/zipkin/write_test.go deleted file mode 100644 index 37437dd59433b..0000000000000 --- a/plugins/inputs/zipkin/write_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package zipkin - -import ( - "fmt" - "log" - "testing" - "time" - - zipkin "github.com/openzipkin/zipkin-go-opentracing" -) - -func BenchmarkWrites(b *testing.B) { - collector, err := zipkin.NewHTTPCollector( - fmt.Sprintf("http://%s:9411/api/v1/spans", "localhost")) - if err != nil { - log.Fatalf("error: %v", err) - } - tracer, err := zipkin.NewTracer( - zipkin.NewRecorder(collector, false, "127.0.0.1:0", "trivial")) - - if err != nil { - log.Fatalf("Error: %v\n", err) - } - - log.Println("Writing 1,000,000 spans to zipkin impl...") - for i := 0; i < 20; i++ { - log.Println("going") - parent := tracer.StartSpan("Parent") - parent.LogEvent(fmt.Sprintf("Trace %d\n", i)) - parent.Finish() - time.Sleep(10 * time.Microsecond) - } - log.Println("Done!") -} From e25d49852c924524ef2fb4630a7227576dfac9cf Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 27 Jul 2017 15:29:49 -0700 Subject: [PATCH 115/154] Update thrift_serialize.go in zipkin plugin --- plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go b/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go index 9f77da1a89f27..15ac2ecbc2986 100644 --- a/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go +++ b/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go @@ -2,7 +2,6 @@ package main import ( "encoding/json" - "flag" "fmt" "io/ioutil" "log" @@ -24,8 +23,6 @@ var ( const usage = `./json_serialize -input -output output`*/ func main() { - - flag.Parse() b, err := ioutil.ReadFile("../testdata/json/distributed_trace_sample.json") if err != nil { log.Fatalf("error: %v\n", err) From 5e9f1abd694720f8a190a6f74dd345d70fc5262f Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 27 Jul 2017 15:30:25 -0700 Subject: [PATCH 116/154] Add informal notes about zipkin plugin performance --- plugins/inputs/zipkin/notes | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 plugins/inputs/zipkin/notes diff --git a/plugins/inputs/zipkin/notes b/plugins/inputs/zipkin/notes new file mode 100644 index 0000000000000..d8fd1af4efac1 --- /dev/null +++ b/plugins/inputs/zipkin/notes @@ -0,0 +1,39 @@ +recommendations: +1. batch size should be large (e.g. 1000) + + +profiling: +telegraf to file 200000 points (100k spans) in 3 seconds + +telegraf to influx not able to finish so far (buffering?) + +Empirical results of default indexing vs. tsi + +Write: +Default: finished in 11.278 seconds (2.7 user) +TSI: Didn't finish +TSI(master): 1:37 (9.3 user) + +select * from zipkin where "id"='trace_id' +Default: instantaneous +TSI: No data +TSI(master): + +select count("duration") from zipkin +Default: ~1s +TSI: 30s + +show tag values with key="service_name": +Default: +nearly instantaneous +TSI: Significant delay + +show tag values with key="name" where "service_name"='trivial': +Default: +Nearly instantaneous +TSI: +Significant Delay + +select count("duration") from zipkin where time > now() - 30m and time < now() group by "id","parent_id", time(1d): +Default: ~10 seconds +TSI: Didn't finish From 20e72901d401955b4bc89726e68464a07d905fd8 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 10:45:12 -0600 Subject: [PATCH 117/154] Add README --- plugins/inputs/zipkin/README.md | 136 ++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 plugins/inputs/zipkin/README.md diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md new file mode 100644 index 0000000000000..204a845230e6a --- /dev/null +++ b/plugins/inputs/zipkin/README.md @@ -0,0 +1,136 @@ +# Zipkin Plugin + +This plugin implements the Zipkin http server to gather trace and timing data needed to troubleshoot latency problems in microservice architectures. + + + +### Configuration: +```toml +[[inputs.zipkin]] + path = "/api/v1/spans" #Path on which Telegraf listens for spans + port = 9411 # Port on which Telegraf listens +``` + +### Tracing: + +*This plugin uses Annotations tags and fields to track data from spans + +- TRACE : is a set of spans that share a single root span. +Traces are built by collecting all Spans that share a traceId. + +- SPAN : is a set of Annotations and BinaryAnnotations that correspond to a particular RPC. + +- Annotations : for each annotation & binary annotation of a span a metric is output + + +#### Annotations: records an occurrence in time at the beginning and end of a request + - CS (client start) : beginning of span, request is made. + - SR (server receive): server receives request and will start processing it + network latency & clock jitters differ it from cs + - SS (server send) : server is done processing and sends request back to client + amount of time it took to process request will differ it from sr + - CR (client receive): end of span, client receives response from server + RPC is considered complete with this annotation + +- TAGS: + _"id":_ The 64 or 128-bit ID of the trace. Every span in a trace shares this ID. + _"parent_id":_ An ID associated with a particular child span. If there is no child span, the parent ID is set to itself. + _"trace_id":_ The 64 or 128-bit ID of a particular trace. Trace ID High concat Trace ID Low. + _"name":_ Defines a span + _"service_name":_ Defines a service + _"annotation_value":_ Defines each individual annotation + _"endpoint_host":_ listening port concat with IPV4 + +-FIELDS + "annotation_timestamp": Start time of an annotation. If time is nil we set it to the current UTC time. + "duration": The time in microseconds between the end and beginning of a span. + +### BINARY ANNOTATIONS: + +-TAGS: Contains the same tags as annotations plus these additions + + "key": Acts as a pointer to some address which houses the value + _"type"_: Given data type + +-FIELDS: + + "duration": The time in microseconds between the end and beginning of a span. + + + +### Sample Queries: + +- Get All Span Names for Service `my_web_server` +```sql +SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_server'``` + - __Description:__ returns a list containing the names of the spans which have annotations with the given `service_name` of `my_web_server`. + +- Get All Service Names + ```sql + SHOW TAG VALUES FROM "zipkin" WITH KEY = "service_name"``` + - __Description:__ returns a list of all `distinct` endpoint service names. + +- Find spans with longest duration + ```sql + SELECT max("duration") FROM "zipkin" WHERE "service_name" = 'my_service' AND "name" = 'my_span_name' AND time > now() - 20m GROUP BY "trace_id",time(30s) LIMIT 5 + ``` + - __Description:__ In the last 20 minutes find the top 5 longest span durations for service `my_server` and span name `my_span_name` + + + +### Example Input Trace: + +- [Cli microservice with two services Test](https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services) +- [Test data from distributed trace repo sample json](https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json) + +#### Trace Example +``` { + "traceId": "bd7a977555f6b982", + "name": "query", + "id": "be2d01e33cc78d97", + "parentId": "ebf33e1a81dc6f71", + "timestamp": 1458702548786000, + "duration": 13000, + "annotations": [ + { + "endpoint": { + "serviceName": "zipkin-query", + "ipv4": "192.168.1.2", + "port": 9411 + }, + "timestamp": 1458702548786000, + "value": "cs" + }, + { + "endpoint": { + "serviceName": "zipkin-query", + "ipv4": "192.168.1.2", + "port": 9411 + }, + "timestamp": 1458702548799000, + "value": "cr" + } + ], + "binaryAnnotations": [ + { + "key": "jdbc.query", + "value": "select distinct `zipkin_spans`.`trace_id` from `zipkin_spans` join `zipkin_annotations` on (`zipkin_spans`.`trace_id` = `zipkin_annotations`.`trace_id` and `zipkin_spans`.`id` = `zipkin_annotations`.`span_id`) where (`zipkin_annotations`.`endpoint_service_name` = ? and `zipkin_spans`.`start_ts` between ? and ?) order by `zipkin_spans`.`start_ts` desc limit ?", + "endpoint": { + "serviceName": "zipkin-query", + "ipv4": "192.168.1.2", + "port": 9411 + } + }, + { + "key": "sa", + "value": true, + "endpoint": { + "serviceName": "spanstore-jdbc", + "ipv4": "127.0.0.1", + "port": 3306 + } + } + ] + },``` + +### Recommended installation From 53e7a070c7da806064f0034aca77355cd5071302 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 15:55:40 -0600 Subject: [PATCH 118/154] Add gzip and cors middleware to zipkin plugin --- plugins/inputs/zipkin/handler.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/plugins/inputs/zipkin/handler.go b/plugins/inputs/zipkin/handler.go index 18532ee2bd47e..d9bd07828159a 100644 --- a/plugins/inputs/zipkin/handler.go +++ b/plugins/inputs/zipkin/handler.go @@ -1,13 +1,12 @@ package zipkin import ( - "fmt" + "compress/gzip" "io/ioutil" "net/http" "strings" "sync" - "github.com/NYTimes/gziphandler" "github.com/apache/thrift/lib/go/thrift" "github.com/gorilla/mux" "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" @@ -33,8 +32,8 @@ func cors(next http.HandlerFunc) http.HandlerFunc { if origin := r.Header.Get("Origin"); origin != "" { w.Header().Set(`Access-Control-Allow-Origin`, origin) w.Header().Set(`Access-Control-Allow-Methods`, strings.Join([]string{ - `GET`, `OPTIONS`, + `POST`, }, ", ")) w.Header().Set(`Access-Control-Allow-Headers`, strings.Join([]string{ @@ -60,26 +59,36 @@ func cors(next http.HandlerFunc) http.HandlerFunc { // Register implements the Service interface. Register accepts zipkin thrift data // POSTed to the path of the mux router func (s *SpanHandler) Register(router *mux.Router, recorder Recorder) error { - handler := gziphandler.GzipHandler(http.HandlerFunc(s.Spans)) - //TODO: add more cors middleware - - router.Handle(s.Path, handler).Methods("POST") + handler := cors(http.HandlerFunc(s.Spans)) + router.Handle(s.Path, handler).Methods("POST", "OPTIONS") s.recorder = recorder return nil } // Spans handles zipkin thrift spans func (s *SpanHandler) Spans(w http.ResponseWriter, r *http.Request) { - fmt.Println("Got request from host: ", r.Host) defer r.Body.Close() - body, err := ioutil.ReadAll(r.Body) + body := r.Body + var err error + // Handle gzip decoding of the body + if r.Header.Get("Content-Encoding") == "gzip" { + body, err = gzip.NewReader(r.Body) + if err != nil { + s.recorder.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + defer body.Close() + } + + octets, err := ioutil.ReadAll(body) if err != nil { s.recorder.Error(err) w.WriteHeader(http.StatusInternalServerError) return } - spans, err := unmarshalThrift(body) + spans, err := unmarshalThrift(octets) if err != nil { s.recorder.Error(err) w.WriteHeader(http.StatusInternalServerError) From 753fb6b70e8fa96d6ad38c340d8dc757e16d8640 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 16:03:03 -0600 Subject: [PATCH 119/154] Remove annotation_timestamp from Zipkin plugin --- plugins/inputs/zipkin/convert.go | 2 -- plugins/inputs/zipkin/convert_test.go | 13 ++++--------- plugins/inputs/zipkin/zipkin_test.go | 15 +++++---------- 3 files changed, 9 insertions(+), 21 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 94bcf44159b99..087e632c51a2b 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -35,8 +35,6 @@ func (l *LineProtocolConverter) Record(t Trace) error { for _, s := range t { for _, a := range s.Annotations { fields := map[string]interface{}{ - // TODO: Maybe we don't need "annotation_timestamp"? - "annotation_timestamp": a.Timestamp.Unix(), "duration": s.Duration, } diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index 314fd0f12949d..ae9946c51a744 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -156,8 +156,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "name": "Parent", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1498688360), - "duration": time.Duration(103680) * time.Microsecond, + "duration": time.Duration(103680) * time.Microsecond, }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -173,8 +172,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "name": "Parent", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1498688360), - "duration": time.Duration(103680) * time.Microsecond, + "duration": time.Duration(103680) * time.Microsecond, }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -190,8 +188,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "id": "22964302721410078", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1498688360), - "duration": time.Duration(103680) * time.Microsecond, + "duration": time.Duration(103680) * time.Microsecond, }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -266,7 +263,6 @@ func TestLineProtocolConverter_Record(t *testing.T) { "endpoint_host": "0:0", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(149981795), "duration": time.Duration(2888) * time.Microsecond, }, Time: time.Unix(0, 1499817952283903000).UTC(), @@ -334,8 +330,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1433330263), - "duration": time.Duration(1) * time.Nanosecond, + "duration": time.Duration(1) * time.Nanosecond, }, Time: time.Unix(1, 0).UTC(), }, diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 9afe69793910f..d59c93f2866de 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -83,8 +83,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Parent", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1498688360), - "duration": time.Duration(103680) * time.Microsecond, + "duration": time.Duration(103680) * time.Microsecond, }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -100,8 +99,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Parent", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1498688360), - "duration": time.Duration(103680) * time.Microsecond, + "duration": time.Duration(103680) * time.Microsecond, }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -117,8 +115,7 @@ func TestZipkinPlugin(t *testing.T) { "id": "22964302721410078", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1498688360), - "duration": time.Duration(103680) * time.Microsecond, + "duration": time.Duration(103680) * time.Microsecond, }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -162,8 +159,7 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1433330263), - "duration": time.Duration(1) * time.Microsecond, + "duration": time.Duration(1) * time.Microsecond, }, //Time: time.Unix(1, 0).UTC(), Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), @@ -180,8 +176,7 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ - "annotation_timestamp": int64(1433330263), - "duration": time.Duration(1) * time.Microsecond, + "duration": time.Duration(1) * time.Microsecond, }, Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), }, From f5c3a4aceb39a710d4d3b9afd7eaa56b9e058634 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 16:15:21 -0600 Subject: [PATCH 120/154] Change duration to duration_ns --- plugins/inputs/zipkin/convert.go | 4 ++-- plugins/inputs/zipkin/convert_test.go | 18 +++++++++--------- plugins/inputs/zipkin/zipkin_test.go | 16 ++++++++-------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 087e632c51a2b..09e044f55c335 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -35,7 +35,7 @@ func (l *LineProtocolConverter) Record(t Trace) error { for _, s := range t { for _, a := range s.Annotations { fields := map[string]interface{}{ - "duration": s.Duration, + "duration_ns": s.Duration.Nanoseconds(), } tags := map[string]string{ @@ -52,7 +52,7 @@ func (l *LineProtocolConverter) Record(t Trace) error { for _, b := range s.BinaryAnnotations { fields := map[string]interface{}{ - "duration": s.Duration, + "duration_ns": s.Duration.Nanoseconds(), } tags := map[string]string{ diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index ae9946c51a744..bbbb43f4fc716 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -122,7 +122,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "type": "STRING", }, Fields: map[string]interface{}{ - "duration": time.Duration(53106) * time.Microsecond, + "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851331000).UTC(), }, @@ -140,7 +140,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "type": "STRING", }, Fields: map[string]interface{}{ - "duration": time.Duration(50410) * time.Microsecond, + "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360904552000).UTC(), }, @@ -156,7 +156,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "name": "Parent", }, Fields: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -172,7 +172,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "name": "Parent", }, Fields: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -188,7 +188,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "id": "22964302721410078", }, Fields: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -206,7 +206,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "endpoint_host": "2130706433:0", }, Fields: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -263,7 +263,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "endpoint_host": "0:0", }, Fields: map[string]interface{}{ - "duration": time.Duration(2888) * time.Microsecond, + "duration_ns": (time.Duration(2888) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1499817952283903000).UTC(), }, @@ -281,7 +281,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "endpoint_host": "0:0", }, Fields: map[string]interface{}{ - "duration": time.Duration(2888) * time.Microsecond, + "duration_ns": (time.Duration(2888) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1499817952283903000).UTC(), }, @@ -330,7 +330,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ - "duration": time.Duration(1) * time.Nanosecond, + "duration_ns": (time.Duration(1) * time.Nanosecond).Nanoseconds(), }, Time: time.Unix(1, 0).UTC(), }, diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index d59c93f2866de..51b3ce3097632 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -49,7 +49,7 @@ func TestZipkinPlugin(t *testing.T) { "type": "STRING", }, Fields: map[string]interface{}{ - "duration": time.Duration(53106) * time.Microsecond, + "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851331000).UTC(), }, @@ -67,7 +67,7 @@ func TestZipkinPlugin(t *testing.T) { "type": "STRING", }, Fields: map[string]interface{}{ - "duration": time.Duration(50410) * time.Microsecond, + "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360904552000).UTC(), }, @@ -83,7 +83,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Parent", }, Fields: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -99,7 +99,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Parent", }, Fields: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -115,7 +115,7 @@ func TestZipkinPlugin(t *testing.T) { "id": "22964302721410078", }, Fields: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -133,7 +133,7 @@ func TestZipkinPlugin(t *testing.T) { "endpoint_host": "127.0.0.1:0", }, Fields: map[string]interface{}{ - "duration": time.Duration(103680) * time.Microsecond, + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), }, @@ -159,7 +159,7 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ - "duration": time.Duration(1) * time.Microsecond, + "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), }, //Time: time.Unix(1, 0).UTC(), Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), @@ -176,7 +176,7 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ - "duration": time.Duration(1) * time.Microsecond, + "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), }, From 576685fba7badc75b6e1a3a8d5b51926afdf1181 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 16:18:40 -0600 Subject: [PATCH 121/154] Change key to annotation_key --- plugins/inputs/zipkin/convert.go | 2 +- plugins/inputs/zipkin/convert_test.go | 8 ++++---- plugins/inputs/zipkin/zipkin_test.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 09e044f55c335..a51ac8f6a100f 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -63,7 +63,7 @@ func (l *LineProtocolConverter) Record(t Trace) error { "service_name": b.ServiceName, "annotation_value": b.Value, "endpoint_host": b.Host, - "key": b.Key, + "annotation_key": b.Key, "type": b.Type, } l.acc.AddFields("zipkin", fields, tags, s.Timestamp) diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index bbbb43f4fc716..ca638e8624e4d 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -118,7 +118,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "service_name": "trivial", "annotation_value": "dHJpdmlhbA==", "endpoint_host": "2130706433:0", - "key": "lc", + "annotation_key": "lc", "type": "STRING", }, Fields: map[string]interface{}{ @@ -136,7 +136,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "service_name": "trivial", "annotation_value": "dHJpdmlhbA==", "endpoint_host": "2130706433:0", - "key": "lc", + "annotation_key": "lc", "type": "STRING", }, Fields: map[string]interface{}{ @@ -198,7 +198,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "trace_id": "2505404965370368069", "service_name": "trivial", "annotation_value": "dHJpdmlhbA==", - "key": "lc", + "annotation_key": "lc", "type": "STRING", "id": "22964302721410078", "parent_id": "22964302721410078", @@ -273,7 +273,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "trace_id": "2505404965370368069", "service_name": "cli", "annotation_value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", - "key": "http.url", + "annotation_key": "http.url", "type": "STRING", "id": "22964302721410078", "parent_id": "22964302721410078", diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 51b3ce3097632..72b9045ba01df 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -45,7 +45,7 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "trivial", "annotation_value": "trivial", //base64: dHJpdmlhbA== "endpoint_host": "127.0.0.1:0", - "key": "lc", + "annotation_key": "lc", "type": "STRING", }, Fields: map[string]interface{}{ @@ -63,7 +63,7 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "trivial", "annotation_value": "trivial", //base64: dHJpdmlhbA== "endpoint_host": "127.0.0.1:0", - "key": "lc", + "annotation_key": "lc", "type": "STRING", }, Fields: map[string]interface{}{ @@ -125,7 +125,7 @@ func TestZipkinPlugin(t *testing.T) { "trace_id": "0:2505404965370368069", "service_name": "trivial", "annotation_value": "trivial", //base64: dHJpdmlhbA== - "key": "lc", + "annotation_key": "lc", "type": "STRING", "id": "22964302721410078", "parent_id": "22964302721410078", From 63caa1d2c8b0b3740f275c6e4c0b6a182af75879 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 16:25:13 -0600 Subject: [PATCH 122/154] Change annotation_value to annotation --- plugins/inputs/zipkin/convert.go | 34 +++---- plugins/inputs/zipkin/convert_test.go | 114 +++++++++++------------ plugins/inputs/zipkin/zipkin_test.go | 126 +++++++++++++------------- 3 files changed, 137 insertions(+), 137 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index a51ac8f6a100f..1710b9562d9e7 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -35,17 +35,17 @@ func (l *LineProtocolConverter) Record(t Trace) error { for _, s := range t { for _, a := range s.Annotations { fields := map[string]interface{}{ - "duration_ns": s.Duration.Nanoseconds(), + "duration_ns": s.Duration.Nanoseconds(), } tags := map[string]string{ - "id": s.ID, - "parent_id": s.ParentID, - "trace_id": s.TraceID, - "name": s.Name, - "service_name": a.ServiceName, - "annotation_value": a.Value, - "endpoint_host": a.Host, + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + "service_name": a.ServiceName, + "annotation": a.Value, + "endpoint_host": a.Host, } l.acc.AddFields("zipkin", fields, tags, s.Timestamp) } @@ -56,15 +56,15 @@ func (l *LineProtocolConverter) Record(t Trace) error { } tags := map[string]string{ - "id": s.ID, - "parent_id": s.ParentID, - "trace_id": s.TraceID, - "name": s.Name, - "service_name": b.ServiceName, - "annotation_value": b.Value, - "endpoint_host": b.Host, - "annotation_key": b.Key, - "type": b.Type, + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + "service_name": b.ServiceName, + "annotation": b.Value, + "endpoint_host": b.Host, + "annotation_key": b.Key, + "type": b.Type, } l.acc.AddFields("zipkin", fields, tags, s.Timestamp) } diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index ca638e8624e4d..87d079072d68b 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -111,15 +111,15 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "dHJpdmlhbA==", - "endpoint_host": "2130706433:0", - "annotation_key": "lc", - "type": "STRING", + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation": "dHJpdmlhbA==", + "endpoint_host": "2130706433:0", + "annotation_key": "lc", + "type": "STRING", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), @@ -129,15 +129,15 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "dHJpdmlhbA==", - "endpoint_host": "2130706433:0", - "annotation_key": "lc", - "type": "STRING", + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation": "dHJpdmlhbA==", + "endpoint_host": "2130706433:0", + "annotation_key": "lc", + "type": "STRING", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), @@ -147,13 +147,13 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "service_name": "trivial", - "annotation_value": "Starting child #0", - "endpoint_host": "2130706433:0", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Parent", + "service_name": "trivial", + "annotation": "Starting child #0", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -163,13 +163,13 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "service_name": "trivial", - "annotation_value": "Starting child #1", - "endpoint_host": "2130706433:0", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Parent", + "service_name": "trivial", + "annotation": "Starting child #1", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -179,13 +179,13 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "A Log", - "endpoint_host": "2130706433:0", - "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation": "A Log", + "endpoint_host": "2130706433:0", + "id": "22964302721410078", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -195,15 +195,15 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "trace_id": "2505404965370368069", - "service_name": "trivial", - "annotation_value": "dHJpdmlhbA==", - "annotation_key": "lc", - "type": "STRING", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "name": "Parent", - "endpoint_host": "2130706433:0", + "trace_id": "2505404965370368069", + "service_name": "trivial", + "annotation": "dHJpdmlhbA==", + "annotation_key": "lc", + "type": "STRING", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "name": "Parent", + "endpoint_host": "2130706433:0", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -259,7 +259,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "trace_id": "8269862291023777619:243463817635710260", "name": "Concat", "service_name": "cli", - "annotation_value": "cs", + "annotation": "cs", "endpoint_host": "0:0", }, Fields: map[string]interface{}{ @@ -272,7 +272,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { Tags: map[string]string{ "trace_id": "2505404965370368069", "service_name": "cli", - "annotation_value": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", + "annotation": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", "annotation_key": "http.url", "type": "STRING", "id": "22964302721410078", @@ -321,13 +321,13 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "annotation_value": "cs", - "endpoint_host": "0:9410", - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", - "name": "main.dud", - "service_name": "go-zipkin-testclient", + "annotation": "cs", + "endpoint_host": "0:9410", + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(1) * time.Nanosecond).Nanoseconds(), diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 72b9045ba01df..4e0e951e8e105 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -24,7 +24,7 @@ func TestZipkinPlugin(t *testing.T) { tests := []struct { name string fields fields - thriftDataFile string //pathname to a binary thrift data file which contains test data + thriftDataFile string //path name to a binary thrift data file which contains test data wantErr bool want []testutil.Metric }{ @@ -38,15 +38,15 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "trivial", //base64: dHJpdmlhbA== - "endpoint_host": "127.0.0.1:0", - "annotation_key": "lc", - "type": "STRING", + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation": "trivial", //base64: dHJpdmlhbA== + "endpoint_host": "127.0.0.1:0", + "annotation_key": "lc", + "type": "STRING", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), @@ -56,15 +56,15 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Child", - "service_name": "trivial", - "annotation_value": "trivial", //base64: dHJpdmlhbA== - "endpoint_host": "127.0.0.1:0", - "annotation_key": "lc", - "type": "STRING", + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Child", + "service_name": "trivial", + "annotation": "trivial", //base64: dHJpdmlhbA== + "endpoint_host": "127.0.0.1:0", + "annotation_key": "lc", + "type": "STRING", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), @@ -74,13 +74,13 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "service_name": "trivial", - "annotation_value": "Starting child #0", - "endpoint_host": "127.0.0.1:0", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", + "service_name": "trivial", + "annotation": "Starting child #0", + "endpoint_host": "127.0.0.1:0", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -90,13 +90,13 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "service_name": "trivial", - "annotation_value": "Starting child #1", - "endpoint_host": "127.0.0.1:0", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", + "service_name": "trivial", + "annotation": "Starting child #1", + "endpoint_host": "127.0.0.1:0", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -106,13 +106,13 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", - "name": "Parent", - "service_name": "trivial", - "annotation_value": "A Log", - "endpoint_host": "127.0.0.1:0", - "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + "service_name": "trivial", + "annotation": "A Log", + "endpoint_host": "127.0.0.1:0", + "id": "22964302721410078", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -122,15 +122,15 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "trace_id": "0:2505404965370368069", - "service_name": "trivial", - "annotation_value": "trivial", //base64: dHJpdmlhbA== - "annotation_key": "lc", - "type": "STRING", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "name": "Parent", - "endpoint_host": "127.0.0.1:0", + "trace_id": "0:2505404965370368069", + "service_name": "trivial", + "annotation": "trivial", //base64: dHJpdmlhbA== + "annotation_key": "lc", + "type": "STRING", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "name": "Parent", + "endpoint_host": "127.0.0.1:0", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -150,13 +150,13 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "annotation_value": "cs", - "endpoint_host": "0.0.0.0:9410", - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", - "name": "main.dud", - "service_name": "go-zipkin-testclient", + "annotation": "cs", + "endpoint_host": "0.0.0.0:9410", + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), @@ -167,13 +167,13 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "annotation_value": "cr", - "endpoint_host": "0.0.0.0:9410", - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", - "name": "main.dud", - "service_name": "go-zipkin-testclient", + "annotation": "cr", + "endpoint_host": "0.0.0.0:9410", + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "go-zipkin-testclient", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), From a708987ca108dfc46a57a76dbacd424616bc87cc Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 16:31:19 -0600 Subject: [PATCH 123/154] Remove type from binary annotations in zipkin plugin --- plugins/inputs/zipkin/convert.go | 1 - plugins/inputs/zipkin/convert_test.go | 4 ---- plugins/inputs/zipkin/zipkin_test.go | 3 --- 3 files changed, 8 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 1710b9562d9e7..f3770c4c479f3 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -64,7 +64,6 @@ func (l *LineProtocolConverter) Record(t Trace) error { "annotation": b.Value, "endpoint_host": b.Host, "annotation_key": b.Key, - "type": b.Type, } l.acc.AddFields("zipkin", fields, tags, s.Timestamp) } diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index 87d079072d68b..c4d81d59e2a62 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -119,7 +119,6 @@ func TestLineProtocolConverter_Record(t *testing.T) { "annotation": "dHJpdmlhbA==", "endpoint_host": "2130706433:0", "annotation_key": "lc", - "type": "STRING", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), @@ -137,7 +136,6 @@ func TestLineProtocolConverter_Record(t *testing.T) { "annotation": "dHJpdmlhbA==", "endpoint_host": "2130706433:0", "annotation_key": "lc", - "type": "STRING", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), @@ -199,7 +197,6 @@ func TestLineProtocolConverter_Record(t *testing.T) { "service_name": "trivial", "annotation": "dHJpdmlhbA==", "annotation_key": "lc", - "type": "STRING", "id": "22964302721410078", "parent_id": "22964302721410078", "name": "Parent", @@ -274,7 +271,6 @@ func TestLineProtocolConverter_Record(t *testing.T) { "service_name": "cli", "annotation": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", "annotation_key": "http.url", - "type": "STRING", "id": "22964302721410078", "parent_id": "22964302721410078", "name": "Concat", diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 4e0e951e8e105..da8f381991fc9 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -46,7 +46,6 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "trivial", //base64: dHJpdmlhbA== "endpoint_host": "127.0.0.1:0", "annotation_key": "lc", - "type": "STRING", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), @@ -64,7 +63,6 @@ func TestZipkinPlugin(t *testing.T) { "annotation": "trivial", //base64: dHJpdmlhbA== "endpoint_host": "127.0.0.1:0", "annotation_key": "lc", - "type": "STRING", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), @@ -126,7 +124,6 @@ func TestZipkinPlugin(t *testing.T) { "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== "annotation_key": "lc", - "type": "STRING", "id": "22964302721410078", "parent_id": "22964302721410078", "name": "Parent", From df7c93b890d271d22b27f0ba0c53d63956ed43ca Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 16:33:49 -0600 Subject: [PATCH 124/154] Update README in zipkin plugin --- plugins/inputs/zipkin/README.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index 204a845230e6a..7568aae78855c 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -13,7 +13,7 @@ This plugin implements the Zipkin http server to gather trace and timing data ne ### Tracing: -*This plugin uses Annotations tags and fields to track data from spans +This plugin uses Annotations tags and fields to track data from spans - TRACE : is a set of spans that share a single root span. Traces are built by collecting all Spans that share a traceId. @@ -33,28 +33,26 @@ Traces are built by collecting all Spans that share a traceId. RPC is considered complete with this annotation - TAGS: - _"id":_ The 64 or 128-bit ID of the trace. Every span in a trace shares this ID. + "id" The 64 or 128-bit ID of the trace. Every span in a trace shares this ID. _"parent_id":_ An ID associated with a particular child span. If there is no child span, the parent ID is set to itself. _"trace_id":_ The 64 or 128-bit ID of a particular trace. Trace ID High concat Trace ID Low. _"name":_ Defines a span - _"service_name":_ Defines a service - _"annotation_value":_ Defines each individual annotation - _"endpoint_host":_ listening port concat with IPV4 + "__service_name__": Defines a service + _"annotation":_ The value of an annotation + _"endpoint_host":_ Listening port concat with IPV4 -FIELDS - "annotation_timestamp": Start time of an annotation. If time is nil we set it to the current UTC time. - "duration": The time in microseconds between the end and beginning of a span. + "duration_ns": The time in nanoseconds between the end and beginning of a span. ### BINARY ANNOTATIONS: -TAGS: Contains the same tags as annotations plus these additions - "key": Acts as a pointer to some address which houses the value - _"type"_: Given data type + "annotation_key": label describing the annotation -FIELDS: - "duration": The time in microseconds between the end and beginning of a span. + "duration_ns": The time in nanoseconds between the end and beginning of a span. @@ -62,7 +60,7 @@ Traces are built by collecting all Spans that share a traceId. - Get All Span Names for Service `my_web_server` ```sql -SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_server'``` +SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_server' ``` - __Description:__ returns a list containing the names of the spans which have annotations with the given `service_name` of `my_web_server`. - Get All Service Names @@ -72,7 +70,7 @@ SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_ser - Find spans with longest duration ```sql - SELECT max("duration") FROM "zipkin" WHERE "service_name" = 'my_service' AND "name" = 'my_span_name' AND time > now() - 20m GROUP BY "trace_id",time(30s) LIMIT 5 + SELECT max("duration_ns") FROM "zipkin" WHERE "service_name" = 'my_service' AND "name" = 'my_span_name' AND time > now() - 20m GROUP BY "trace_id",time(30s) LIMIT 5 ``` - __Description:__ In the last 20 minutes find the top 5 longest span durations for service `my_server` and span name `my_span_name` @@ -84,7 +82,7 @@ SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_ser - [Test data from distributed trace repo sample json](https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json) #### Trace Example -``` { +```{ "traceId": "bd7a977555f6b982", "name": "query", "id": "be2d01e33cc78d97", @@ -131,6 +129,7 @@ SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_ser } } ] - },``` + }, + ``` ### Recommended installation From 60b4988ebca2547035ad53d21da36f279f8260d5 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 17:09:23 -0600 Subject: [PATCH 125/154] Add additional span with no annotation in zipkin plugin --- Godeps | 1 + plugins/inputs/zipkin/convert.go | 19 +++++---- plugins/inputs/zipkin/convert_test.go | 59 +++++++++++++++++++++++++-- plugins/inputs/zipkin/zipkin_test.go | 59 +++++++++++++++++++++++++-- 4 files changed, 124 insertions(+), 14 deletions(-) diff --git a/Godeps b/Godeps index a2d1ffa30f1ca..5b08b0a30bbc6 100644 --- a/Godeps +++ b/Godeps @@ -23,6 +23,7 @@ github.com/gobwas/glob bea32b9cd2d6f55753d94a28e959b13f0244797a github.com/gogo/protobuf 7b6c6391c4ff245962047fc1e2c6e08b1cdfa0e8 github.com/golang/protobuf 8ee79997227bf9b34611aee7946ae64735e6fd93 github.com/golang/snappy 7db9049039a047d955fe8c19b83c8ff5abd765c7 +github.com/google/go-cmp f94e52cad91c65a63acc1e75d4be223ea22e99bc github.com/gorilla/mux 392c28fe23e1c45ddba891b0320b3b5df220beea github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478 github.com/hashicorp/consul 63d2fc68239b996096a1c55a0d4b400ea4c2583f diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index f3770c4c479f3..85d8441ac0f8c 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -33,11 +33,18 @@ func NewLineProtocolConverter(acc telegraf.Accumulator) *LineProtocolConverter { // telegraf.Accumulator. func (l *LineProtocolConverter) Record(t Trace) error { for _, s := range t { - for _, a := range s.Annotations { - fields := map[string]interface{}{ - "duration_ns": s.Duration.Nanoseconds(), - } + fields := map[string]interface{}{ + "duration_ns": s.Duration.Nanoseconds(), + } + tags := map[string]string{ + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + } + l.acc.AddFields("zipkin", fields, tags, s.Timestamp) + for _, a := range s.Annotations { tags := map[string]string{ "id": s.ID, "parent_id": s.ParentID, @@ -51,10 +58,6 @@ func (l *LineProtocolConverter) Record(t Trace) error { } for _, b := range s.BinaryAnnotations { - fields := map[string]interface{}{ - "duration_ns": s.Duration.Nanoseconds(), - } - tags := map[string]string{ "id": s.ID, "parent_id": s.ParentID, diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index c4d81d59e2a62..abd1e1756e955 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" @@ -108,6 +109,19 @@ func TestLineProtocolConverter_Record(t *testing.T) { }, }, want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + }, + Fields: map[string]interface{}{ + "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), + }, + Time: time.Unix(0, 1498688360851331000).UTC(), + }, testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ @@ -125,6 +139,19 @@ func TestLineProtocolConverter_Record(t *testing.T) { }, Time: time.Unix(0, 1498688360851331000).UTC(), }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Child", + }, + Fields: map[string]interface{}{ + "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), + }, + Time: time.Unix(0, 1498688360904552000).UTC(), + }, testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ @@ -142,6 +169,19 @@ func TestLineProtocolConverter_Record(t *testing.T) { }, Time: time.Unix(0, 1498688360904552000).UTC(), }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "name": "Parent", + }, + Fields: map[string]interface{}{ + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), + }, + Time: time.Unix(0, 1498688360851318000).UTC(), + }, testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ @@ -314,6 +354,19 @@ func TestLineProtocolConverter_Record(t *testing.T) { }, }, want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + }, + Fields: map[string]interface{}{ + "duration_ns": (time.Duration(1) * time.Nanosecond).Nanoseconds(), + }, + Time: time.Unix(1, 0).UTC(), + }, testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ @@ -333,7 +386,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { }, }, } - for _, tt := range tests { + for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockAcc.ClearMetrics() l := &LineProtocolConverter{ @@ -346,8 +399,8 @@ func TestLineProtocolConverter_Record(t *testing.T) { for _, metric := range mockAcc.Metrics { got = append(got, *metric) } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("LineProtocolConverter.Record() error = \n%#v\n, want \n%#v\n", got, tt.want) + if !cmp.Equal(got, tt.want) { + t.Errorf("LineProtocolConverter.Record()/%s/%d error = %s ", tt.name, i, cmp.Diff(got, tt.want)) } }) } diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index da8f381991fc9..56a7719e68b25 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -5,10 +5,10 @@ import ( "fmt" "io/ioutil" "net/http" - "reflect" "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" ) @@ -35,6 +35,19 @@ func TestZipkinPlugin(t *testing.T) { }, thriftDataFile: "testdata/threespans.dat", want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Child", + }, + Fields: map[string]interface{}{ + "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), + }, + Time: time.Unix(0, 1498688360851331000).UTC(), + }, testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ @@ -52,6 +65,19 @@ func TestZipkinPlugin(t *testing.T) { }, Time: time.Unix(0, 1498688360851331000).UTC(), }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Child", + }, + Fields: map[string]interface{}{ + "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), + }, + Time: time.Unix(0, 1498688360904552000).UTC(), + }, testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ @@ -69,6 +95,19 @@ func TestZipkinPlugin(t *testing.T) { }, Time: time.Unix(0, 1498688360904552000).UTC(), }, + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "0:2505404965370368069", + "name": "Parent", + }, + Fields: map[string]interface{}{ + "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), + }, + Time: time.Unix(0, 1498688360851318000).UTC(), + }, testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ @@ -144,6 +183,20 @@ func TestZipkinPlugin(t *testing.T) { }, thriftDataFile: "testdata/distributed_trace_sample.dat", want: []testutil.Metric{ + testutil.Metric{ + Measurement: "zipkin", + Tags: map[string]string{ + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + }, + Fields: map[string]interface{}{ + "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), + }, + //Time: time.Unix(1, 0).UTC(), + Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), + }, testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ @@ -209,8 +262,8 @@ func TestZipkinPlugin(t *testing.T) { got = append(got, *m) } - if !reflect.DeepEqual(got, tt.want) { - t.Fatalf("Got != Want\n Got: %#v\n, Want: %#v\n", got, tt.want) + if !cmp.Equal(got, tt.want) { + t.Fatalf("Got != Want\n %s", cmp.Diff(got, tt.want)) } }) } From 6050cc112ba5545a401d2513c9784e1b30719bb4 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 17:22:21 -0600 Subject: [PATCH 126/154] Change trace ID high and low format to hexidecimal in zipkin plugin --- plugins/inputs/zipkin/convert.go | 6 +++++- plugins/inputs/zipkin/handler_test.go | 6 +++--- plugins/inputs/zipkin/zipkin_test.go | 24 ++++++++++++------------ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 85d8441ac0f8c..75706884ddf78 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -2,6 +2,7 @@ package zipkin import ( "encoding/binary" + "fmt" "net" "strconv" "time" @@ -138,7 +139,10 @@ func formatID(id int64) string { } func formatTraceID(high, low int64) string { - return formatID(high) + ":" + formatID(low) + if high == 0 { + return fmt.Sprintf("%x", low) + } + return fmt.Sprintf("%x%016x", high, low) } func minMax(span *zipkincore.Span) (time.Time, time.Time) { diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index ce71ec716115e..7299e39b6f4aa 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -55,7 +55,7 @@ func TestSpanHandler(t *testing.T) { Span{ Name: "Child", ID: "8090652509916334619", - TraceID: "0:2505404965370368069", + TraceID: "22c4fc8ab3669045", ParentID: parentID, Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)).UTC(), Duration: time.Duration(53106) * time.Microsecond, @@ -73,7 +73,7 @@ func TestSpanHandler(t *testing.T) { Span{ Name: "Child", ID: "103618986556047333", - TraceID: "0:2505404965370368069", + TraceID: "22c4fc8ab3669045", ParentID: parentID, Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)).UTC(), Duration: time.Duration(50410) * time.Microsecond, @@ -91,7 +91,7 @@ func TestSpanHandler(t *testing.T) { Span{ Name: "Parent", ID: "22964302721410078", - TraceID: "0:2505404965370368069", + TraceID: "22c4fc8ab3669045", ParentID: "22964302721410078", Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)).UTC(), Duration: time.Duration(103680) * time.Microsecond, diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 56a7719e68b25..59348c44ae9cc 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -40,7 +40,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "id": "8090652509916334619", "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "name": "Child", }, Fields: map[string]interface{}{ @@ -53,7 +53,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "id": "8090652509916334619", "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "name": "Child", "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== @@ -70,7 +70,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "id": "103618986556047333", "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "name": "Child", }, Fields: map[string]interface{}{ @@ -83,7 +83,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "id": "103618986556047333", "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "name": "Child", "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== @@ -100,7 +100,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "id": "22964302721410078", "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "name": "Parent", }, Fields: map[string]interface{}{ @@ -116,7 +116,7 @@ func TestZipkinPlugin(t *testing.T) { "endpoint_host": "127.0.0.1:0", "id": "22964302721410078", "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "name": "Parent", }, Fields: map[string]interface{}{ @@ -132,7 +132,7 @@ func TestZipkinPlugin(t *testing.T) { "endpoint_host": "127.0.0.1:0", "id": "22964302721410078", "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "name": "Parent", }, Fields: map[string]interface{}{ @@ -144,7 +144,7 @@ func TestZipkinPlugin(t *testing.T) { Measurement: "zipkin", Tags: map[string]string{ "parent_id": "22964302721410078", - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "name": "Parent", "service_name": "trivial", "annotation": "A Log", @@ -159,7 +159,7 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "trace_id": "0:2505404965370368069", + "trace_id": "22c4fc8ab3669045", "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== "annotation_key": "lc", @@ -188,7 +188,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "id": "6802735349851856000", "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", + "trace_id": "5e682bc21ce99c80", "name": "main.dud", }, Fields: map[string]interface{}{ @@ -204,7 +204,7 @@ func TestZipkinPlugin(t *testing.T) { "endpoint_host": "0.0.0.0:9410", "id": "6802735349851856000", "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", + "trace_id": "5e682bc21ce99c80", "name": "main.dud", "service_name": "go-zipkin-testclient", }, @@ -221,7 +221,7 @@ func TestZipkinPlugin(t *testing.T) { "endpoint_host": "0.0.0.0:9410", "id": "6802735349851856000", "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", + "trace_id": "5e682bc21ce99c80", "name": "main.dud", "service_name": "go-zipkin-testclient", }, From 52066d6d597b67380e5feb183b4e3bdb029399e3 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Fri, 28 Jul 2017 17:27:43 -0600 Subject: [PATCH 127/154] Update README definitions --- plugins/inputs/zipkin/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index 7568aae78855c..29905c0780620 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -33,9 +33,9 @@ Traces are built by collecting all Spans that share a traceId. RPC is considered complete with this annotation - TAGS: - "id" The 64 or 128-bit ID of the trace. Every span in a trace shares this ID. - _"parent_id":_ An ID associated with a particular child span. If there is no child span, the parent ID is set to itself. - _"trace_id":_ The 64 or 128-bit ID of a particular trace. Trace ID High concat Trace ID Low. + "id" The 64 bit ID of the span. + _"parent_id":_ An ID associated with a particular child span. If there is no child span, the parent ID is set to ID. + _"trace_id":_ The 64 or 128-bit ID of a particular trace. Every span in a trace shares this ID. Concatenation of high and low and converted to hexadecimal. _"name":_ Defines a span "__service_name__": Defines a service _"annotation":_ The value of an annotation From 7078fa4fe33a274524a95a3b11d7d1536acd03b7 Mon Sep 17 00:00:00 2001 From: Ayrdrie Date: Fri, 28 Jul 2017 17:52:35 -0600 Subject: [PATCH 128/154] Update README formatting for zipkin plugin --- plugins/inputs/zipkin/README.md | 74 ++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index 29905c0780620..af28f8b43aee8 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -33,13 +33,13 @@ Traces are built by collecting all Spans that share a traceId. RPC is considered complete with this annotation - TAGS: - "id" The 64 bit ID of the span. - _"parent_id":_ An ID associated with a particular child span. If there is no child span, the parent ID is set to ID. - _"trace_id":_ The 64 or 128-bit ID of a particular trace. Every span in a trace shares this ID. Concatenation of high and low and converted to hexadecimal. - _"name":_ Defines a span - "__service_name__": Defines a service - _"annotation":_ The value of an annotation - _"endpoint_host":_ Listening port concat with IPV4 + * __"id":__ The 64 bit ID of the span. + * __"parent_id":__ An ID associated with a particular child span. If there is no child span, the parent ID is set to ID. + * __"trace_id":__ The 64 or 128-bit ID of a particular trace. Every span in a trace shares this ID. Concatenation of high and low and converted to hexadecimal. + * __"name":__ Defines a span + * __"service_name":__ Defines a service + * __"annotation":__ The value of an annotation + * __"endpoint_host":__ Listening port concat with IPV4 -FIELDS "duration_ns": The time in nanoseconds between the end and beginning of a span. @@ -58,21 +58,23 @@ Traces are built by collecting all Spans that share a traceId. ### Sample Queries: -- Get All Span Names for Service `my_web_server` +__Get All Span Names for Service__ `my_web_server` ```sql -SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_server' ``` - - __Description:__ returns a list containing the names of the spans which have annotations with the given `service_name` of `my_web_server`. +SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_server' +``` + - _Description:_ returns a list containing the names of the spans which have annotations with the given `service_name` of `my_web_server`. -- Get All Service Names - ```sql - SHOW TAG VALUES FROM "zipkin" WITH KEY = "service_name"``` - - __Description:__ returns a list of all `distinct` endpoint service names. +__Get All Service Names__ +```sql +SHOW TAG VALUES FROM "zipkin" WITH KEY = "service_name" +``` + - _Description:_ returns a list of all `distinct` endpoint service names. -- Find spans with longest duration - ```sql - SELECT max("duration_ns") FROM "zipkin" WHERE "service_name" = 'my_service' AND "name" = 'my_span_name' AND time > now() - 20m GROUP BY "trace_id",time(30s) LIMIT 5 - ``` - - __Description:__ In the last 20 minutes find the top 5 longest span durations for service `my_server` and span name `my_span_name` +__Find spans with longest duration__ +```sql +SELECT max("duration_ns") FROM "zipkin" WHERE "service_name" = 'my_service' AND "name" = 'my_span_name' AND time > now() - 20m GROUP BY "trace_id",time(30s) LIMIT 5 +``` + - _Description:_ In the last 20 minutes find the top 5 longest span durations for service `my_server` and span name `my_span_name` @@ -133,3 +135,37 @@ SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_ser ``` ### Recommended installation + +We recomend using the [tsi influxDB engine](https://www.influxdata.com/path-1-billion-time-series-influxdb-high-cardinality-indexing-ready-testing/) as it can accept high cardinality data. +#### How To Set Up InfluxDB For Work With Zipkin + +##### Steps +___Update___ InfluxDB to >= 1.3, in order to use the new tsi engine. + +___Generate___ a config file with the following command: + `influxd config > /path/for/config/file` + +___Add___ the following to your config file, under the `[data]` tab: + +```toml +[data] + dir = "/Users/goller/.influxdb/data" + index-version = "tsi1" + wal-dir = "/Users/goller/.influxdb/wal" + query-log-enabled = true + cache-max-memory-size = 1073741824 + cache-snapshot-memory-size = 26214400 + cache-snapshot-write-cold-duration = "10m0s" + compact-full-write-cold-duration = "4h0m0s" + max-series-per-database = 1000000 + max-values-per-tag = 100000 + trace-logging-enabled = false + ``` + + ___Start___ `influxd` with your new config file: + `$ influxd -config=/path/to/your/config/file` + +___Update___ your retention policy: +```sql +ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m +``` From 23143881ecd0519e2831b4415b95267c0918a020 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 31 Jul 2017 11:07:42 -0700 Subject: [PATCH 129/154] Update spans to have unique service names --- plugins/inputs/zipkin/convert.go | 41 +++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 75706884ddf78..6db2d955e127c 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -84,6 +84,7 @@ func (l *LineProtocolConverter) Error(err error) { func NewTrace(spans []*zipkincore.Span) Trace { trace := make(Trace, len(spans)) for i, span := range spans { + endpoint := serviceEndpoint(span.GetAnnotations(), span.GetBinaryAnnotations()) trace[i] = Span{ ID: formatID(span.GetID()), TraceID: formatTraceID(span.GetTraceIDHigh(), span.GetTraceID()), @@ -91,8 +92,8 @@ func NewTrace(spans []*zipkincore.Span) Trace { Timestamp: guessTimestamp(span), Duration: convertDuration(span), ParentID: parentID(span), - Annotations: NewAnnotations(span.GetAnnotations()), - BinaryAnnotations: NewBinaryAnnotations(span.GetBinaryAnnotations()), + Annotations: NewAnnotations(span.GetAnnotations(), endpoint), + BinaryAnnotations: NewBinaryAnnotations(span.GetBinaryAnnotations(), endpoint), } } return trace @@ -100,12 +101,12 @@ func NewTrace(spans []*zipkincore.Span) Trace { // NewAnnotations converts a slice of *zipkincore.Annotation into a slice // of new Annotations -func NewAnnotations(annotations []*zipkincore.Annotation) []Annotation { +func NewAnnotations(annotations []*zipkincore.Annotation, endpoint *zipkincore.Endpoint) []Annotation { formatted := make([]Annotation, len(annotations)) for i, annotation := range annotations { formatted[i] = Annotation{ - Host: host(annotation.GetHost()), - ServiceName: serviceName(annotation.GetHost()), + Host: host(endpoint), + ServiceName: serviceName(endpoint), Timestamp: microToTime(annotation.GetTimestamp()), Value: annotation.GetValue(), } @@ -116,12 +117,12 @@ func NewAnnotations(annotations []*zipkincore.Annotation) []Annotation { // NewBinaryAnnotations is very similar to NewAnnotations, but it // converts zipkincore.BinaryAnnotations instead of the normal zipkincore.Annotation -func NewBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation) []BinaryAnnotation { +func NewBinaryAnnotations(annotations []*zipkincore.BinaryAnnotation, endpoint *zipkincore.Endpoint) []BinaryAnnotation { formatted := make([]BinaryAnnotation, len(annotations)) for i, annotation := range annotations { formatted[i] = BinaryAnnotation{ - Host: host(annotation.GetHost()), - ServiceName: serviceName(annotation.GetHost()), + Host: host(endpoint), + ServiceName: serviceName(endpoint), Key: annotation.GetKey(), Value: string(annotation.GetValue()), Type: annotation.GetAnnotationType().String(), @@ -210,3 +211,27 @@ func serviceName(h *zipkincore.Endpoint) string { } return h.GetServiceName() } + +func serviceEndpoint(ann []*zipkincore.Annotation, bann []*zipkincore.BinaryAnnotation) *zipkincore.Endpoint { + for _, a := range ann { + switch a.Value { + case zipkincore.SERVER_RECV, zipkincore.SERVER_SEND, zipkincore.CLIENT_RECV, zipkincore.CLIENT_SEND: + if a.Host != nil && a.Host.ServiceName != "" { + return a.Host + } + } + } + + for _, a := range bann { + if a.Key == zipkincore.LOCAL_COMPONENT && a.Host != nil && a.Host.ServiceName != "" { + return a.Host + } + } + // Unable to find any "standard" endpoint host, so, use any that exist in the regular annotations + for _, a := range ann { + if a.Host != nil && a.Host.ServiceName != "" { + return a.Host + } + } + return nil +} From 667858677adf58beaafcb8eb46841c4ec0eb2422 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Mon, 31 Jul 2017 15:30:07 -0600 Subject: [PATCH 130/154] Fix zipkins signed unsigned port number conflict --- plugins/inputs/zipkin/convert.go | 5 ++++- plugins/inputs/zipkin/convert_test.go | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 6db2d955e127c..0915c02880802 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -202,7 +202,10 @@ func host(h *zipkincore.Endpoint) string { if h == nil { return "" } - return ipv4(h.GetIpv4()) + ":" + strconv.Itoa(int(h.GetPort())) + // Zipkin uses a signed int16 for the port, but, warns us that they actually treat it + // as an unsigned int16. So, we convert from int16 to int32 followed by taking & 0xffff + // to convert from signed to unsigned + return ipv4(h.GetIpv4()) + ":" + strconv.FormatInt(int64(int(h.GetPort())&0xffff), 10) } func serviceName(h *zipkincore.Endpoint) string { diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index abd1e1756e955..b6d6b9199310f 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -554,6 +554,16 @@ func Test_host(t *testing.T) { }, want: "", }, + { + name: "int overflow zipkin uses an int16 type as an unsigned int 16.", + args: args{ + h: &zipkincore.Endpoint{ + Ipv4: 1234, + Port: -1, + }, + }, + want: "0.0.4.210:65535", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 4b6930170d63dcb4d9ab56e4997767b11e65878b Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Mon, 31 Jul 2017 15:40:08 -0600 Subject: [PATCH 131/154] Update zipkin description and sample configuration --- plugins/inputs/zipkin/README.md | 14 ++------------ plugins/inputs/zipkin/zipkin.go | 6 +++--- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index af28f8b43aee8..f2889d4c030df 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -7,7 +7,7 @@ This plugin implements the Zipkin http server to gather trace and timing data ne ### Configuration: ```toml [[inputs.zipkin]] - path = "/api/v1/spans" #Path on which Telegraf listens for spans + path = "/api/v1/spans" # URL path for span data port = 9411 # Port on which Telegraf listens ``` @@ -60,7 +60,7 @@ Traces are built by collecting all Spans that share a traceId. __Get All Span Names for Service__ `my_web_server` ```sql -SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_server' +SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_server' ``` - _Description:_ returns a list containing the names of the spans which have annotations with the given `service_name` of `my_web_server`. @@ -149,17 +149,7 @@ ___Add___ the following to your config file, under the `[data]` tab: ```toml [data] - dir = "/Users/goller/.influxdb/data" index-version = "tsi1" - wal-dir = "/Users/goller/.influxdb/wal" - query-log-enabled = true - cache-max-memory-size = 1073741824 - cache-snapshot-memory-size = 26214400 - cache-snapshot-write-cold-duration = "10m0s" - compact-full-write-cold-duration = "4h0m0s" - max-series-per-database = 1000000 - max-values-per-tag = 100000 - trace-logging-enabled = false ``` ___Start___ `influxd` with your new config file: diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index b4efb5162a958..b521ae8f2a4c3 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -79,8 +79,8 @@ type Trace []Span const sampleConfig = ` ## - # path = /path/your/zipkin/impl/posts/to - # port = + # path = "/api/v1/spans" # URL path for span data + # port = 9411 # Port on which Telegraf listens ` // Zipkin is a telegraf configuration structure for the zipkin input plugin, @@ -97,7 +97,7 @@ type Zipkin struct { // Description is a necessary method implementation from telegraf.ServiceInput func (z Zipkin) Description() string { - return "Allows for the collection of zipkin tracing spans for storage in InfluxDB" + return "This plugin implements the Zipkin http server to gather trace and timing data needed to troubleshoot latency problems in microservice architectures." } // SampleConfig is a necessary method implementation from telegraf.ServiceInput From 6aa335c1455639ad12af807e5c1dfbcb92e36a13 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Mon, 31 Jul 2017 15:42:41 -0600 Subject: [PATCH 132/154] Remove unnecessary files in zipkin plugin --- .../inputs/zipkin/influx_sample_config.txt | 123 ------------------ .../zipkin/influx_setup_instructions.md | 32 ----- 2 files changed, 155 deletions(-) delete mode 100644 plugins/inputs/zipkin/influx_sample_config.txt delete mode 100644 plugins/inputs/zipkin/influx_setup_instructions.md diff --git a/plugins/inputs/zipkin/influx_sample_config.txt b/plugins/inputs/zipkin/influx_sample_config.txt deleted file mode 100644 index 76105ea0df548..0000000000000 --- a/plugins/inputs/zipkin/influx_sample_config.txt +++ /dev/null @@ -1,123 +0,0 @@ -reporting-disabled = false -bind-address = "127.0.0.1:8088" - -[meta] - dir = "/Users/adamperlin/.influxdb/meta" - retention-autocreate = true - logging-enabled = true - -[data] - dir = "/Users/adamperlin/.influxdb/data" - index-version = "tsi1" - wal-dir = "/Users/adamperlin/.influxdb/wal" - query-log-enabled = true - cache-max-memory-size = 1073741824 - cache-snapshot-memory-size = 26214400 - cache-snapshot-write-cold-duration = "10m0s" - compact-full-write-cold-duration = "4h0m0s" - max-series-per-database = 1000000 - max-values-per-tag = 100000 - trace-logging-enabled = false - -[coordinator] - write-timeout = "10s" - max-concurrent-queries = 0 - query-timeout = "0s" - log-queries-after = "0s" - max-select-point = 0 - max-select-series = 0 - max-select-buckets = 0 - -[retention] - enabled = true - check-interval = "30m0s" - -[shard-precreation] - enabled = true - check-interval = "10m0s" - advance-period = "30m0s" - -[monitor] - store-enabled = true - store-database = "_internal" - store-interval = "10s" - -[subscriber] - enabled = true - http-timeout = "30s" - insecure-skip-verify = false - ca-certs = "" - write-concurrency = 40 - write-buffer-size = 1000 - -[http] - enabled = true - bind-address = ":8086" - auth-enabled = false - log-enabled = true - write-tracing = false - pprof-enabled = true - https-enabled = false - https-certificate = "/etc/ssl/influxdb.pem" - https-private-key = "" - max-row-limit = 0 - max-connection-limit = 0 - shared-secret = "" - realm = "InfluxDB" - unix-socket-enabled = false - bind-socket = "/var/run/influxdb.sock" - -[[graphite]] - enabled = false - bind-address = ":2003" - database = "graphite" - retention-policy = "" - protocol = "tcp" - batch-size = 5000 - batch-pending = 10 - batch-timeout = "1s" - consistency-level = "one" - separator = "." - udp-read-buffer = 0 - -[[collectd]] - enabled = false - bind-address = ":25826" - database = "collectd" - retention-policy = "" - batch-size = 5000 - batch-pending = 10 - batch-timeout = "10s" - read-buffer = 0 - typesdb = "/usr/share/collectd/types.db" - security-level = "none" - auth-file = "/etc/collectd/auth_file" - -[[opentsdb]] - enabled = false - bind-address = ":4242" - database = "opentsdb" - retention-policy = "" - consistency-level = "one" - tls-enabled = false - certificate = "/etc/ssl/influxdb.pem" - batch-size = 1000 - batch-pending = 5 - batch-timeout = "1s" - log-point-errors = true - -[[udp]] - enabled = false - bind-address = ":8089" - database = "udp" - retention-policy = "" - batch-size = 5000 - batch-pending = 10 - read-buffer = 0 - batch-timeout = "1s" - precision = "" - -[continuous_queries] - log-enabled = true - enabled = true - run-interval = "1s" diff --git a/plugins/inputs/zipkin/influx_setup_instructions.md b/plugins/inputs/zipkin/influx_setup_instructions.md deleted file mode 100644 index 07658e123ee7e..0000000000000 --- a/plugins/inputs/zipkin/influx_setup_instructions.md +++ /dev/null @@ -1,32 +0,0 @@ -# How To Set Up InfluxDB For Work With Zipkin - -## Steps -___Update___ InfluxDB to >= 1.3, in order to use the new tsi engine. - -___Generate___ a config file with the following command: - `influxd config > /path/for/config/file` - -___Add___ the following to your config file, under the `[data]` tab: - -```toml -[data] - dir = "/Users/goller/.influxdb/data" - index-version = "tsi1" - wal-dir = "/Users/goller/.influxdb/wal" - query-log-enabled = true - cache-max-memory-size = 1073741824 - cache-snapshot-memory-size = 26214400 - cache-snapshot-write-cold-duration = "10m0s" - compact-full-write-cold-duration = "4h0m0s" - max-series-per-database = 1000000 - max-values-per-tag = 100000 - trace-logging-enabled = false - ``` - - ___Start___ `influxd` with your new config file: - `$ influxd -config=/path/to/your/config/file` - -___Update___ your retention policy: -```sql -ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m -``` From db5d153736587fbf361367732adb7ea6141721d2 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Mon, 31 Jul 2017 15:49:07 -0600 Subject: [PATCH 133/154] Change host so that port is not shown if zero --- plugins/inputs/zipkin/convert.go | 3 +++ plugins/inputs/zipkin/handler_test.go | 12 ++++++------ plugins/inputs/zipkin/zipkin_test.go | 12 ++++++------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 0915c02880802..1d1c02d930b46 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -202,6 +202,9 @@ func host(h *zipkincore.Endpoint) string { if h == nil { return "" } + if h.GetPort() == 0 { + return ipv4(h.GetIpv4()) + } // Zipkin uses a signed int16 for the port, but, warns us that they actually treat it // as an unsigned int16. So, we convert from int16 to int32 followed by taking & 0xffff // to convert from signed to unsigned diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 7299e39b6f4aa..3155c2cbd9ec8 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -64,7 +64,7 @@ func TestSpanHandler(t *testing.T) { BinaryAnnotation{ Key: "lc", Value: "trivial", - Host: "127.0.0.1:0", + Host: "127.0.0.1", ServiceName: "trivial", Type: "STRING", }, @@ -82,7 +82,7 @@ func TestSpanHandler(t *testing.T) { BinaryAnnotation{ Key: "lc", Value: "trivial", - Host: "127.0.0.1:0", + Host: "127.0.0.1", ServiceName: "trivial", Type: "STRING", }, @@ -99,19 +99,19 @@ func TestSpanHandler(t *testing.T) { Annotation{ Timestamp: time.Unix(0, 1498688360851325*int64(time.Microsecond)).UTC(), Value: "Starting child #0", - Host: "127.0.0.1:0", + Host: "127.0.0.1", ServiceName: "trivial", }, Annotation{ Timestamp: time.Unix(0, 1498688360904545*int64(time.Microsecond)).UTC(), Value: "Starting child #1", - Host: "127.0.0.1:0", + Host: "127.0.0.1", ServiceName: "trivial", }, Annotation{ Timestamp: time.Unix(0, 1498688360954992*int64(time.Microsecond)).UTC(), Value: "A Log", - Host: "127.0.0.1:0", + Host: "127.0.0.1", ServiceName: "trivial", }, }, @@ -119,7 +119,7 @@ func TestSpanHandler(t *testing.T) { BinaryAnnotation{ Key: "lc", Value: "trivial", - Host: "127.0.0.1:0", + Host: "127.0.0.1", ServiceName: "trivial", Type: "STRING", }, diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 59348c44ae9cc..72cf218d86de8 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -57,7 +57,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Child", "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== - "endpoint_host": "127.0.0.1:0", + "endpoint_host": "127.0.0.1", "annotation_key": "lc", }, Fields: map[string]interface{}{ @@ -87,7 +87,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Child", "service_name": "trivial", "annotation": "trivial", //base64: dHJpdmlhbA== - "endpoint_host": "127.0.0.1:0", + "endpoint_host": "127.0.0.1", "annotation_key": "lc", }, Fields: map[string]interface{}{ @@ -113,7 +113,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "service_name": "trivial", "annotation": "Starting child #0", - "endpoint_host": "127.0.0.1:0", + "endpoint_host": "127.0.0.1", "id": "22964302721410078", "parent_id": "22964302721410078", "trace_id": "22c4fc8ab3669045", @@ -129,7 +129,7 @@ func TestZipkinPlugin(t *testing.T) { Tags: map[string]string{ "service_name": "trivial", "annotation": "Starting child #1", - "endpoint_host": "127.0.0.1:0", + "endpoint_host": "127.0.0.1", "id": "22964302721410078", "parent_id": "22964302721410078", "trace_id": "22c4fc8ab3669045", @@ -148,7 +148,7 @@ func TestZipkinPlugin(t *testing.T) { "name": "Parent", "service_name": "trivial", "annotation": "A Log", - "endpoint_host": "127.0.0.1:0", + "endpoint_host": "127.0.0.1", "id": "22964302721410078", }, Fields: map[string]interface{}{ @@ -166,7 +166,7 @@ func TestZipkinPlugin(t *testing.T) { "id": "22964302721410078", "parent_id": "22964302721410078", "name": "Parent", - "endpoint_host": "127.0.0.1:0", + "endpoint_host": "127.0.0.1", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), From 33654e02acb9ccd20577956e44324cb9a0389299 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 31 Jul 2017 16:01:21 -0700 Subject: [PATCH 134/154] Add a global span service name to zipkin span schema --- plugins/inputs/zipkin/zipkin.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index b521ae8f2a4c3..4c9fba31aaad9 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -68,6 +68,7 @@ type Span struct { TraceID string // zipkin traceid high concat with traceid Name string ParentID string + ServiceName string Timestamp time.Time // If zipkin input is nil then time.Now() Duration time.Duration Annotations []Annotation From 980981507017cf2e70a7d1ba292a3e7c1cb93a17 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 31 Jul 2017 16:04:12 -0700 Subject: [PATCH 135/154] Add a global span service name to zipkin span schema --- plugins/inputs/zipkin/convert.go | 1 + plugins/inputs/zipkin/handler_test.go | 27 +++++++++++++-------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 1d1c02d930b46..c4fba9e9849b6 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -92,6 +92,7 @@ func NewTrace(spans []*zipkincore.Span) Trace { Timestamp: guessTimestamp(span), Duration: convertDuration(span), ParentID: parentID(span), + ServiceName: serviceName(endpoint), Annotations: NewAnnotations(span.GetAnnotations(), endpoint), BinaryAnnotations: NewBinaryAnnotations(span.GetBinaryAnnotations(), endpoint), } diff --git a/plugins/inputs/zipkin/handler_test.go b/plugins/inputs/zipkin/handler_test.go index 3155c2cbd9ec8..0946d982f1b8f 100644 --- a/plugins/inputs/zipkin/handler_test.go +++ b/plugins/inputs/zipkin/handler_test.go @@ -2,14 +2,14 @@ package zipkin import ( "bytes" - "fmt" "io/ioutil" "net/http" "net/http/httptest" - "reflect" "strconv" "testing" "time" + + "github.com/google/go-cmp/cmp" ) type MockRecorder struct { @@ -59,6 +59,7 @@ func TestSpanHandler(t *testing.T) { ParentID: parentID, Timestamp: time.Unix(0, 1498688360851331*int64(time.Microsecond)).UTC(), Duration: time.Duration(53106) * time.Microsecond, + ServiceName: "trivial", Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ @@ -77,6 +78,7 @@ func TestSpanHandler(t *testing.T) { ParentID: parentID, Timestamp: time.Unix(0, 1498688360904552*int64(time.Microsecond)).UTC(), Duration: time.Duration(50410) * time.Microsecond, + ServiceName: "trivial", Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ @@ -89,12 +91,13 @@ func TestSpanHandler(t *testing.T) { }, }, Span{ - Name: "Parent", - ID: "22964302721410078", - TraceID: "22c4fc8ab3669045", - ParentID: "22964302721410078", - Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)).UTC(), - Duration: time.Duration(103680) * time.Microsecond, + Name: "Parent", + ID: "22964302721410078", + TraceID: "22c4fc8ab3669045", + ParentID: "22964302721410078", + Timestamp: time.Unix(0, 1498688360851318*int64(time.Microsecond)).UTC(), + Duration: time.Duration(103680) * time.Microsecond, + ServiceName: "trivial", Annotations: []Annotation{ Annotation{ Timestamp: time.Unix(0, 1498688360851325*int64(time.Microsecond)).UTC(), @@ -127,11 +130,7 @@ func TestSpanHandler(t *testing.T) { }, } - for i, s := range got { - if !reflect.DeepEqual(s, want[i]) { - fmt.Printf("index %d wasn't equal", i) - fmt.Println(s, want[i]) - t.Fatalf("Got %#v\n != want %#v\n, Fields weren't unmarshalled correctly", s, want[i]) - } + if !cmp.Equal(got, want) { + t.Fatalf("Got != Want\n %s", cmp.Diff(got, want)) } } From ecf371278051376327032fcf3f12b51da247fe43 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 31 Jul 2017 16:28:22 -0700 Subject: [PATCH 136/154] Add a global span service name to zipkin span schema --- plugins/inputs/zipkin/convert.go | 9 +- plugins/inputs/zipkin/convert_test.go | 138 +++++++------------------- plugins/inputs/zipkin/zipkin_test.go | 40 ++++---- 3 files changed, 63 insertions(+), 124 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index c4fba9e9849b6..3956793211cd0 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -38,10 +38,11 @@ func (l *LineProtocolConverter) Record(t Trace) error { "duration_ns": s.Duration.Nanoseconds(), } tags := map[string]string{ - "id": s.ID, - "parent_id": s.ParentID, - "trace_id": s.TraceID, - "name": s.Name, + "id": s.ID, + "parent_id": s.ParentID, + "trace_id": s.TraceID, + "name": s.Name, + "service_name": s.ServiceName, } l.acc.AddFields("zipkin", fields, tags, s.Timestamp) diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index b6d6b9199310f..cc7fee4305c2d 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -40,6 +40,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { ParentID: "22964302721410078", Timestamp: time.Unix(0, 1498688360851331000).UTC(), Duration: time.Duration(53106) * time.Microsecond, + ServiceName: "trivial", Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ @@ -58,6 +59,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { ParentID: "22964302721410078", Timestamp: time.Unix(0, 1498688360904552000).UTC(), Duration: time.Duration(50410) * time.Microsecond, + ServiceName: "trivial", Annotations: []Annotation{}, BinaryAnnotations: []BinaryAnnotation{ BinaryAnnotation{ @@ -70,12 +72,13 @@ func TestLineProtocolConverter_Record(t *testing.T) { }, }, Span{ - ID: "22964302721410078", - TraceID: "2505404965370368069", - Name: "Parent", - ParentID: "22964302721410078", - Timestamp: time.Unix(0, 1498688360851318000).UTC(), - Duration: time.Duration(103680) * time.Microsecond, + ID: "22964302721410078", + TraceID: "2505404965370368069", + Name: "Parent", + ParentID: "22964302721410078", + Timestamp: time.Unix(0, 1498688360851318000).UTC(), + Duration: time.Duration(103680) * time.Microsecond, + ServiceName: "trivial", Annotations: []Annotation{ Annotation{ Timestamp: time.Unix(0, 1498688360851325000).UTC(), @@ -112,10 +115,11 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Child", + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "service_name": "trivial", + "name": "Child", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), @@ -142,10 +146,11 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Child", + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "service_name": "trivial", + "name": "Child", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), @@ -172,10 +177,11 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "2505404965370368069", - "name": "Parent", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "2505404965370368069", + "service_name": "trivial", + "name": "Parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -251,80 +257,6 @@ func TestLineProtocolConverter_Record(t *testing.T) { wantErr: false, }, - // Test data from zipkin cli app: - //https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services - /* { - name: "cli", - fields: fields{ - acc: &mockAcc, - }, - args: args{ - t: Trace{ - Span{ - ID: "3383422996321511664", - TraceID: "243463817635710260", - Name: "Concat", - ParentID: "4574092882326506380", - Timestamp: time.Unix(0, 1499817952283903000).UTC(), - Duration: time.Duration(2888) * time.Microsecond, - Annotations: []Annotation{ - Annotation{ - Timestamp: time.Unix(0, 1499817952283903000).UTC(), - Value: "cs", - Host: "0:0", - ServiceName: "cli", - }, - }, - BinaryAnnotations: []BinaryAnnotation{ - BinaryAnnotation{ - Key: "http.url", - Value: "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", - Host: "0:0", - ServiceName: "cli", - Type: "STRING", - }, - }, - }, - }, - }, - want: []testutil.Metric{ - testutil.Metric{ - Measurement: "zipkin", - Tags: map[string]string{ - "id": "3383422996321511664", - "parent_id": "4574092882326506380", - "trace_id": "8269862291023777619:243463817635710260", - "name": "Concat", - "service_name": "cli", - "annotation": "cs", - "endpoint_host": "0:0", - }, - Fields: map[string]interface{}{ - "duration_ns": (time.Duration(2888) * time.Microsecond).Nanoseconds(), - }, - Time: time.Unix(0, 1499817952283903000).UTC(), - }, - testutil.Metric{ - Measurement: "zipkin", - Tags: map[string]string{ - "trace_id": "2505404965370368069", - "service_name": "cli", - "annotation": "aHR0cDovL2xvY2FsaG9zdDo2MTAwMS9jb25jYXQv", - "annotation_key": "http.url", - "id": "22964302721410078", - "parent_id": "22964302721410078", - "name": "Concat", - "endpoint_host": "0:0", - }, - Fields: map[string]interface{}{ - "duration_ns": (time.Duration(2888) * time.Microsecond).Nanoseconds(), - }, - Time: time.Unix(0, 1499817952283903000).UTC(), - }, - }, - wantErr: false, - }, - */ //// Test data from distributed trace repo sample json // https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json { @@ -335,12 +267,13 @@ func TestLineProtocolConverter_Record(t *testing.T) { args: args{ t: Trace{ Span{ - ID: "6802735349851856000", - TraceID: "0:6802735349851856000", - Name: "main.dud", - ParentID: "6802735349851856000", - Timestamp: time.Unix(1, 0).UTC(), - Duration: 1, + ID: "6802735349851856000", + TraceID: "0:6802735349851856000", + Name: "main.dud", + ParentID: "6802735349851856000", + Timestamp: time.Unix(1, 0).UTC(), + Duration: 1, + ServiceName: "trivial", Annotations: []Annotation{ Annotation{ Timestamp: time.Unix(0, 1433330263415871000).UTC(), @@ -357,10 +290,11 @@ func TestLineProtocolConverter_Record(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "0:6802735349851856000", - "name": "main.dud", + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "0:6802735349851856000", + "name": "main.dud", + "service_name": "trivial", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(1) * time.Nanosecond).Nanoseconds(), diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 72cf218d86de8..10650075c5335 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -38,10 +38,11 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "8090652509916334619", - "parent_id": "22964302721410078", - "trace_id": "22c4fc8ab3669045", - "name": "Child", + "id": "8090652509916334619", + "parent_id": "22964302721410078", + "trace_id": "22c4fc8ab3669045", + "service_name": "trivial", + "name": "Child", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), @@ -68,10 +69,11 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "103618986556047333", - "parent_id": "22964302721410078", - "trace_id": "22c4fc8ab3669045", - "name": "Child", + "id": "103618986556047333", + "parent_id": "22964302721410078", + "trace_id": "22c4fc8ab3669045", + "service_name": "trivial", + "name": "Child", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), @@ -98,10 +100,11 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "22964302721410078", - "parent_id": "22964302721410078", - "trace_id": "22c4fc8ab3669045", - "name": "Parent", + "id": "22964302721410078", + "parent_id": "22964302721410078", + "trace_id": "22c4fc8ab3669045", + "service_name": "trivial", + "name": "Parent", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), @@ -186,10 +189,11 @@ func TestZipkinPlugin(t *testing.T) { testutil.Metric{ Measurement: "zipkin", Tags: map[string]string{ - "id": "6802735349851856000", - "parent_id": "6802735349851856000", - "trace_id": "5e682bc21ce99c80", - "name": "main.dud", + "id": "6802735349851856000", + "parent_id": "6802735349851856000", + "trace_id": "5e682bc21ce99c80", + "service_name": "go-zipkin-testclient", + "name": "main.dud", }, Fields: map[string]interface{}{ "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), @@ -262,8 +266,8 @@ func TestZipkinPlugin(t *testing.T) { got = append(got, *m) } - if !cmp.Equal(got, tt.want) { - t.Fatalf("Got != Want\n %s", cmp.Diff(got, tt.want)) + if !cmp.Equal(tt.want, got) { + t.Fatalf("Got != Want\n %s", cmp.Diff(tt.want, got)) } }) } From c37a1b04713c69dee453a055f194d1be0e6f67d3 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Tue, 1 Aug 2017 06:37:33 -0600 Subject: [PATCH 137/154] Update README in zipkin plugin --- plugins/inputs/zipkin/README.md | 95 +++++++++++++++++---------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index f2889d4c030df..4258e920498b4 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -1,58 +1,58 @@ # Zipkin Plugin -This plugin implements the Zipkin http server to gather trace and timing data needed to troubleshoot latency problems in microservice architectures. +*This plugin implements the Zipkin http server to gather trace and timing data needed to troubleshoot latency problems in microservice architectures.* -### Configuration: +## Configuration: ```toml [[inputs.zipkin]] path = "/api/v1/spans" # URL path for span data port = 9411 # Port on which Telegraf listens ``` -### Tracing: +## Tracing: This plugin uses Annotations tags and fields to track data from spans -- TRACE : is a set of spans that share a single root span. +- __TRACE:__ is a set of spans that share a single root span. Traces are built by collecting all Spans that share a traceId. -- SPAN : is a set of Annotations and BinaryAnnotations that correspond to a particular RPC. +- __SPAN:__ is a set of Annotations and BinaryAnnotations that correspond to a particular RPC. -- Annotations : for each annotation & binary annotation of a span a metric is output +- __Annotations:__ for each annotation & binary annotation of a span a metric is output -#### Annotations: records an occurrence in time at the beginning and end of a request - - CS (client start) : beginning of span, request is made. - - SR (server receive): server receives request and will start processing it - network latency & clock jitters differ it from cs - - SS (server send) : server is done processing and sends request back to client - amount of time it took to process request will differ it from sr - - CR (client receive): end of span, client receives response from server - RPC is considered complete with this annotation +### Annotations: +*Records an occurrence in time at the beginning and end of a request* + - __CS (client start):__ beginning of span, request is made. + - __SR (server receive):__ server receives request and will start processing it + network latency & clock jitters differ it from cs + - __SS (server send):__ server is done processing and sends request back to client + amount of time it took to process request will differ it from sr + - __CR (client receive):__ end of span, client receives response from server + RPC is considered complete with this annotation -- TAGS: +- __TAGS:__ * __"id":__ The 64 bit ID of the span. * __"parent_id":__ An ID associated with a particular child span. If there is no child span, the parent ID is set to ID. * __"trace_id":__ The 64 or 128-bit ID of a particular trace. Every span in a trace shares this ID. Concatenation of high and low and converted to hexadecimal. * __"name":__ Defines a span * __"service_name":__ Defines a service * __"annotation":__ The value of an annotation - * __"endpoint_host":__ Listening port concat with IPV4 + * __"endpoint_host":__ Listening port concat with IPV4, if port is not present it will not be concatenated --FIELDS - "duration_ns": The time in nanoseconds between the end and beginning of a span. -### BINARY ANNOTATIONS: +### Binary Annotations: --TAGS: Contains the same tags as annotations plus these additions +- __TAGS:__ Contains the same tags as annotations plus these additions - "annotation_key": label describing the annotation + * __"annotation_key":__ label describing the annotation --FIELDS: - "duration_ns": The time in nanoseconds between the end and beginning of a span. +- __FIELDS:__ *are the same for both annotations and binary annotations* + + * __"duration_ns":__ The time in nanoseconds between the end and beginning of a span. @@ -62,19 +62,19 @@ __Get All Span Names for Service__ `my_web_server` ```sql SHOW TAG VALUES FROM "zipkin" with key="name" WHERE "service_name" = 'my_web_server' ``` - - _Description:_ returns a list containing the names of the spans which have annotations with the given `service_name` of `my_web_server`. + - __Description:__ returns a list containing the names of the spans which have annotations with the given `service_name` of `my_web_server`. __Get All Service Names__ ```sql SHOW TAG VALUES FROM "zipkin" WITH KEY = "service_name" ``` - - _Description:_ returns a list of all `distinct` endpoint service names. + - __Description:__ returns a list of all `distinct` endpoint service names. __Find spans with longest duration__ ```sql SELECT max("duration_ns") FROM "zipkin" WHERE "service_name" = 'my_service' AND "name" = 'my_span_name' AND time > now() - 20m GROUP BY "trace_id",time(30s) LIMIT 5 ``` - - _Description:_ In the last 20 minutes find the top 5 longest span durations for service `my_server` and span name `my_span_name` + - __Description:__ In the last 20 minutes find the top 5 longest span durations for service `my_server` and span name `my_span_name` @@ -139,23 +139,26 @@ SELECT max("duration_ns") FROM "zipkin" WHERE "service_name" = 'my_service' AND We recomend using the [tsi influxDB engine](https://www.influxdata.com/path-1-billion-time-series-influxdb-high-cardinality-indexing-ready-testing/) as it can accept high cardinality data. #### How To Set Up InfluxDB For Work With Zipkin -##### Steps -___Update___ InfluxDB to >= 1.3, in order to use the new tsi engine. - -___Generate___ a config file with the following command: - `influxd config > /path/for/config/file` - -___Add___ the following to your config file, under the `[data]` tab: - -```toml -[data] - index-version = "tsi1" - ``` - - ___Start___ `influxd` with your new config file: - `$ influxd -config=/path/to/your/config/file` - -___Update___ your retention policy: -```sql -ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m -``` + ##### Steps + 1. ___Update___ InfluxDB to >= 1.3, in order to use the new tsi engine. + + 2. ___Generate___ a config file with the following command: + ```sql + `influxd config > /path/for/config/file` + ``` + 3. ___Add___ the following to your config file, under the `[data]` tab: + + ```toml + [data] + index-version = "tsi1" + ``` + + 4. ___Start___ `influxd` with your new config file: + ```sql + `$ influxd -config=/path/to/your/config/file` + ``` + + 5. ___Update___ your retention policy: + ```sql + ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m + ``` From 0c33bbeb50a2cc49c0ed4bf9289d5cde4c56a11f Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Tue, 1 Aug 2017 12:19:12 -0600 Subject: [PATCH 138/154] Update README in zipkin plugin --- plugins/inputs/zipkin/README.md | 122 ++++++++++++++++---------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index 4258e920498b4..701cac8e21440 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -84,55 +84,56 @@ SELECT max("duration_ns") FROM "zipkin" WHERE "service_name" = 'my_service' AND - [Test data from distributed trace repo sample json](https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json) #### Trace Example -```{ - "traceId": "bd7a977555f6b982", - "name": "query", - "id": "be2d01e33cc78d97", - "parentId": "ebf33e1a81dc6f71", +```json +{ + "traceId": "bd7a977555f6b982", + "name": "query", + "id": "be2d01e33cc78d97", + "parentId": "ebf33e1a81dc6f71", + "timestamp": 1458702548786000, + "duration": 13000, + "annotations": [ + { + "endpoint": { + "serviceName": "zipkin-query", + "ipv4": "192.168.1.2", + "port": 9411 + }, "timestamp": 1458702548786000, - "duration": 13000, - "annotations": [ - { - "endpoint": { - "serviceName": "zipkin-query", - "ipv4": "192.168.1.2", - "port": 9411 - }, - "timestamp": 1458702548786000, - "value": "cs" - }, - { - "endpoint": { - "serviceName": "zipkin-query", - "ipv4": "192.168.1.2", - "port": 9411 - }, - "timestamp": 1458702548799000, - "value": "cr" - } - ], - "binaryAnnotations": [ - { - "key": "jdbc.query", - "value": "select distinct `zipkin_spans`.`trace_id` from `zipkin_spans` join `zipkin_annotations` on (`zipkin_spans`.`trace_id` = `zipkin_annotations`.`trace_id` and `zipkin_spans`.`id` = `zipkin_annotations`.`span_id`) where (`zipkin_annotations`.`endpoint_service_name` = ? and `zipkin_spans`.`start_ts` between ? and ?) order by `zipkin_spans`.`start_ts` desc limit ?", - "endpoint": { - "serviceName": "zipkin-query", - "ipv4": "192.168.1.2", - "port": 9411 - } - }, - { - "key": "sa", - "value": true, - "endpoint": { - "serviceName": "spanstore-jdbc", - "ipv4": "127.0.0.1", - "port": 3306 - } - } - ] + "value": "cs" }, - ``` + { + "endpoint": { + "serviceName": "zipkin-query", + "ipv4": "192.168.1.2", + "port": 9411 + }, + "timestamp": 1458702548799000, + "value": "cr" + } + ], + "binaryAnnotations": [ + { + "key": "jdbc.query", + "value": "select distinct `zipkin_spans`.`trace_id` from `zipkin_spans` join `zipkin_annotations` on (`zipkin_spans`.`trace_id` = `zipkin_annotations`.`trace_id` and `zipkin_spans`.`id` = `zipkin_annotations`.`span_id`) where (`zipkin_annotations`.`endpoint_service_name` = ? and `zipkin_spans`.`start_ts` between ? and ?) order by `zipkin_spans`.`start_ts` desc limit ?", + "endpoint": { + "serviceName": "zipkin-query", + "ipv4": "192.168.1.2", + "port": 9411 + } + }, + { + "key": "sa", + "value": true, + "endpoint": { + "serviceName": "spanstore-jdbc", + "ipv4": "127.0.0.1", + "port": 3306 + } + } + ] +} +``` ### Recommended installation @@ -143,22 +144,21 @@ We recomend using the [tsi influxDB engine](https://www.influxdata.com/path-1-bi 1. ___Update___ InfluxDB to >= 1.3, in order to use the new tsi engine. 2. ___Generate___ a config file with the following command: - ```sql - `influxd config > /path/for/config/file` - ``` +```sh +influxd config > /path/for/config/file +``` 3. ___Add___ the following to your config file, under the `[data]` tab: - - ```toml - [data] - index-version = "tsi1" - ``` +```toml +[data] + index-version = "tsi1" +``` 4. ___Start___ `influxd` with your new config file: - ```sql - `$ influxd -config=/path/to/your/config/file` - ``` +```sh +influxd -config=/path/to/your/config/file +``` 5. ___Update___ your retention policy: - ```sql - ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m - ``` +```sql +ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m +``` From 59634af5129a1f6ac386ce1893c7d59ebc922ed0 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Tue, 1 Aug 2017 12:30:42 -0600 Subject: [PATCH 139/154] Update README in zipkin plugin --- plugins/inputs/zipkin/README.md | 51 +++++++++++++++++---------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index 701cac8e21440..23e618ef86dce 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -20,38 +20,39 @@ Traces are built by collecting all Spans that share a traceId. - __SPAN:__ is a set of Annotations and BinaryAnnotations that correspond to a particular RPC. -- __Annotations:__ for each annotation & binary annotation of a span a metric is output - - -### Annotations: -*Records an occurrence in time at the beginning and end of a request* - - __CS (client start):__ beginning of span, request is made. - - __SR (server receive):__ server receives request and will start processing it - network latency & clock jitters differ it from cs - - __SS (server send):__ server is done processing and sends request back to client - amount of time it took to process request will differ it from sr - - __CR (client receive):__ end of span, client receives response from server - RPC is considered complete with this annotation - -- __TAGS:__ - * __"id":__ The 64 bit ID of the span. - * __"parent_id":__ An ID associated with a particular child span. If there is no child span, the parent ID is set to ID. - * __"trace_id":__ The 64 or 128-bit ID of a particular trace. Every span in a trace shares this ID. Concatenation of high and low and converted to hexadecimal. - * __"name":__ Defines a span - * __"service_name":__ Defines a service - * __"annotation":__ The value of an annotation - * __"endpoint_host":__ Listening port concat with IPV4, if port is not present it will not be concatenated +- __Annotations:__ for each annotation & binary annotation of a span a metric is output. *Records an occurrence in time at the beginning and end of a request.* + Annotations may have the following values: -### Binary Annotations: + - __CS (client start):__ beginning of span, request is made. + - __SR (server receive):__ server receives request and will start processing it + network latency & clock jitters differ it from cs + - __SS (server send):__ server is done processing and sends request back to client + amount of time it took to process request will differ it from sr + - __CR (client receive):__ end of span, client receives response from server + RPC is considered complete with this annotation -- __TAGS:__ Contains the same tags as annotations plus these additions +### Tags +* __"id":__ The 64 bit ID of the span. +* __"parent_id":__ An ID associated with a particular child span. If there is no child span, the parent ID is set to ID. +* __"trace_id":__ The 64 or 128-bit ID of a particular trace. Every span in a trace shares this ID. Concatenation of high and low and converted to hexadecimal. +* __"name":__ Defines a span - * __"annotation_key":__ label describing the annotation +##### Annotations have these additional tags: + +* __"service_name":__ Defines a service +* __"annotation":__ The value of an annotation +* __"endpoint_host":__ Listening port concat with IPV4, if port is not present it will not be concatenated +##### Binary Annotations have these additional tag: + + * __"service_name":__ Defines a service + * __"annotation":__ The value of an annotation + * __"endpoint_host":__ Listening port concat with IPV4, if port is not present it will not be concatenated + * __"annotation_key":__ label describing the annotation -- __FIELDS:__ *are the same for both annotations and binary annotations* +### Fields: * __"duration_ns":__ The time in nanoseconds between the end and beginning of a span. From 131e1c38b1518d9567ae9b59cdd8d32910f7c068 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Tue, 1 Aug 2017 15:52:13 -0600 Subject: [PATCH 140/154] Add link to explain zipkin code --- plugins/inputs/zipkin/convert.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 3956793211cd0..900d1153fcb32 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -210,6 +210,7 @@ func host(h *zipkincore.Endpoint) string { // Zipkin uses a signed int16 for the port, but, warns us that they actually treat it // as an unsigned int16. So, we convert from int16 to int32 followed by taking & 0xffff // to convert from signed to unsigned + // https://github.com/openzipkin/zipkin/blob/57dc2ec9c65fe6144e401c0c933b4400463a69df/zipkin/src/main/java/zipkin/Endpoint.java#L44 return ipv4(h.GetIpv4()) + ":" + strconv.FormatInt(int64(int(h.GetPort())&0xffff), 10) } From befbe7edb15aebf09b0a5a0f686c05e33ddae891 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Tue, 1 Aug 2017 16:22:03 -0600 Subject: [PATCH 141/154] Add better description of trace example in README --- plugins/inputs/zipkin/README.md | 60 ++++++++++++++++----------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index 23e618ef86dce..798bc6732ce62 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -1,6 +1,6 @@ # Zipkin Plugin -*This plugin implements the Zipkin http server to gather trace and timing data needed to troubleshoot latency problems in microservice architectures.* +This plugin implements the Zipkin http server to gather trace and timing data needed to troubleshoot latency problems in microservice architectures. @@ -78,13 +78,39 @@ SELECT max("duration_ns") FROM "zipkin" WHERE "service_name" = 'my_service' AND - __Description:__ In the last 20 minutes find the top 5 longest span durations for service `my_server` and span name `my_span_name` +### Recommended InfluxDB setup + +This test will create high cardinality data so we reccomend using the [tsi influxDB engine](https://www.influxdata.com/path-1-billion-time-series-influxdb-high-cardinality-indexing-ready-testing/). +#### How To Set Up InfluxDB For Work With Zipkin + + ##### Steps + 1. ___Update___ InfluxDB to >= 1.3, in order to use the new tsi engine. + + 2. ___Generate___ a config file with the following command: +```sh +influxd config > /path/for/config/file +``` + 3. ___Add___ the following to your config file, under the `[data]` tab: +```toml +[data] + index-version = "tsi1" +``` + + 4. ___Start___ `influxd` with your new config file: +```sh +influxd -config=/path/to/your/config/file +``` + + 5. ___Update___ your retention policy: +```sql +ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m +``` ### Example Input Trace: - [Cli microservice with two services Test](https://github.com/openzipkin/zipkin-go-opentracing/tree/master/examples/cli_with_2_services) - [Test data from distributed trace repo sample json](https://github.com/mattkanwisher/distributedtrace/blob/master/testclient/sample.json) - -#### Trace Example +#### [Trace Example from Zipkin model](http://zipkin.io/pages/data_model.html) ```json { "traceId": "bd7a977555f6b982", @@ -135,31 +161,3 @@ SELECT max("duration_ns") FROM "zipkin" WHERE "service_name" = 'my_service' AND ] } ``` - -### Recommended installation - -We recomend using the [tsi influxDB engine](https://www.influxdata.com/path-1-billion-time-series-influxdb-high-cardinality-indexing-ready-testing/) as it can accept high cardinality data. -#### How To Set Up InfluxDB For Work With Zipkin - - ##### Steps - 1. ___Update___ InfluxDB to >= 1.3, in order to use the new tsi engine. - - 2. ___Generate___ a config file with the following command: -```sh -influxd config > /path/for/config/file -``` - 3. ___Add___ the following to your config file, under the `[data]` tab: -```toml -[data] - index-version = "tsi1" -``` - - 4. ___Start___ `influxd` with your new config file: -```sh -influxd -config=/path/to/your/config/file -``` - - 5. ___Update___ your retention policy: -```sql -ALTER RETENTION POLICY "autogen" ON "telegraf" DURATION 1d SHARD DURATION 30m -``` From 01cfdbc940133c1dec16a2f8121e237e9ebf5dbf Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 15:57:39 -0700 Subject: [PATCH 142/154] Refactor thrift_serialize.go into development tool --- .../cmd/thrift_serialize/thrift_serialize.go | 134 ++++++++++++------ 1 file changed, 88 insertions(+), 46 deletions(-) diff --git a/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go b/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go index 15ac2ecbc2986..5a65384d4aafb 100644 --- a/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go +++ b/plugins/inputs/zipkin/cmd/thrift_serialize/thrift_serialize.go @@ -1,7 +1,32 @@ +/* +A small cli utility meant to convert json to zipkin thrift binary format, and +vice versa. + +To convert from json to thrift, +the json is unmarshalled, converted to zipkincore.Span structures, and +marshalled into thrift binary protocol. The json must be in an array format (even if it only has one object), +because the tool automatically tries to unmarshall the json into an array of structs. + +To convert from thrift to json, +the opposite process must happen. The thrift binary data must be read into an array of +zipkin span structures, and those spans must be marshalled into json. + +Usage: + +./thrift_serialize -input -output -deserialize + +If `deserialize` is set to true (false by default), the tool will interpret the input file as +thrift, and write it as json to the output file. +Otherwise, the input file will be interpreted as json, and the output will be encoded as thrift. + + +*/ package main import ( "encoding/json" + "errors" + "flag" "fmt" "io/ioutil" "log" @@ -11,30 +36,64 @@ import ( ) var ( - filename *string - outFileName *string + filename string + outFileName string + inputType string ) -/*func init() { - filename = flag.String("input", "", usage) - outFileName = flag.String("output", "", usage) -} +const usage = `./json_serialize -input -output output -input-type` -const usage = `./json_serialize -input -output output`*/ +func init() { + flag.StringVar(&filename, "input", "", usage) + flag.StringVar(&outFileName, "output", "", usage) + flag.StringVar(&inputType, "input-type", "thrift", usage) +} func main() { - b, err := ioutil.ReadFile("../testdata/json/distributed_trace_sample.json") + flag.Parse() + contents, err := ioutil.ReadFile(filename) if err != nil { - log.Fatalf("error: %v\n", err) + log.Fatalf("Error reading file: %v\n", err) + } + + switch inputType { + case "json": + raw, err := jsonToZipkinThrift(contents) + if err != nil { + log.Fatalf("%v\n", err) + } + ioutil.WriteFile(outFileName, raw, 0644) + case "thrift": + raw, err := thriftToJSONSpans(contents) + if err != nil { + log.Fatalf("%v\n", err) + } + ioutil.WriteFile(outFileName, raw, 0644) + default: + log.Fatalf("Unsupported input type") + } +} + +func jsonToZipkinThrift(jsonRaw []byte) ([]byte, error) { + if len(jsonRaw) == 0 { + return nil, errors.New("no data") + } + + if string(jsonRaw)[0] != '[' { + return nil, errors.New("cannot unmarshal non array type") } var spans []*zipkincore.Span - span, err := serializeJSON(b) - spans = append(spans, span) + err := json.Unmarshal(jsonRaw, &spans) + if err != nil { + return nil, fmt.Errorf("error unmarshalling: %v", err) + } + var zspans []*zipkincore.Span if err != nil { - log.Fatalf("error: %v\n", err) + return nil, err } + zspans = append(zspans, spans...) fmt.Println(spans) @@ -42,70 +101,53 @@ func main() { transport := thrift.NewTBinaryProtocolTransport(buf) if err = transport.WriteListBegin(thrift.STRUCT, len(spans)); err != nil { - panic(err) + return nil, fmt.Errorf("error in beginning thrift write: %v", err) } - for _, s := range spans { - err = s.Write(transport) + for _, span := range zspans { + err = span.Write(transport) if err != nil { - panic(err) + return nil, fmt.Errorf("error converting zipkin struct to thrift: %v", err) } } if err = transport.WriteListEnd(); err != nil { - panic(err) + return nil, fmt.Errorf("error finishing thrift write: %v", err) } - log.Println(buf.Buffer.String()) - - ioutil.WriteFile("../testdata/out.dat", buf.Buffer.Bytes(), 0644) - - b, err = ioutil.ReadFile("../testdata/out.dat") - - //log.Println("read bytes: ", b) - if err != nil { - log.Printf("%v\n", err) - } - deserializeThrift(b) -} - -func serializeJSON(b []byte) (*zipkincore.Span, error) { - var span *zipkincore.Span - err := json.Unmarshal(b, &span) - return span, err + return buf.Bytes(), nil } -func deserializeThrift(b []byte) { +func thriftToJSONSpans(thriftData []byte) ([]byte, error) { buffer := thrift.NewTMemoryBuffer() - if _, err := buffer.Write(b); err != nil { - log.Println("Error in buffer write: ", err) - return + if _, err := buffer.Write(thriftData); err != nil { + err = fmt.Errorf("error in buffer write: %v", err) + return nil, err } transport := thrift.NewTBinaryProtocolTransport(buffer) _, size, err := transport.ReadListBegin() if err != nil { - log.Printf("Error in ReadListBegin: %s", err) - return + err = fmt.Errorf("error in ReadListBegin: %v", err) + return nil, err } var spans []*zipkincore.Span for i := 0; i < size; i++ { zs := &zipkincore.Span{} if err = zs.Read(transport); err != nil { - log.Printf("Error reading into zipkin struct: %s", err) - return + err = fmt.Errorf("Error reading into zipkin struct: %v", err) + return nil, err } spans = append(spans, zs) } err = transport.ReadListEnd() if err != nil { - log.Printf("%s", err) - return + err = fmt.Errorf("error ending thrift read: %v", err) + return nil, err } - //marshal json for debugging purposes out, _ := json.MarshalIndent(spans, "", " ") - log.Println(string(out)) + return out, nil } From db8ac9a9ef7f6d1620f9b4ec970de603771f86f5 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 16:00:06 -0700 Subject: [PATCH 143/154] Change testdata/json/distributed_trace_sample.json to array json format --- .../inputs/zipkin/testdata/json/distributed_trace_sample.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/testdata/json/distributed_trace_sample.json b/plugins/inputs/zipkin/testdata/json/distributed_trace_sample.json index a92856e060d19..b41eaccf4a84a 100644 --- a/plugins/inputs/zipkin/testdata/json/distributed_trace_sample.json +++ b/plugins/inputs/zipkin/testdata/json/distributed_trace_sample.json @@ -1,4 +1,4 @@ -{ +[{ "trace_id": 6802735349851856000, "name": "main.dud", "id": 6802735349851856000, @@ -27,4 +27,4 @@ ], "binary_annotations": [], "debug": true -} +}] From 08698d9bd1f8bd7487f312f54103b7eb4d20bb0c Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 17:07:39 -0700 Subject: [PATCH 144/154] Move server initialization logic into start method of zipkin plugin; Ensure no error on clean shutdown --- plugins/inputs/zipkin/zipkin.go | 36 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 4c9fba31aaad9..aaa1f685031a2 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -114,13 +114,21 @@ func (z *Zipkin) Gather(acc telegraf.Accumulator) error { return nil } // passing in a telegraf.Accumulator such that data can be collected. func (z *Zipkin) Start(acc telegraf.Accumulator) error { log.Println("starting zipkin plugin...") - if z.handler == nil { - z.handler = NewSpanHandler(z.Path) - } + + z.handler = NewSpanHandler(z.Path) var wg sync.WaitGroup z.waitGroup = &wg + router := mux.NewRouter() + converter := NewLineProtocolConverter(acc) + z.handler.Register(router, converter) + + z.server = &http.Server{ + Addr: ":" + strconv.Itoa(z.Port), + Handler: router, + } + go func() { wg.Add(1) defer wg.Done() @@ -134,27 +142,25 @@ func (z *Zipkin) Start(acc telegraf.Accumulator) error { // Stop shuts the internal http server down with via context.Context func (z *Zipkin) Stop() { ctx, cancel := context.WithTimeout(context.Background(), DefaultShutdownTimeout) - defer cancel() defer z.waitGroup.Wait() + defer cancel() + z.server.Shutdown(ctx) } // Listen creates an http server on the zipkin instance it is called with, and // serves http until it is stopped by Zipkin's (*Zipkin).Stop() method. func (z *Zipkin) Listen(acc telegraf.Accumulator) { - router := mux.NewRouter() - converter := NewLineProtocolConverter(acc) - z.handler.Register(router, converter) - - if z.server == nil { - z.server = &http.Server{ - Addr: ":" + strconv.Itoa(z.Port), - Handler: router, - } - } if err := z.server.ListenAndServe(); err != nil { - acc.AddError(fmt.Errorf("E! Error listening: %v", err)) + // Because of the clean shutdown in `(*Zipkin).Stop()` + // We're expecting a server closed error at some point + // So we don't want to display it as an error. + // This interferes with telegraf's internal data collection, + // by making it appear as if a serious error occurred. + if err != http.ErrServerClosed { + acc.AddError(fmt.Errorf("E! Error listening: %v", err)) + } } } From c33bb4d549c766ca6e8649b88a24cf1b312461a5 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 17:14:02 -0700 Subject: [PATCH 145/154] Add test case for clean shutdown error to zipkin plugin --- plugins/inputs/zipkin/zipkin_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 10650075c5335..3b071c12d5af0 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -271,6 +271,12 @@ func TestZipkinPlugin(t *testing.T) { } }) } + mockAcc.ClearMetrics() + z.Stop() + // Make sure there is no erroneous error on shutdown + if len(mockAcc.Errors) != 0 { + t.Fatal("Expected no errors on shutdown") + } } func postThriftData(datafile string) error { From c03e54d99d956225befbb193fc5ae67422681b5e Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 17:26:29 -0700 Subject: [PATCH 146/154] Change default hostname value when no endpoint is provided in zipkin plugin Meant to prevent an empty string from being added as part of a metric --- plugins/inputs/zipkin/convert.go | 2 +- plugins/inputs/zipkin/convert_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 3956793211cd0..d75dc56914594 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -202,7 +202,7 @@ func ipv4(addr int32) string { func host(h *zipkincore.Endpoint) string { if h == nil { - return "" + return ipv4(int32(0)) } if h.GetPort() == 0 { return ipv4(h.GetIpv4()) diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index cc7fee4305c2d..1e801c6e080e8 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -486,7 +486,7 @@ func Test_host(t *testing.T) { args: args{ h: nil, }, - want: "", + want: "0.0.0.0", }, { name: "int overflow zipkin uses an int16 type as an unsigned int 16.", From 7f991fd840dec354dc3a209ba0c42f35766251ad Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 17:42:55 -0700 Subject: [PATCH 147/154] Change default service name when no endpoint is provided Empty strings should not be added to telegraf's internal metrics --- plugins/inputs/zipkin/convert.go | 4 +++- plugins/inputs/zipkin/convert_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 3956793211cd0..154c00d19f1cb 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -11,6 +11,8 @@ import ( "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) +const DefaultServiceName = "unknown" + //now is a moackable time for now var now = time.Now @@ -215,7 +217,7 @@ func host(h *zipkincore.Endpoint) string { func serviceName(h *zipkincore.Endpoint) string { if h == nil { - return "" + return DefaultServiceName } return h.GetServiceName() } diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index cc7fee4305c2d..4750f87cc8b74 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -531,7 +531,7 @@ func Test_serviceName(t *testing.T) { args: args{ h: nil, }, - want: "", + want: "unknown", }, } for _, tt := range tests { From 19ba03ef0899cc1b10a304aeee98a7dbdceea726 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 17:55:10 -0700 Subject: [PATCH 148/154] Add explanatory comment to cmd/stress_test_write/stress_test_write.go --- .../cmd/stress_test_write/stress_test_write.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go b/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go index fdd14c651f458..f4bc134f91607 100644 --- a/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go +++ b/plugins/inputs/zipkin/cmd/stress_test_write/stress_test_write.go @@ -1,3 +1,21 @@ +/* +This is a development testing cli tool meant to stress the zipkin telegraf plugin. +It writes a specified number of zipkin spans to the plugin endpoint, with other +parameters which dictate batch size and flush timeout. + +Usage as follows: + +`./stress_test_write -batch_size= -max_backlog= -batch_interval= -span_count -zipkin_host=` + +Or with a timer: + +`time ./stress_test_write -batch_size= -max_backlog= -batch_interval= -span_count -zipkin_host=` + +However, the flag defaults work just fine for a good write stress test (and are what +this tool has mainly been tested with), so there shouldn't be much need to +manually tweak the parameters. +*/ + package main import ( From 17e2b6bd47b76c4da3ee813496488753e8f476ee Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 18:00:32 -0700 Subject: [PATCH 149/154] Remove unused struct from zipkin integration table test --- plugins/inputs/zipkin/zipkin_test.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 10650075c5335..f288a438f4f65 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -9,30 +9,24 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" ) func TestZipkinPlugin(t *testing.T) { mockAcc := testutil.Accumulator{} - type fields struct { - acc telegraf.Accumulator - } + type args struct { t Trace } + tests := []struct { name string - fields fields thriftDataFile string //path name to a binary thrift data file which contains test data wantErr bool want []testutil.Metric }{ { - name: "threespan", - fields: fields{ - acc: &mockAcc, - }, + name: "threespan", thriftDataFile: "testdata/threespans.dat", want: []testutil.Metric{ testutil.Metric{ @@ -180,10 +174,7 @@ func TestZipkinPlugin(t *testing.T) { wantErr: false, }, { - name: "distributed_trace_sample", - fields: fields{ - acc: &mockAcc, - }, + name: "distributed_trace_sample", thriftDataFile: "testdata/distributed_trace_sample.dat", want: []testutil.Metric{ testutil.Metric{ From b1ec4bd1334d2e0b45fcd68d6aeccb2215d50930 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 1 Aug 2017 18:34:36 -0700 Subject: [PATCH 150/154] Update LICENSE_OF_DEPENDENCIES.md for zipkin plugin --- docs/LICENSE_OF_DEPENDENCIES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index da031220100be..575522222a4c8 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -25,9 +25,11 @@ works: - github.com/eclipse/paho.mqtt.golang [ECLIPSE](https://github.com/eclipse/paho.mqtt.golang/blob/master/LICENSE) - github.com/fsouza/go-dockerclient [BSD](https://github.com/fsouza/go-dockerclient/blob/master/LICENSE) - github.com/gobwas/glob [MIT](https://github.com/gobwas/glob/blob/master/LICENSE) +- github.com/google/go-cmp [BSD](https://github.com/google/go-cmp/blob/master/LICENSE) - github.com/gogo/protobuf [BSD](https://github.com/gogo/protobuf/blob/master/LICENSE) - github.com/golang/protobuf [BSD](https://github.com/golang/protobuf/blob/master/LICENSE) - github.com/golang/snappy [BSD](https://github.com/golang/snappy/blob/master/LICENSE) +- github.com/go-logfmt/logfmt [MIT](https://github.com/go-logfmt/logfmt/blob/master/LICENSE) - github.com/gorilla/mux [BSD](https://github.com/gorilla/mux/blob/master/LICENSE) - github.com/go-sql-driver/mysql [MPL](https://github.com/go-sql-driver/mysql/blob/master/LICENSE) - github.com/hailocab/go-hostpool [MIT](https://github.com/hailocab/go-hostpool/blob/master/LICENSE) @@ -51,6 +53,9 @@ works: - github.com/nats-io/nats [MIT](https://github.com/nats-io/nats/blob/master/LICENSE) - github.com/nats-io/nuid [MIT](https://github.com/nats-io/nuid/blob/master/LICENSE) - github.com/nsqio/go-nsq [MIT](https://github.com/nsqio/go-nsq/blob/master/LICENSE) +- github.com/opentracing-contrib/go-observer [APACHE](https://github.com/opentracing-contrib/go-observer/blob/master/LICENSE) +- github.com/opentracing/opentracing-go [MIT](https://github.com/opentracing/opentracing-go/blob/master/LICENSE) +- github.com/openzipkin/zipkin-go-opentracing [MIT](https://github.com/openzipkin/zipkin-go-opentracing/blob/master/LICENSE) - github.com/pierrec/lz4 [BSD](https://github.com/pierrec/lz4/blob/master/LICENSE) - github.com/pierrec/xxHash [BSD](https://github.com/pierrec/xxHash/blob/master/LICENSE) - github.com/pkg/errors [BSD](https://github.com/pkg/errors/blob/master/LICENSE) From eb0f8cc7227a82022c128f97ec4bf5fd45962658 Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 2 Aug 2017 12:00:46 -0600 Subject: [PATCH 151/154] Remove unused log statement and notes file --- plugins/inputs/zipkin/notes | 39 ---------------------------- plugins/inputs/zipkin/zipkin.go | 3 --- plugins/inputs/zipkin/zipkin_test.go | 4 +-- 3 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 plugins/inputs/zipkin/notes diff --git a/plugins/inputs/zipkin/notes b/plugins/inputs/zipkin/notes deleted file mode 100644 index d8fd1af4efac1..0000000000000 --- a/plugins/inputs/zipkin/notes +++ /dev/null @@ -1,39 +0,0 @@ -recommendations: -1. batch size should be large (e.g. 1000) - - -profiling: -telegraf to file 200000 points (100k spans) in 3 seconds - -telegraf to influx not able to finish so far (buffering?) - -Empirical results of default indexing vs. tsi - -Write: -Default: finished in 11.278 seconds (2.7 user) -TSI: Didn't finish -TSI(master): 1:37 (9.3 user) - -select * from zipkin where "id"='trace_id' -Default: instantaneous -TSI: No data -TSI(master): - -select count("duration") from zipkin -Default: ~1s -TSI: 30s - -show tag values with key="service_name": -Default: -nearly instantaneous -TSI: Significant delay - -show tag values with key="name" where "service_name"='trivial': -Default: -Nearly instantaneous -TSI: -Significant Delay - -select count("duration") from zipkin where time > now() - 30m and time < now() group by "id","parent_id", time(1d): -Default: ~10 seconds -TSI: Didn't finish diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 4c9fba31aaad9..8e04852d3ad4d 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -3,7 +3,6 @@ package zipkin import ( "context" "fmt" - "log" "net/http" "strconv" "sync" @@ -79,7 +78,6 @@ type Span struct { type Trace []Span const sampleConfig = ` - ## # path = "/api/v1/spans" # URL path for span data # port = 9411 # Port on which Telegraf listens ` @@ -113,7 +111,6 @@ func (z *Zipkin) Gather(acc telegraf.Accumulator) error { return nil } // Start launches a separate goroutine for collecting zipkin client http requests, // passing in a telegraf.Accumulator such that data can be collected. func (z *Zipkin) Start(acc telegraf.Accumulator) error { - log.Println("starting zipkin plugin...") if z.handler == nil { z.handler = NewSpanHandler(z.Path) } diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 10650075c5335..3031f902f8188 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -18,9 +18,7 @@ func TestZipkinPlugin(t *testing.T) { type fields struct { acc telegraf.Accumulator } - type args struct { - t Trace - } + tests := []struct { name string fields fields From 079b8b791177691c076c3cefeb2c3866d602932f Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 2 Aug 2017 15:15:31 -0600 Subject: [PATCH 152/154] Fix linter error in zipkin plugin --- plugins/inputs/zipkin/convert.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/zipkin/convert.go b/plugins/inputs/zipkin/convert.go index 6ac6348d146f0..dad099771d91c 100644 --- a/plugins/inputs/zipkin/convert.go +++ b/plugins/inputs/zipkin/convert.go @@ -11,6 +11,7 @@ import ( "github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore" ) +// DefaultServiceName when the span does not have any serviceName const DefaultServiceName = "unknown" //now is a moackable time for now From 0c23e19d7109e4cc03af621b24c7b4b9cb25a61a Mon Sep 17 00:00:00 2001 From: Ayrdrie Palmer Date: Wed, 2 Aug 2017 15:58:56 -0600 Subject: [PATCH 153/154] Update zipkin integration test so that a random port is chosen --- plugins/inputs/zipkin/zipkin.go | 26 +++++++++++++++++++------- plugins/inputs/zipkin/zipkin_test.go | 10 +++++----- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index f6510b8db68cd..597ae0b270074 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -3,6 +3,8 @@ package zipkin import ( "context" "fmt" + "log" + "net" "net/http" "strconv" "sync" @@ -89,9 +91,11 @@ type Zipkin struct { ServiceAddress string Port int Path string - handler Handler - server *http.Server - waitGroup *sync.WaitGroup + + address string + handler Handler + server *http.Server + waitGroup *sync.WaitGroup } // Description is a necessary method implementation from telegraf.ServiceInput @@ -121,15 +125,23 @@ func (z *Zipkin) Start(acc telegraf.Accumulator) error { z.handler.Register(router, converter) z.server = &http.Server{ - Addr: ":" + strconv.Itoa(z.Port), Handler: router, } + addr := ":" + strconv.Itoa(z.Port) + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + + z.address = ln.Addr().String() + log.Printf("I! Started the zipkin listener on %s", z.address) + go func() { wg.Add(1) defer wg.Done() - z.Listen(acc) + z.Listen(ln, acc) }() return nil @@ -147,8 +159,8 @@ func (z *Zipkin) Stop() { // Listen creates an http server on the zipkin instance it is called with, and // serves http until it is stopped by Zipkin's (*Zipkin).Stop() method. -func (z *Zipkin) Listen(acc telegraf.Accumulator) { - if err := z.server.ListenAndServe(); err != nil { +func (z *Zipkin) Listen(ln net.Listener, acc telegraf.Accumulator) { + if err := z.server.Serve(ln); err != nil { // Because of the clean shutdown in `(*Zipkin).Stop()` // We're expecting a server closed error at some point // So we don't want to display it as an error. diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index fd95b7939d358..9447e67d4a8b9 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -227,7 +227,7 @@ func TestZipkinPlugin(t *testing.T) { z := &Zipkin{ Path: "/api/v1/spans", - Port: 9411, + Port: 0, } err := z.Start(&mockAcc) @@ -240,7 +240,7 @@ func TestZipkinPlugin(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockAcc.ClearMetrics() - if err := postThriftData(tt.thriftDataFile); err != nil { + if err := postThriftData(tt.thriftDataFile, z.address); err != nil { t.Fatalf("Posting data to http endpoint /api/v1/spans failed. Error: %s\n", err) } mockAcc.Wait(len(tt.want)) //Since the server is running concurrently, we need to wait for the number of data points we want to test to be added to the Accumulator. @@ -266,13 +266,13 @@ func TestZipkinPlugin(t *testing.T) { } } -func postThriftData(datafile string) error { +func postThriftData(datafile, address string) error { dat, err := ioutil.ReadFile(datafile) if err != nil { return fmt.Errorf("could not read from data file %s", datafile) } - req, err := http.NewRequest("POST", "http://localhost:9411/api/v1/spans", bytes.NewReader(dat)) + req, err := http.NewRequest("POST", fmt.Sprintf("http://%s/api/v1/spans", address), bytes.NewReader(dat)) if err != nil { return fmt.Errorf("HTTP request creation failed") @@ -282,7 +282,7 @@ func postThriftData(datafile string) error { client := &http.Client{} _, err = client.Do(req) if err != nil { - return fmt.Errorf("HTTP POST request to zipkin endpoint http://localhost:/9411/v1/spans failed") + return fmt.Errorf("HTTP POST request to zipkin endpoint %s failed %v", address, err) } return nil From d9e40575053102e260a85c94628fd9159814abc0 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 2 Aug 2017 15:01:27 -0700 Subject: [PATCH 154/154] Add experimental disclaimer to README.md --- plugins/inputs/zipkin/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zipkin/README.md b/plugins/inputs/zipkin/README.md index 798bc6732ce62..af5ea6117e008 100644 --- a/plugins/inputs/zipkin/README.md +++ b/plugins/inputs/zipkin/README.md @@ -2,7 +2,8 @@ This plugin implements the Zipkin http server to gather trace and timing data needed to troubleshoot latency problems in microservice architectures. - +*Please Note: This plugin is experimental; Its data schema may be subject to change +based on its main usage cases and the evolution of the OpenTracing standard.* ## Configuration: ```toml