Skip to content

Commit

Permalink
[Netflow] Fix flow ID and locality calculation (elastic#9891) (elasti…
Browse files Browse the repository at this point in the history 
…c#9904)

Flow ID, Community ID and locality calculation was broken in the
NetFlow input. 0.0.0.0 was always used as destination IP.

This caused:
- The wrong flow-id and community-id to be generated
- Different ids for the reverse-flow.
- The wrong locality being assigned to the flow.

(cherry picked from commit 511e74a)
  • Loading branch information
@adriansr
adriansr authored Jan 4, 2019
1 parent 4f414d4 commit 7c6f87b
Show file tree
Hide file tree
Showing 37 changed files with 857 additions and 774 deletions.
24 changes: 12 additions & 12 deletions x-pack/filebeat/input/netflow/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,18 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
ecsSource["mac"] = mac
}

// ECS Fields -- destination
if ip, found := getKeyIP(flow.Fields, "destinationIPv4Address"); found {
ecsDest["ip"] = ip
ecsDest["locality"] = getIPLocality(ip).String()
}
if destPort, found := getKeyUint64(flow.Fields, "destinationTransportPort"); found {
ecsDest["port"] = destPort
}
if mac, found := getKeyString(flow.Fields, "destinationMacAddress"); found {
ecsDest["mac"] = mac
}

// ECS Fields -- Flow
ecsFlow := common.MapStr{}
var srcIP, dstIP net.IP
Expand Down Expand Up @@ -227,18 +239,6 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
ecsFlow["id"] = flowID(srcIP, dstIP, srcPort, dstPort, uint8(protocol))
ecsFlow["locality"] = getIPLocality(srcIP, dstIP).String()

// ECS Fields -- destination
if ip, found := getKeyIP(flow.Fields, "destinationIPv4Address"); found {
ecsDest["ip"] = ip
ecsDest["locality"] = getIPLocality(ip).String()
}
if destPort, found := getKeyUint64(flow.Fields, "destinationTransportPort"); found {
ecsDest["port"] = destPort
}
if mac, found := getKeyString(flow.Fields, "destinationMacAddress"); found {
ecsDest["mac"] = mac
}

// ECS Fields -- network
ecsNetwork := common.MapStr{}
if proto, found := getKeyUint64(flow.Fields, "protocolIdentifier"); found {
Expand Down
83 changes: 83 additions & 0 deletions x-pack/filebeat/input/netflow/netflow_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/x-pack/filebeat/input/netflow/decoder"
"github.com/elastic/beats/x-pack/filebeat/input/netflow/decoder/protocol"
"github.com/elastic/beats/x-pack/filebeat/input/netflow/decoder/record"
"github.com/elastic/beats/x-pack/filebeat/input/netflow/decoder/test"
)

Expand Down Expand Up @@ -260,3 +261,85 @@ func readGoldenFile(t testing.TB, file string) TestResult {
}
return tr
}

// This test converts a flow and its reverse flow to a Beat event
// to check that they have the same flow.id, locality and community-id.
func TestReverseFlows(t *testing.T) {
parseMAC := func(s string) net.HardwareAddr {
addr, err := net.ParseMAC(s)
if err != nil {
t.Fatal(err)
}
return addr
}
flows := []record.Record{
{
Type: record.Flow,
Fields: record.Map{
"ingressInterface": uint64(2),
"destinationTransportPort": uint64(50285),
"sourceTransportPort": uint64(993),
"packetDeltaCount": uint64(26),
"ipVersion": uint64(4),
"sourceIPv4Address": net.ParseIP("203.0.113.123").To4(),
"deltaFlowCount": uint64(0),
"sourceMacAddress": parseMAC("10:00:00:00:00:02"),
"flowDirection": uint64(0),
"flowEndSysUpTime": uint64(64526131),
"vlanId": uint64(0),
"ipClassOfService": uint64(0),
"mplsLabelStackLength": uint64(3),
"tcpControlBits": uint64(27),
"egressInterface": uint64(3),
"destinationIPv4Address": net.ParseIP("10.111.111.96").To4(),
"protocolIdentifier": uint64(6),
"flowStartSysUpTime": uint64(64523806),
"destinationMacAddress": parseMAC("10:00:00:00:00:03"),
"octetDeltaCount": uint64(12852),
},
},
{
Type: record.Flow,
Fields: record.Map{
"ingressInterface": uint64(3),
"destinationTransportPort": uint64(993),
"sourceTransportPort": uint64(50285),
"packetDeltaCount": uint64(26),
"ipVersion": uint64(4),
"destinationIPv4Address": net.ParseIP("203.0.113.123").To4(),
"deltaFlowCount": uint64(0),
"postDestinationMacAddress": parseMAC("10:00:00:00:00:03"),
"flowDirection": uint64(1),
"flowEndSysUpTime": uint64(64526131),
"vlanId": uint64(0),
"ipClassOfService": uint64(0),
"mplsLabelStackLength": uint64(3),
"tcpControlBits": uint64(27),
"egressInterface": uint64(3),
"sourceIPv4Address": net.ParseIP("10.111.111.96").To4(),
"protocolIdentifier": uint64(6),
"flowStartSysUpTime": uint64(64523806),
"postSourceMacAddress": parseMAC("10:00:00:00:00:02"),
"octetDeltaCount": uint64(12852),
},
},
}

var evs []beat.Event
for _, f := range flows {
evs = append(evs, toBeatEvent(f))
}
if !assert.Len(t, evs, 2) {
t.Fatal()
}
for _, key := range []string{"flow.id", "flow.locality", "network.community_id"} {
var keys [2]interface{}
for i := range keys {
var err error
if keys[i], err = evs[i].Fields.GetValue(key); err != nil {
t.Fatal(err, "event num=", i, "key=", key)
}
}
assert.Equal(t, keys[0], keys[1], key)
}
}
10 changes: 5 additions & 5 deletions ...t/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
"kind": "event"
},
"flow": {
"id": "xrwrnNR_Xgs",
"locality": "private"
"id": "kSpZ1WuBhjc",
"locality": "public"
},
"netflow": {
"audit_counter": 4157725,
Expand Down Expand Up @@ -61,7 +61,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:hxVadzb8UUZ5YGGYtqj+E2mpQe8=",
"community_id": "1:3g7/10xslZq/7OW7ucdoDYgE3IY=",
"direction": "unknown",
"iana_number": 6,
"packets": 0,
Expand Down Expand Up @@ -96,7 +96,7 @@
"kind": "event"
},
"flow": {
"id": "7BBLL0QpIjw",
"id": "kSpZ1WuBhjc",
"locality": "public"
},
"netflow": {
Expand Down Expand Up @@ -139,7 +139,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:4oQU8uYa7F7RE/lIflbJOuU8Vc8=",
"community_id": "1:3g7/10xslZq/7OW7ucdoDYgE3IY=",
"direction": "unknown",
"iana_number": 6,
"packets": 0,
Expand Down
32 changes: 16 additions & 16 deletions x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"kind": "event"
},
"flow": {
"id": "iuHn3GlQ8XA",
"id": "2vFIarATx_4",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -49,7 +49,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:UAtYiTxp798P/gsK6Pp/IjMZLiU=",
"community_id": "1:hn30QwbDmwNihxKr9rCALGUWPgE=",
"direction": "unknown",
"iana_number": 17,
"packets": 0,
Expand Down Expand Up @@ -84,7 +84,7 @@
"kind": "event"
},
"flow": {
"id": "SJgQadmL3WE",
"id": "2vFIarATx_4",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -115,7 +115,7 @@
},
"network": {
"bytes": 81,
"community_id": "1:Z1aQRJ93TBJeX7YDGHG1zysbYVM=",
"community_id": "1:hn30QwbDmwNihxKr9rCALGUWPgE=",
"direction": "unknown",
"iana_number": 17,
"packets": 1,
Expand Down Expand Up @@ -150,7 +150,7 @@
"kind": "event"
},
"flow": {
"id": "iuHn3GlQ8XA",
"id": "wU3G8idsscw",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -181,7 +181,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:UAtYiTxp798P/gsK6Pp/IjMZLiU=",
"community_id": "1:ocm1auwAPO+Yk9MSSqJM5efL6qY=",
"direction": "unknown",
"iana_number": 17,
"packets": 0,
Expand Down Expand Up @@ -216,7 +216,7 @@
"kind": "event"
},
"flow": {
"id": "eIt31cWru0w",
"id": "wU3G8idsscw",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -247,7 +247,7 @@
},
"network": {
"bytes": 81,
"community_id": "1:Xb8wZBfARygVu6rohygs9ZhX5Og=",
"community_id": "1:ocm1auwAPO+Yk9MSSqJM5efL6qY=",
"direction": "unknown",
"iana_number": 17,
"packets": 1,
Expand Down Expand Up @@ -282,7 +282,7 @@
"kind": "event"
},
"flow": {
"id": "MNe2gxAuU4I",
"id": "rOmj8EdZ2dc",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -313,7 +313,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:ElddGTwZIDATUPp6inGMG4Ii4Xw=",
"community_id": "1:bcQGBQMaIVFnAydHjNGt5YPnRAY=",
"direction": "unknown",
"iana_number": 17,
"packets": 0,
Expand Down Expand Up @@ -348,7 +348,7 @@
"kind": "event"
},
"flow": {
"id": "Fm5S3cg6Lcw",
"id": "rOmj8EdZ2dc",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -379,7 +379,7 @@
},
"network": {
"bytes": 113,
"community_id": "1:+74AKYSI83q3lzZWoLzV4bxxCfU=",
"community_id": "1:bcQGBQMaIVFnAydHjNGt5YPnRAY=",
"direction": "unknown",
"iana_number": 17,
"packets": 1,
Expand Down Expand Up @@ -414,7 +414,7 @@
"kind": "event"
},
"flow": {
"id": "1srU81eLhhw",
"id": "JE7pThaMwJY",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -445,7 +445,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:r8af+dFTXsLQoTuHm+3HOu/Gpzc=",
"community_id": "1:ojn8oXkIUR5w+o320kdpJMiPmmM=",
"direction": "unknown",
"iana_number": 17,
"packets": 0,
Expand Down Expand Up @@ -480,7 +480,7 @@
"kind": "event"
},
"flow": {
"id": "b-peOmD4XyI",
"id": "JE7pThaMwJY",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -511,7 +511,7 @@
},
"network": {
"bytes": 113,
"community_id": "1:yBQpMR0U13RQyD1dOmOn6u8mts0=",
"community_id": "1:ojn8oXkIUR5w+o320kdpJMiPmmM=",
"direction": "unknown",
"iana_number": 17,
"packets": 1,
Expand Down
Loading

0 comments on commit 7c6f87b

Please sign in to comment.