Skip to content

Commit

Permalink
[Netflow] Fix flow ID and locality calculation (#9891) (#9904)
Browse files Browse the repository at this point in the history 
Flow ID, Community ID and locality calculation was broken in the
NetFlow input. 0.0.0.0 was always used as destination IP.

This caused:
- The wrong flow-id and community-id to be generated
- Different ids for the reverse-flow.
- The wrong locality being assigned to the flow.

(cherry picked from commit 511e74a)
  • Loading branch information
@adriansr
adriansr authored Jan 4, 2019
1 parent 4f414d4 commit 7c6f87b
Show file tree
Hide file tree
Showing 37 changed files with 857 additions and 774 deletions.
24 changes: 12 additions & 12 deletions x-pack/filebeat/input/netflow/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,18 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
ecsSource["mac"] = mac
}

// ECS Fields -- destination
if ip, found := getKeyIP(flow.Fields, "destinationIPv4Address"); found {
ecsDest["ip"] = ip
ecsDest["locality"] = getIPLocality(ip).String()
}
if destPort, found := getKeyUint64(flow.Fields, "destinationTransportPort"); found {
ecsDest["port"] = destPort
}
if mac, found := getKeyString(flow.Fields, "destinationMacAddress"); found {
ecsDest["mac"] = mac
}

// ECS Fields -- Flow
ecsFlow := common.MapStr{}
var srcIP, dstIP net.IP
Expand Down Expand Up @@ -227,18 +239,6 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
ecsFlow["id"] = flowID(srcIP, dstIP, srcPort, dstPort, uint8(protocol))
ecsFlow["locality"] = getIPLocality(srcIP, dstIP).String()

// ECS Fields -- destination
if ip, found := getKeyIP(flow.Fields, "destinationIPv4Address"); found {
ecsDest["ip"] = ip
ecsDest["locality"] = getIPLocality(ip).String()
}
if destPort, found := getKeyUint64(flow.Fields, "destinationTransportPort"); found {
ecsDest["port"] = destPort
}
if mac, found := getKeyString(flow.Fields, "destinationMacAddress"); found {
ecsDest["mac"] = mac
}

// ECS Fields -- network
ecsNetwork := common.MapStr{}
if proto, found := getKeyUint64(flow.Fields, "protocolIdentifier"); found {
Expand Down
83 changes: 83 additions & 0 deletions x-pack/filebeat/input/netflow/netflow_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/x-pack/filebeat/input/netflow/decoder"
"github.com/elastic/beats/x-pack/filebeat/input/netflow/decoder/protocol"
"github.com/elastic/beats/x-pack/filebeat/input/netflow/decoder/record"
"github.com/elastic/beats/x-pack/filebeat/input/netflow/decoder/test"
)

Expand Down Expand Up @@ -260,3 +261,85 @@ func readGoldenFile(t testing.TB, file string) TestResult {
}
return tr
}

// This test converts a flow and its reverse flow to a Beat event
// to check that they have the same flow.id, locality and community-id.
func TestReverseFlows(t *testing.T) {
parseMAC := func(s string) net.HardwareAddr {
addr, err := net.ParseMAC(s)
if err != nil {
t.Fatal(err)
}
return addr
}
flows := []record.Record{
{
Type: record.Flow,
Fields: record.Map{
"ingressInterface": uint64(2),
"destinationTransportPort": uint64(50285),
"sourceTransportPort": uint64(993),
"packetDeltaCount": uint64(26),
"ipVersion": uint64(4),
"sourceIPv4Address": net.ParseIP("203.0.113.123").To4(),
"deltaFlowCount": uint64(0),
"sourceMacAddress": parseMAC("10:00:00:00:00:02"),
"flowDirection": uint64(0),
"flowEndSysUpTime": uint64(64526131),
"vlanId": uint64(0),
"ipClassOfService": uint64(0),
"mplsLabelStackLength": uint64(3),
"tcpControlBits": uint64(27),
"egressInterface": uint64(3),
"destinationIPv4Address": net.ParseIP("10.111.111.96").To4(),
"protocolIdentifier": uint64(6),
"flowStartSysUpTime": uint64(64523806),
"destinationMacAddress": parseMAC("10:00:00:00:00:03"),
"octetDeltaCount": uint64(12852),
},
},
{
Type: record.Flow,
Fields: record.Map{
"ingressInterface": uint64(3),
"destinationTransportPort": uint64(993),
"sourceTransportPort": uint64(50285),
"packetDeltaCount": uint64(26),
"ipVersion": uint64(4),
"destinationIPv4Address": net.ParseIP("203.0.113.123").To4(),
"deltaFlowCount": uint64(0),
"postDestinationMacAddress": parseMAC("10:00:00:00:00:03"),
"flowDirection": uint64(1),
"flowEndSysUpTime": uint64(64526131),
"vlanId": uint64(0),
"ipClassOfService": uint64(0),
"mplsLabelStackLength": uint64(3),
"tcpControlBits": uint64(27),
"egressInterface": uint64(3),
"sourceIPv4Address": net.ParseIP("10.111.111.96").To4(),
"protocolIdentifier": uint64(6),
"flowStartSysUpTime": uint64(64523806),
"postSourceMacAddress": parseMAC("10:00:00:00:00:02"),
"octetDeltaCount": uint64(12852),
},
},
}

var evs []beat.Event
for _, f := range flows {
evs = append(evs, toBeatEvent(f))
}
if !assert.Len(t, evs, 2) {
t.Fatal()
}
for _, key := range []string{"flow.id", "flow.locality", "network.community_id"} {
var keys [2]interface{}
for i := range keys {
var err error
if keys[i], err = evs[i].Fields.GetValue(key); err != nil {
t.Fatal(err, "event num=", i, "key=", key)
}
}
assert.Equal(t, keys[0], keys[1], key)
}
}
10 changes: 5 additions & 5 deletions ...t/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
"kind": "event"
},
"flow": {
"id": "xrwrnNR_Xgs",
"locality": "private"
"id": "kSpZ1WuBhjc",
"locality": "public"
},
"netflow": {
"audit_counter": 4157725,
Expand Down Expand Up @@ -61,7 +61,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:hxVadzb8UUZ5YGGYtqj+E2mpQe8=",
"community_id": "1:3g7/10xslZq/7OW7ucdoDYgE3IY=",
"direction": "unknown",
"iana_number": 6,
"packets": 0,
Expand Down Expand Up @@ -96,7 +96,7 @@
"kind": "event"
},
"flow": {
"id": "7BBLL0QpIjw",
"id": "kSpZ1WuBhjc",
"locality": "public"
},
"netflow": {
Expand Down Expand Up @@ -139,7 +139,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:4oQU8uYa7F7RE/lIflbJOuU8Vc8=",
"community_id": "1:3g7/10xslZq/7OW7ucdoDYgE3IY=",
"direction": "unknown",
"iana_number": 6,
"packets": 0,
Expand Down
32 changes: 16 additions & 16 deletions x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"kind": "event"
},
"flow": {
"id": "iuHn3GlQ8XA",
"id": "2vFIarATx_4",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -49,7 +49,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:UAtYiTxp798P/gsK6Pp/IjMZLiU=",
"community_id": "1:hn30QwbDmwNihxKr9rCALGUWPgE=",
"direction": "unknown",
"iana_number": 17,
"packets": 0,
Expand Down Expand Up @@ -84,7 +84,7 @@
"kind": "event"
},
"flow": {
"id": "SJgQadmL3WE",
"id": "2vFIarATx_4",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -115,7 +115,7 @@
},
"network": {
"bytes": 81,
"community_id": "1:Z1aQRJ93TBJeX7YDGHG1zysbYVM=",
"community_id": "1:hn30QwbDmwNihxKr9rCALGUWPgE=",
"direction": "unknown",
"iana_number": 17,
"packets": 1,
Expand Down Expand Up @@ -150,7 +150,7 @@
"kind": "event"
},
"flow": {
"id": "iuHn3GlQ8XA",
"id": "wU3G8idsscw",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -181,7 +181,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:UAtYiTxp798P/gsK6Pp/IjMZLiU=",
"community_id": "1:ocm1auwAPO+Yk9MSSqJM5efL6qY=",
"direction": "unknown",
"iana_number": 17,
"packets": 0,
Expand Down Expand Up @@ -216,7 +216,7 @@
"kind": "event"
},
"flow": {
"id": "eIt31cWru0w",
"id": "wU3G8idsscw",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -247,7 +247,7 @@
},
"network": {
"bytes": 81,
"community_id": "1:Xb8wZBfARygVu6rohygs9ZhX5Og=",
"community_id": "1:ocm1auwAPO+Yk9MSSqJM5efL6qY=",
"direction": "unknown",
"iana_number": 17,
"packets": 1,
Expand Down Expand Up @@ -282,7 +282,7 @@
"kind": "event"
},
"flow": {
"id": "MNe2gxAuU4I",
"id": "rOmj8EdZ2dc",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -313,7 +313,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:ElddGTwZIDATUPp6inGMG4Ii4Xw=",
"community_id": "1:bcQGBQMaIVFnAydHjNGt5YPnRAY=",
"direction": "unknown",
"iana_number": 17,
"packets": 0,
Expand Down Expand Up @@ -348,7 +348,7 @@
"kind": "event"
},
"flow": {
"id": "Fm5S3cg6Lcw",
"id": "rOmj8EdZ2dc",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -379,7 +379,7 @@
},
"network": {
"bytes": 113,
"community_id": "1:+74AKYSI83q3lzZWoLzV4bxxCfU=",
"community_id": "1:bcQGBQMaIVFnAydHjNGt5YPnRAY=",
"direction": "unknown",
"iana_number": 17,
"packets": 1,
Expand Down Expand Up @@ -414,7 +414,7 @@
"kind": "event"
},
"flow": {
"id": "1srU81eLhhw",
"id": "JE7pThaMwJY",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -445,7 +445,7 @@
},
"network": {
"bytes": 0,
"community_id": "1:r8af+dFTXsLQoTuHm+3HOu/Gpzc=",
"community_id": "1:ojn8oXkIUR5w+o320kdpJMiPmmM=",
"direction": "unknown",
"iana_number": 17,
"packets": 0,
Expand Down Expand Up @@ -480,7 +480,7 @@
"kind": "event"
},
"flow": {
"id": "b-peOmD4XyI",
"id": "JE7pThaMwJY",
"locality": "private"
},
"netflow": {
Expand Down Expand Up @@ -511,7 +511,7 @@
},
"network": {
"bytes": 113,
"community_id": "1:yBQpMR0U13RQyD1dOmOn6u8mts0=",
"community_id": "1:ojn8oXkIUR5w+o320kdpJMiPmmM=",
"direction": "unknown",
"iana_number": 17,
"packets": 1,
Expand Down
Loading

0 comments on commit 7c6f87b

Please sign in to comment.