[Filebeat] Fixes for Snyk API changes (#27358) (#27360)

* adding fixes for pagination for snyk * adding changelog entry (cherry picked from commit 3925601) Co-authored-by: Marius Iversen <marius.iversen@elastic.co>
elastic · Aug 13, 2021 · 6976638 · 6976638
1 parent caaedae
commit 6976638
Show file tree

Hide file tree

Showing 7 changed files with 44 additions and 50 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -180,6 +180,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add json body check for sqs message. {pull}21727[21727]
 - Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]
 - Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity.  Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766]
+- Convert the o365 module's `client.port` and `source.port` to numbers (from strings) in events. {pull}22939[22939]
+- Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
+- Fix s3 input when there is a blank line in the log file. {pull}25357[25357]
+- Fixes the Snyk module to work with the new API changes. {pull}27358[27358]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -1929,7 +1929,6 @@ filebeat.modules:
 - module: snyk
   audit:
     enabled: true
-
     # Set which input to use between httpjson (default) or file.
     #var.input: httpjson
     #
@@ -1939,19 +1938,15 @@ filebeat.modules:
     # The ID related to the audit_type. If audit type is group, then this value should be
     # the group ID and if it is organization it should be the organization ID to collect from.
     #var.audit_id: 1235432-asdfdf-2341234-asdgjhg
-
     # How often the API should be polled, defaults to 1 hour.
     #var.interval: 1h
     # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc).
     #var.first_interval: 24h
-
     # The API token that is created for a specific user, found in the Snyk management dashboard.
     #var.api_token:
-
     # Event filtering.
     # All configuration items below is OPTIONAL and the default options will be overwritten
     # for each entry that is not commented out.
-
     # Will return only logs for this specific project.
     #var.project_id: ""
     # User public ID. Will fetch only audit logs originated from this user's actions.
@@ -1963,33 +1958,26 @@ filebeat.modules:
 
   vulnerabilities:
     enabled: true
-
     # Set which input to use between httpjson (default) or file.
     #var.input: httpjson
-
     # How often the API should be polled. Data from the Snyk API is automatically updated
     # once per day, so the default interval is 24 hours.
     #var.interval: 24h
-
     # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc).
     #var.first_interval: 24h
-
     # The API token that is created for a specific user, found in the Snyk management dashboard.
     #var.api_token:
-
     # The list of org IDs to filter the results by.
     # One organization ID per line, starting with a - sign
     #var.orgs:
     #  - 12354-asdfdf-123543-asdsdfg
     #  - 76554-jhggfd-654342-hgrfasd
-
-
     # Event filtering.
     # All configuration items below is OPTIONAL and the default options will be overwritten
     # for each entry that is not commented out.
-
     # The severity levels of issues to filter the results by.
     #var.included_severity:
+    #  - critical
     #  - high
     #  - medium
     #  - low
@@ -2005,6 +1993,7 @@ filebeat.modules:
     #var.types:
     #  - vuln
     #  - license
+    #  - configuration
     #
     # The type of languages to filter the results by.
     #var.languages:
@@ -2016,8 +2005,13 @@ filebeat.modules:
     #  - golang
     #  - php
     #  - dotnet
-    #  - swift
+    #  - swift-objective-c
+    #  - elixir
     #  - docker
+    #  - terraform
+    #  - kubernetes
+    #  - helm
+    #  - cloudformation
     #
     # Search term to filter issue name by, or an exact CVE or CWE.
     #var.identifier:
@@ -2035,7 +2029,6 @@ filebeat.modules:
     #var.min_priority_score: 0
     #var.max_priority_score: 1000
 
-
 #----------------------------- Sonicwall-FW Module -----------------------------
 - module: sonicwall
   firewall:

diff --git a/x-pack/filebeat/module/snyk/_meta/config.yml b/x-pack/filebeat/module/snyk/_meta/config.yml
@@ -1,7 +1,6 @@
 - module: snyk
   audit:
     enabled: true
-
     # Set which input to use between httpjson (default) or file.
     #var.input: httpjson
     #
@@ -11,19 +10,15 @@
     # The ID related to the audit_type. If audit type is group, then this value should be
     # the group ID and if it is organization it should be the organization ID to collect from.
     #var.audit_id: 1235432-asdfdf-2341234-asdgjhg
-
     # How often the API should be polled, defaults to 1 hour.
     #var.interval: 1h
     # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc).
     #var.first_interval: 24h
-
     # The API token that is created for a specific user, found in the Snyk management dashboard.
     #var.api_token:
-
     # Event filtering.
     # All configuration items below is OPTIONAL and the default options will be overwritten
     # for each entry that is not commented out.
-
     # Will return only logs for this specific project.
     #var.project_id: ""
     # User public ID. Will fetch only audit logs originated from this user's actions.
@@ -35,33 +30,26 @@
 
   vulnerabilities:
     enabled: true
-
     # Set which input to use between httpjson (default) or file.
     #var.input: httpjson
-
     # How often the API should be polled. Data from the Snyk API is automatically updated
     # once per day, so the default interval is 24 hours.
     #var.interval: 24h
-
     # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc).
     #var.first_interval: 24h
-
     # The API token that is created for a specific user, found in the Snyk management dashboard.
     #var.api_token:
-
     # The list of org IDs to filter the results by.
     # One organization ID per line, starting with a - sign
     #var.orgs:
     #  - 12354-asdfdf-123543-asdsdfg
     #  - 76554-jhggfd-654342-hgrfasd
-
-
     # Event filtering.
     # All configuration items below is OPTIONAL and the default options will be overwritten
     # for each entry that is not commented out.
-
     # The severity levels of issues to filter the results by.
     #var.included_severity:
+    #  - critical
     #  - high
     #  - medium
     #  - low
@@ -77,6 +65,7 @@
     #var.types:
     #  - vuln
     #  - license
+    #  - configuration
     #
     # The type of languages to filter the results by.
     #var.languages:
@@ -88,8 +77,13 @@
     #  - golang
     #  - php
     #  - dotnet
-    #  - swift
+    #  - swift-objective-c
+    #  - elixir
     #  - docker
+    #  - terraform
+    #  - kubernetes
+    #  - helm
+    #  - cloudformation
     #
     # Search term to filter issue name by, or an exact CVE or CWE.
     #var.identifier:
@@ -106,4 +100,3 @@
     # The priority score ranging between 0-1000
     #var.min_priority_score: 0
     #var.max_priority_score: 1000
-
diff --git a/x-pack/filebeat/module/snyk/audit/config/config.yml b/x-pack/filebeat/module/snyk/audit/config/config.yml
@@ -39,10 +39,12 @@ request.body:
     event: {{ .event }}
     projectId: {{ .project_id }}
 
+response.request_body_on_pagination: true
 response.pagination:
 - set:
-    target: url.value
-    value: '[[ getRFC5988Link "next" .last_response.header.Link ]]'
+    target: url.params.page
+    value: '[[add .last_response.page 1]]'
+    fail_on_template_error: true
 
 {{ else if eq .input "file" }}
 

diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml
@@ -59,10 +59,12 @@ request.body:
       min: {{ .min_priority_score }}
       max: {{ .max_priority_score }}
 
+response.request_body_on_pagination: true
 response.pagination:
 - set:
-    target: url.value
-    value: '[[ getRFC5988Link "next" .last_response.header.Link ]]'
+    target: url.params.page
+    value: '[[add .last_response.page 1]]'
+    fail_on_template_error: true
 
 response.split:
   target: body.results

diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml b/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml
@@ -16,6 +16,7 @@ var:
   - name: orgs
   - name: included_severity
     default:
+      - critical
       - high
       - medium
       - low
@@ -29,6 +30,7 @@ var:
     default:
       - vuln
       - license
+      - configuration
   - name: languages
     default:
       - javascript
@@ -39,8 +41,13 @@ var:
       - golang
       - php
       - dotnet
-      - swift
+      - swift-objective-c
+      - elixir
       - docker
+      - terraform
+      - kubernetes
+      - helm
+      - cloudformation
   - name: identifier
     default: ""
   - name: ignored

diff --git a/x-pack/filebeat/modules.d/snyk.yml.disabled b/x-pack/filebeat/modules.d/snyk.yml.disabled
@@ -4,7 +4,6 @@
 - module: snyk
   audit:
     enabled: true
-
     # Set which input to use between httpjson (default) or file.
     #var.input: httpjson
     #
@@ -14,19 +13,15 @@
     # The ID related to the audit_type. If audit type is group, then this value should be
     # the group ID and if it is organization it should be the organization ID to collect from.
     #var.audit_id: 1235432-asdfdf-2341234-asdgjhg
-
     # How often the API should be polled, defaults to 1 hour.
     #var.interval: 1h
     # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc).
     #var.first_interval: 24h
-
     # The API token that is created for a specific user, found in the Snyk management dashboard.
     #var.api_token:
-
     # Event filtering.
     # All configuration items below is OPTIONAL and the default options will be overwritten
     # for each entry that is not commented out.
-
     # Will return only logs for this specific project.
     #var.project_id: ""
     # User public ID. Will fetch only audit logs originated from this user's actions.
@@ -38,33 +33,26 @@
 
   vulnerabilities:
     enabled: true
-
     # Set which input to use between httpjson (default) or file.
     #var.input: httpjson
-
     # How often the API should be polled. Data from the Snyk API is automatically updated
     # once per day, so the default interval is 24 hours.
     #var.interval: 24h
-
     # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc).
     #var.first_interval: 24h
-
     # The API token that is created for a specific user, found in the Snyk management dashboard.
     #var.api_token:
-
     # The list of org IDs to filter the results by.
     # One organization ID per line, starting with a - sign
     #var.orgs:
     #  - 12354-asdfdf-123543-asdsdfg
     #  - 76554-jhggfd-654342-hgrfasd
-
-
     # Event filtering.
     # All configuration items below is OPTIONAL and the default options will be overwritten
     # for each entry that is not commented out.
-
     # The severity levels of issues to filter the results by.
     #var.included_severity:
+    #  - critical
     #  - high
     #  - medium
     #  - low
@@ -80,6 +68,7 @@
     #var.types:
     #  - vuln
     #  - license
+    #  - configuration
     #
     # The type of languages to filter the results by.
     #var.languages:
@@ -91,8 +80,13 @@
     #  - golang
     #  - php
     #  - dotnet
-    #  - swift
+    #  - swift-objective-c
+    #  - elixir
     #  - docker
+    #  - terraform
+    #  - kubernetes
+    #  - helm
+    #  - cloudformation
     #
     # Search term to filter issue name by, or an exact CVE or CWE.
     #var.identifier:
@@ -109,4 +103,3 @@
     # The priority score ranging between 0-1000
     #var.min_priority_score: 0
     #var.max_priority_score: 1000
-