[ModSecurity]Add modsecurity integration (#1603)

* initial modsecurity integration

* update fixes

* pipeline changes for date,audit data and url

* update readme

* bump to ecs 1.12.0

* generate test logs with ecs 1.12.0

* remove deault value for tz

* format json

Co-authored-by: Saw Winn Naung <sawwinnnaung@gmail.com>

packages/modsecurity/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@1.12

packages/modsecurity/_dev/build/docs/README.md

@@ -0,0 +1,19 @@
+# Modsecuriy Integration
+
+This integration periodically fetches audit logs from [Modsecurity](https://github.com/SpiderLabs/ModSecurity/) servers. It can parse audit logs created by the HTTP server.
+
+## Compatibility
+
+The logs were tested with Modsecurity v3 with nginx connector.Change the default modsecurity logging format to json as per configuration
+
+```
+SecAuditLogType Serial
+SecAuditLog /var/log/modsec_audit.json
+SecAuditLogFormat JSON
+```
+
+### Audit Log
+
+The `Audit Log` dataset collects Modsecurity Audit logs.
+
+{{fields "auditlog"}}

packages/modsecurity/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2.3'
+services:
+  modsec-audit-logfile:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:rw
+      - ${SERVICE_LOGS_DIR}:/var/log
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"

packages/modsecurity/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1603

packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-audit.log

@@ -0,0 +1,4 @@
+{"transaction":{"client_ip":"176.58.101.217","time_stamp":"Fri May 14 14:52:47 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":44464,"host_ip":"176.58.101.217","host_port":443,"id":"162100396753.595789","request":{"method":"GET","http_version":1.1,"uri":"/owa/","headers":{"Host":"34.87.56.16","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36","Accept":"*/*","Accept-Encoding":"gzip"}},"response":{"http_code":404,"headers":{"Strict-Transport-Security":"max-age=31536000; includeSubDomains","X-Runtime":"0.003894","X-Powered-By":"Phusion Passenger 6.0.2","Connection":"keep-alive","Content-Encoding":"gzip","Vary":"Origin","Status":"404 Not Found","X-Request-Id":"435c78d3-c122-4dee-8ca5-101397fab368","Server":"nginx/1.14.0","Content-Type":"text/html; charset=utf-8","Date":"Fri, 14 May 2021 14:52:47 GMT","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v25,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}}
+{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:11:52 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":40742,"host_ip":"162.62.123.46","host_port":443,"id":"162100511255.595254","request":{"method":"GET","http_version":1.1,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=c94b2c408d9b56b91e00877fb6c21fca; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"dda3a9b33849ca9d88844c0331e9b98f\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:11:52 GMT","Server":"nginx/1.14.0","X-Request-Id":"63b9e1d0-481f-43b5-9ca3-e1606c48c338","X-Download-Options":"noopen","X-Runtime":"0.028032","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}}
+{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:12:01 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":44460,"host_ip":"162.62.123.46","host_port":443,"id":"162100512158.550855","request":{"method":"GET","http_version":1.1,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=db4a0ad600d22d8015c49062844b3ac9; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"4b55096b2de9c691c0e0f67a496dc7d9\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:12:01 GMT","Server":"nginx/1.14.0","X-Request-Id":"b7220068-a82e-4535-be4c-a087fe3901ed","X-Download-Options":"noopen","X-Runtime":"0.029745","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}}
+{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:12:18 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":45952,"host_ip":"162.62.123.46","host_port":443,"id":"162100513893.802359","request":{"method":"GET","http_version":1.0,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=e1e011a4d0188a1453cc4b8b9f3e476c; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"f7e5c631964147f2a3458c4f97647883\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:12:18 GMT","Server":"nginx/1.14.0","X-Request-Id":"15fa3f35-b204-4b2a-bbd8-7aec1d8e4417","X-Download-Options":"noopen","X-Runtime":"0.026203","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}}