diff --git a/packages/modsecurity/_dev/build/build.yml b/packages/modsecurity/_dev/build/build.yml new file mode 100644 index 00000000000..08d85edcf9a --- /dev/null +++ b/packages/modsecurity/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@1.12 diff --git a/packages/modsecurity/_dev/build/docs/README.md b/packages/modsecurity/_dev/build/docs/README.md new file mode 100644 index 00000000000..39eed8515a0 --- /dev/null +++ b/packages/modsecurity/_dev/build/docs/README.md @@ -0,0 +1,19 @@ +# Modsecuriy Integration + +This integration periodically fetches audit logs from [Modsecurity](https://github.com/SpiderLabs/ModSecurity/) servers. It can parse audit logs created by the HTTP server. + +## Compatibility + +The logs were tested with Modsecurity v3 with nginx connector.Change the default modsecurity logging format to json as per configuration + +``` +SecAuditLogType Serial +SecAuditLog /var/log/modsec_audit.json +SecAuditLogFormat JSON +``` + +### Audit Log + +The `Audit Log` dataset collects Modsecurity Audit logs. + +{{fields "auditlog"}} diff --git a/packages/modsecurity/_dev/deploy/docker/docker-compose.yml b/packages/modsecurity/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..181a27f141b --- /dev/null +++ b/packages/modsecurity/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,8 @@ +version: '2.3' +services: + modsec-audit-logfile: + image: alpine + volumes: + - ./sample_logs:/sample_logs:rw + - ${SERVICE_LOGS_DIR}:/var/log + command: /bin/sh -c "cp /sample_logs/* /var/log/" diff --git a/packages/modsecurity/_dev/deploy/docker/sample_logs/modsec-audit.log b/packages/modsecurity/_dev/deploy/docker/sample_logs/modsec-audit.log new file mode 100644 index 00000000000..00c1bf37a1a --- /dev/null +++ b/packages/modsecurity/_dev/deploy/docker/sample_logs/modsec-audit.log @@ -0,0 +1,15 @@ +{"transaction":{"client_ip":"37.120.205.2","time_stamp":"Fri May 14 14:38:37 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":56047,"host_ip":"37.120.205.2","host_port":443,"id":"16210031172.153816","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2734183/finish","headers":{"Authorization":"365dcb63535abc1ecf5a98611a028f82389ad4a07678eea5a6260b12993f0906","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1010","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 14:38:37 GMT","Server":"nginx/1.14.0","X-Request-Id":"5a99f40a-e1b3-4ff9-8acd-a55cb39753f6","X-Download-Options":"noopen","X-Runtime":"0.159787","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"69.160.3.115","time_stamp":"Fri May 14 14:38:39 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":42106,"host_ip":"69.160.3.115","host_port":443,"id":"162100311934.404730","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2755491/finish","headers":{"Authorization":"4f3aeebdb2868a07f6b52a97fc8337419080e2afbaf9d9953ae347065c57197c","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1042","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 14:38:39 GMT","Server":"nginx/1.14.0","X-Request-Id":"caee7ca8-b140-495d-ae58-527bce584ece","X-Download-Options":"noopen","X-Runtime":"0.162241","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"69.160.3.115","time_stamp":"Fri May 14 14:39:01 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":42106,"host_ip":"69.160.3.115","host_port":443,"id":"162100314172.167886","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2755491/finish","headers":{"Authorization":"4f3aeebdb2868a07f6b52a97fc8337419080e2afbaf9d9953ae347065c57197c","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1042","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 14:39:01 GMT","Server":"nginx/1.14.0","X-Request-Id":"a31aeba5-7a57-4f0c-8472-19285fd92a65","X-Download-Options":"noopen","X-Runtime":"0.110758","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"51.81.186.254","time_stamp":"Fri May 14 14:46:53 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":56627,"host_ip":"51.81.186.254","host_port":443,"id":"162100361322.069228","request":{"method":"GET","http_version":1.1,"uri":"/customers/properties?trip_id=0","headers":{"Host":"www.owayride.com","Authorization":"a062a8301dea86879ee7bb3c2115705f420c10aebb1279c504396736d7ae9489","Content-Type":"application/json","Connection":"keep-alive","Accept":"*/*","User-Agent":"Oway Ride/7.0.27 (com.owaycabs.passenger; build:200116; iOS 14.4.2) Alamofire/4.9.0","Accept-Language":"en-MM;q=1.0, my-MM;q=0.9","Accept-Encoding":"gzip;q=1.0, compress;q=0.5"}},"response":{"http_code":500,"headers":{"X-Runtime":"2.583300","X-Powered-By":"Phusion Passenger 6.0.2","Connection":"keep-alive","Vary":"Origin","Status":"500 Internal Server Error","X-Request-Id":"d77491f3-7bf4-4753-b474-01c68daa1a84","Server":"nginx/1.14.0","Content-Type":"text/html; charset=utf-8","Content-Length":"1477","Date":"Fri, 14 May 2021 14:46:55 GMT","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"210.14.100.32","time_stamp":"Fri May 14 14:49:05 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":27034,"host_ip":"210.14.100.32","host_port":443,"id":"162100374575.622721","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2762450/finish","headers":{"Authorization":"694293942460dee77c1a7d11c01adc3ba66b0b44b3ae7dab5f79a422a5e464ba","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1074","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 14:49:06 GMT","Server":"nginx/1.14.0","X-Request-Id":"11b01166-6955-4458-9506-492ed3cc276c","X-Download-Options":"noopen","X-Runtime":"0.176057","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"128.14.80.81","time_stamp":"Fri May 14 14:49:13 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":49618,"host_ip":"128.14.80.81","host_port":443,"id":"162100375347.540524","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2762450/finish","headers":{"Authorization":"694293942460dee77c1a7d11c01adc3ba66b0b44b3ae7dab5f79a422a5e464ba","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1074","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 14:49:13 GMT","Server":"nginx/1.14.0","X-Request-Id":"35c798ee-6446-432a-8b10-bc50937cf9a6","X-Download-Options":"noopen","X-Runtime":"0.156531","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"176.58.101.217","time_stamp":"Fri May 14 14:52:47 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":44464,"host_ip":"176.58.101.217","host_port":443,"id":"162100396753.595789","request":{"method":"GET","http_version":1.1,"uri":"/owa/","headers":{"Host":"34.87.56.16","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36","Accept":"*/*","Accept-Encoding":"gzip"}},"response":{"http_code":404,"headers":{"Strict-Transport-Security":"max-age=31536000; includeSubDomains","X-Runtime":"0.003894","X-Powered-By":"Phusion Passenger 6.0.2","Connection":"keep-alive","Content-Encoding":"gzip","Vary":"Origin","Status":"404 Not Found","X-Request-Id":"435c78d3-c122-4dee-8ca5-101397fab368","Server":"nginx/1.14.0","Content-Type":"text/html; charset=utf-8","Date":"Fri, 14 May 2021 14:52:47 GMT","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v25,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}} +{"transaction":{"client_ip":"37.120.205.2","time_stamp":"Fri May 14 15:08:22 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":55175,"host_ip":"37.120.205.2","host_port":443,"id":"162100490217.924032","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2734183/finish","headers":{"Authorization":"365dcb63535abc1ecf5a98611a028f82389ad4a07678eea5a6260b12993f0906","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1010","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 15:08:23 GMT","Server":"nginx/1.14.0","X-Request-Id":"446039c0-0fcf-43f2-af46-6c8227c82f4e","X-Download-Options":"noopen","X-Runtime":"0.165052","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:11:52 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":40742,"host_ip":"162.62.123.46","host_port":443,"id":"162100511255.595254","request":{"method":"GET","http_version":1.1,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=c94b2c408d9b56b91e00877fb6c21fca; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"dda3a9b33849ca9d88844c0331e9b98f\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:11:52 GMT","Server":"nginx/1.14.0","X-Request-Id":"63b9e1d0-481f-43b5-9ca3-e1606c48c338","X-Download-Options":"noopen","X-Runtime":"0.028032","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}} +{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:12:01 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":44460,"host_ip":"162.62.123.46","host_port":443,"id":"162100512158.550855","request":{"method":"GET","http_version":1.1,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=db4a0ad600d22d8015c49062844b3ac9; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"4b55096b2de9c691c0e0f67a496dc7d9\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:12:01 GMT","Server":"nginx/1.14.0","X-Request-Id":"b7220068-a82e-4535-be4c-a087fe3901ed","X-Download-Options":"noopen","X-Runtime":"0.029745","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}} +{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:12:18 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":45952,"host_ip":"162.62.123.46","host_port":443,"id":"162100513893.802359","request":{"method":"GET","http_version":1.0,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=e1e011a4d0188a1453cc4b8b9f3e476c; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"f7e5c631964147f2a3458c4f97647883\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:12:18 GMT","Server":"nginx/1.14.0","X-Request-Id":"15fa3f35-b204-4b2a-bbd8-7aec1d8e4417","X-Download-Options":"noopen","X-Runtime":"0.026203","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}} +{"transaction":{"client_ip":"37.120.205.2","time_stamp":"Fri May 14 15:16:44 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":59740,"host_ip":"37.120.205.2","host_port":443,"id":"162100540477.608948","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2734183/finish","headers":{"Authorization":"365dcb63535abc1ecf5a98611a028f82389ad4a07678eea5a6260b12993f0906","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1010","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 15:16:44 GMT","Server":"nginx/1.14.0","X-Request-Id":"173bac1d-4ac8-4e6c-8fba-1f15a5b9e5f6","X-Download-Options":"noopen","X-Runtime":"0.108246","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"5.8.26.201","time_stamp":"Fri May 14 15:31:08 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":52299,"host_ip":"5.8.26.201","host_port":443,"id":"162100626860.990852","request":{"method":"GET","http_version":1.1,"uri":"/customers/properties?trip_id=0","headers":{"Host":"www.owayride.com","Authorization":"2d47f33d04f8934c35cdd40e09257928aeea4995dd2c31167a265fac0fb5da21","Content-Type":"application/json","Connection":"keep-alive","Accept":"*/*","User-Agent":"Oway Ride/7.0.27 (com.owaycabs.passenger; build:200116; iOS 14.5.1) Alamofire/4.9.0","Accept-Language":"en-MM;q=1.0, my-MM;q=0.9","Accept-Encoding":"gzip;q=1.0, compress;q=0.5"}},"response":{"http_code":500,"headers":{"X-Runtime":"2.495058","X-Powered-By":"Phusion Passenger 6.0.2","Connection":"keep-alive","Vary":"Origin","Status":"500 Internal Server Error","X-Request-Id":"3e75f502-36bc-43f5-a827-0a4724952696","Server":"nginx/1.14.0","Content-Type":"text/html; charset=utf-8","Content-Length":"1477","Date":"Fri, 14 May 2021 15:31:11 GMT","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"210.14.100.32","time_stamp":"Fri May 14 15:36:40 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":27070,"host_ip":"210.14.100.32","host_port":443,"id":"162100660087.362558","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2762450/finish","headers":{"Authorization":"694293942460dee77c1a7d11c01adc3ba66b0b44b3ae7dab5f79a422a5e464ba","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1074","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 15:36:40 GMT","Server":"nginx/1.14.0","X-Request-Id":"142c0213-df87-4237-9215-451e580c9731","X-Download-Options":"noopen","X-Runtime":"0.177242","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} +{"transaction":{"client_ip":"128.14.80.81","time_stamp":"Fri May 14 15:36:53 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":55139,"host_ip":"128.14.80.81","host_port":443,"id":"162100661384.520745","request":{"method":"PUT","http_version":1.1,"uri":"/orders/2762450/finish","headers":{"Authorization":"694293942460dee77c1a7d11c01adc3ba66b0b44b3ae7dab5f79a422a5e464ba","Host":"www.owayride.com","Content-Type":"application/json; charset=UTF-8","Content-Length":"1074","Connection":"Keep-Alive","Accept-Encoding":"gzip","User-Agent":"okhttp/2.7.5"}},"response":{"http_code":400,"headers":{"Content-Encoding":"gzip","Connection":"keep-alive","X-Powered-By":"Phusion Passenger 6.0.2","Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"400 Bad Request","Content-Type":"application/json; charset=utf-8","Date":"Fri, 14 May 2021 15:36:54 GMT","Server":"nginx/1.14.0","X-Request-Id":"32b7ca59-1871-4ade-adbe-0b7a8f6b9dfc","X-Download-Options":"noopen","X-Runtime":"0.110907","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[]}} diff --git a/packages/modsecurity/changelog.yml b/packages/modsecurity/changelog.yml new file mode 100644 index 00000000000..c357a35af95 --- /dev/null +++ b/packages/modsecurity/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.1.0" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1603 diff --git a/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-audit.log b/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-audit.log new file mode 100644 index 00000000000..f8fa8be960e --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-audit.log @@ -0,0 +1,4 @@ +{"transaction":{"client_ip":"176.58.101.217","time_stamp":"Fri May 14 14:52:47 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":44464,"host_ip":"176.58.101.217","host_port":443,"id":"162100396753.595789","request":{"method":"GET","http_version":1.1,"uri":"/owa/","headers":{"Host":"34.87.56.16","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36","Accept":"*/*","Accept-Encoding":"gzip"}},"response":{"http_code":404,"headers":{"Strict-Transport-Security":"max-age=31536000; includeSubDomains","X-Runtime":"0.003894","X-Powered-By":"Phusion Passenger 6.0.2","Connection":"keep-alive","Content-Encoding":"gzip","Vary":"Origin","Status":"404 Not Found","X-Request-Id":"435c78d3-c122-4dee-8ca5-101397fab368","Server":"nginx/1.14.0","Content-Type":"text/html; charset=utf-8","Date":"Fri, 14 May 2021 14:52:47 GMT","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v25,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}} +{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:11:52 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":40742,"host_ip":"162.62.123.46","host_port":443,"id":"162100511255.595254","request":{"method":"GET","http_version":1.1,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=c94b2c408d9b56b91e00877fb6c21fca; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"dda3a9b33849ca9d88844c0331e9b98f\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:11:52 GMT","Server":"nginx/1.14.0","X-Request-Id":"63b9e1d0-481f-43b5-9ca3-e1606c48c338","X-Download-Options":"noopen","X-Runtime":"0.028032","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}} +{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:12:01 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":44460,"host_ip":"162.62.123.46","host_port":443,"id":"162100512158.550855","request":{"method":"GET","http_version":1.1,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=db4a0ad600d22d8015c49062844b3ac9; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"4b55096b2de9c691c0e0f67a496dc7d9\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:12:01 GMT","Server":"nginx/1.14.0","X-Request-Id":"b7220068-a82e-4535-be4c-a087fe3901ed","X-Download-Options":"noopen","X-Runtime":"0.029745","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}} +{"transaction":{"client_ip":"162.62.123.46","time_stamp":"Fri May 14 15:12:18 2021","server_id":"c06217c4ac0d6f8892d2489cd5d92aaceec2508e","client_port":45952,"host_ip":"162.62.123.46","host_port":443,"id":"162100513893.802359","request":{"method":"GET","http_version":1.0,"uri":"/","headers":{"Host":"34.87.56.16","Connection":"close"}},"response":{"http_code":200,"headers":{"Vary":"Accept-Encoding, Origin","X-XSS-Protection":"1; mode=block","Set-Cookie":"_pmcapi_session=e1e011a4d0188a1453cc4b8b9f3e476c; path=/; HttpOnly","X-Permitted-Cross-Domain-Policies":"none","Cache-Control":"max-age=0, private, must-revalidate","ETag":"W/\"f7e5c631964147f2a3458c4f97647883\"","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Status":"200 OK","Connection":"close","X-Powered-By":"Phusion Passenger 6.0.2","Content-Type":"text/html; charset=utf-8","Content-Length":"12475","Date":"Fri, 14 May 2021 15:12:18 GMT","Server":"nginx/1.14.0","X-Request-Id":"15fa3f35-b204-4b2a-bbd8-7aec1d8e4417","X-Download-Options":"noopen","X-Runtime":"0.026203","X-Content-Type-Options":"nosniff","X-Frame-Options":"SAMEORIGIN","Via":"1.1 google"}},"producer":{"modsecurity":"ModSecurity v3.0.2 (Linux)","connector":"ModSecurity-nginx v0.1.1-beta","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )","reference":"o0,11v21,11","ruleId":"920350","file":"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"733","data":"34.87.56.16","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}}]}} diff --git a/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-audit.log-expected.json b/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-audit.log-expected.json new file mode 100644 index 00000000000..3f9a108d8d4 --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-audit.log-expected.json @@ -0,0 +1,280 @@ +{ + "expected": [ + { + "modsec": { + "audit": { + "detail": "Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )" + } + }, + "rule": { + "id": "920350" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "GB-ENG", + "city_name": "London", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "region_name": "England", + "location": { + "lon": -0.0961, + "lat": 51.5132 + } + }, + "as": { + "number": 63949, + "organization": { + "name": "Linode, LLC" + } + }, + "port": 44464, + "ip": "176.58.101.217" + }, + "message": "Host header is a numeric IP address", + "url": { + "path": "/owa/", + "original": "https://34.87.56.16:443/owa/", + "scheme": "https", + "port": 443, + "domain": "34.87.56.16" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-05-14T14:52:47.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "mime_type": "text/html; charset=utf-8", + "status_code": 404 + } + }, + "event": { + "ingested": "2021-09-17T03:51:00.601664853Z", + "original": "{\"transaction\":{\"client_ip\":\"176.58.101.217\",\"time_stamp\":\"Fri May 14 14:52:47 2021\",\"server_id\":\"c06217c4ac0d6f8892d2489cd5d92aaceec2508e\",\"client_port\":44464,\"host_ip\":\"176.58.101.217\",\"host_port\":443,\"id\":\"162100396753.595789\",\"request\":{\"method\":\"GET\",\"http_version\":1.1,\"uri\":\"/owa/\",\"headers\":{\"Host\":\"34.87.56.16\",\"User-Agent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\",\"Accept\":\"*/*\",\"Accept-Encoding\":\"gzip\"}},\"response\":{\"http_code\":404,\"headers\":{\"Strict-Transport-Security\":\"max-age=31536000; includeSubDomains\",\"X-Runtime\":\"0.003894\",\"X-Powered-By\":\"Phusion Passenger 6.0.2\",\"Connection\":\"keep-alive\",\"Content-Encoding\":\"gzip\",\"Vary\":\"Origin\",\"Status\":\"404 Not Found\",\"X-Request-Id\":\"435c78d3-c122-4dee-8ca5-101397fab368\",\"Server\":\"nginx/1.14.0\",\"Content-Type\":\"text/html; charset=utf-8\",\"Date\":\"Fri, 14 May 2021 14:52:47 GMT\",\"Via\":\"1.1 google\"}},\"producer\":{\"modsecurity\":\"ModSecurity v3.0.2 (Linux)\",\"connector\":\"ModSecurity-nginx v0.1.1-beta\",\"secrules_engine\":\"Enabled\",\"components\":[\"OWASP_CRS/3.0.2\\\"\"]},\"messages\":[{\"message\":\"Host header is a numeric IP address\",\"details\":{\"match\":\"Matched \\\"Operator `Rx' with parameter `^[\\\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )\",\"reference\":\"o0,11v25,11\",\"ruleId\":\"920350\",\"file\":\"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf\",\"lineNumber\":\"733\",\"data\":\"34.87.56.16\",\"severity\":\"4\",\"ver\":\"OWASP_CRS/3.0.0\",\"rev\":\"2\",\"tags\":[\"application-multi\",\"language-multi\",\"platform-multi\",\"attack-protocol\",\"OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST\",\"WASCTC/WASC-21\",\"OWASP_TOP_10/A7\",\"PCI/6.5.10\"],\"maturity\":\"9\",\"accuracy\":\"9\"}}]}}", + "category": [ + "web" + ], + "type": [ + "access" + ], + "kind": "event" + }, + "user_agent": { + "name": "Chrome", + "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36", + "os": { + "name": "Windows", + "version": "10", + "full": "Windows 10" + }, + "device": { + "name": "Other" + }, + "version": "60.0.3112.113" + } + }, + { + "modsec": { + "audit": { + "detail": "Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )" + } + }, + "rule": { + "id": "920350" + }, + "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Singapore", + "location": { + "lon": 103.8, + "lat": 1.3667 + }, + "country_iso_code": "SG" + }, + "as": { + "number": 132203, + "organization": { + "name": "Tencent Building, Kejizhongyi Avenue" + } + }, + "port": 40742, + "ip": "162.62.123.46" + }, + "message": "Host header is a numeric IP address", + "url": { + "path": "/", + "original": "https://34.87.56.16:443/", + "scheme": "https", + "port": 443, + "domain": "34.87.56.16" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-05-14T15:11:52.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "mime_type": "text/html; charset=utf-8", + "status_code": 200 + } + }, + "event": { + "ingested": "2021-09-17T03:51:00.601675457Z", + "original": "{\"transaction\":{\"client_ip\":\"162.62.123.46\",\"time_stamp\":\"Fri May 14 15:11:52 2021\",\"server_id\":\"c06217c4ac0d6f8892d2489cd5d92aaceec2508e\",\"client_port\":40742,\"host_ip\":\"162.62.123.46\",\"host_port\":443,\"id\":\"162100511255.595254\",\"request\":{\"method\":\"GET\",\"http_version\":1.1,\"uri\":\"/\",\"headers\":{\"Host\":\"34.87.56.16\",\"Connection\":\"close\"}},\"response\":{\"http_code\":200,\"headers\":{\"Vary\":\"Accept-Encoding, Origin\",\"X-XSS-Protection\":\"1; mode=block\",\"Set-Cookie\":\"_pmcapi_session=c94b2c408d9b56b91e00877fb6c21fca; path=/; HttpOnly\",\"X-Permitted-Cross-Domain-Policies\":\"none\",\"Cache-Control\":\"max-age=0, private, must-revalidate\",\"ETag\":\"W/\\\"dda3a9b33849ca9d88844c0331e9b98f\\\"\",\"Strict-Transport-Security\":\"max-age=31536000; includeSubDomains\",\"Status\":\"200 OK\",\"Connection\":\"close\",\"X-Powered-By\":\"Phusion Passenger 6.0.2\",\"Content-Type\":\"text/html; charset=utf-8\",\"Content-Length\":\"12475\",\"Date\":\"Fri, 14 May 2021 15:11:52 GMT\",\"Server\":\"nginx/1.14.0\",\"X-Request-Id\":\"63b9e1d0-481f-43b5-9ca3-e1606c48c338\",\"X-Download-Options\":\"noopen\",\"X-Runtime\":\"0.028032\",\"X-Content-Type-Options\":\"nosniff\",\"X-Frame-Options\":\"SAMEORIGIN\",\"Via\":\"1.1 google\"}},\"producer\":{\"modsecurity\":\"ModSecurity v3.0.2 (Linux)\",\"connector\":\"ModSecurity-nginx v0.1.1-beta\",\"secrules_engine\":\"Enabled\",\"components\":[\"OWASP_CRS/3.0.2\\\"\"]},\"messages\":[{\"message\":\"Host header is a numeric IP address\",\"details\":{\"match\":\"Matched \\\"Operator `Rx' with parameter `^[\\\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )\",\"reference\":\"o0,11v21,11\",\"ruleId\":\"920350\",\"file\":\"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf\",\"lineNumber\":\"733\",\"data\":\"34.87.56.16\",\"severity\":\"4\",\"ver\":\"OWASP_CRS/3.0.0\",\"rev\":\"2\",\"tags\":[\"application-multi\",\"language-multi\",\"platform-multi\",\"attack-protocol\",\"OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST\",\"WASCTC/WASC-21\",\"OWASP_TOP_10/A7\",\"PCI/6.5.10\"],\"maturity\":\"9\",\"accuracy\":\"9\"}}]}}", + "category": [ + "web" + ], + "type": [ + "access" + ], + "kind": "event" + } + }, + { + "modsec": { + "audit": { + "detail": "Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )" + } + }, + "rule": { + "id": "920350" + }, + "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Singapore", + "location": { + "lon": 103.8, + "lat": 1.3667 + }, + "country_iso_code": "SG" + }, + "as": { + "number": 132203, + "organization": { + "name": "Tencent Building, Kejizhongyi Avenue" + } + }, + "port": 44460, + "ip": "162.62.123.46" + }, + "message": "Host header is a numeric IP address", + "url": { + "path": "/", + "original": "https://34.87.56.16:443/", + "scheme": "https", + "port": 443, + "domain": "34.87.56.16" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-05-14T15:12:01.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "mime_type": "text/html; charset=utf-8", + "status_code": 200 + } + }, + "event": { + "ingested": "2021-09-17T03:51:00.601678749Z", + "original": "{\"transaction\":{\"client_ip\":\"162.62.123.46\",\"time_stamp\":\"Fri May 14 15:12:01 2021\",\"server_id\":\"c06217c4ac0d6f8892d2489cd5d92aaceec2508e\",\"client_port\":44460,\"host_ip\":\"162.62.123.46\",\"host_port\":443,\"id\":\"162100512158.550855\",\"request\":{\"method\":\"GET\",\"http_version\":1.1,\"uri\":\"/\",\"headers\":{\"Host\":\"34.87.56.16\",\"Connection\":\"close\"}},\"response\":{\"http_code\":200,\"headers\":{\"Vary\":\"Accept-Encoding, Origin\",\"X-XSS-Protection\":\"1; mode=block\",\"Set-Cookie\":\"_pmcapi_session=db4a0ad600d22d8015c49062844b3ac9; path=/; HttpOnly\",\"X-Permitted-Cross-Domain-Policies\":\"none\",\"Cache-Control\":\"max-age=0, private, must-revalidate\",\"ETag\":\"W/\\\"4b55096b2de9c691c0e0f67a496dc7d9\\\"\",\"Strict-Transport-Security\":\"max-age=31536000; includeSubDomains\",\"Status\":\"200 OK\",\"Connection\":\"close\",\"X-Powered-By\":\"Phusion Passenger 6.0.2\",\"Content-Type\":\"text/html; charset=utf-8\",\"Content-Length\":\"12475\",\"Date\":\"Fri, 14 May 2021 15:12:01 GMT\",\"Server\":\"nginx/1.14.0\",\"X-Request-Id\":\"b7220068-a82e-4535-be4c-a087fe3901ed\",\"X-Download-Options\":\"noopen\",\"X-Runtime\":\"0.029745\",\"X-Content-Type-Options\":\"nosniff\",\"X-Frame-Options\":\"SAMEORIGIN\",\"Via\":\"1.1 google\"}},\"producer\":{\"modsecurity\":\"ModSecurity v3.0.2 (Linux)\",\"connector\":\"ModSecurity-nginx v0.1.1-beta\",\"secrules_engine\":\"Enabled\",\"components\":[\"OWASP_CRS/3.0.2\\\"\"]},\"messages\":[{\"message\":\"Host header is a numeric IP address\",\"details\":{\"match\":\"Matched \\\"Operator `Rx' with parameter `^[\\\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )\",\"reference\":\"o0,11v21,11\",\"ruleId\":\"920350\",\"file\":\"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf\",\"lineNumber\":\"733\",\"data\":\"34.87.56.16\",\"severity\":\"4\",\"ver\":\"OWASP_CRS/3.0.0\",\"rev\":\"2\",\"tags\":[\"application-multi\",\"language-multi\",\"platform-multi\",\"attack-protocol\",\"OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST\",\"WASCTC/WASC-21\",\"OWASP_TOP_10/A7\",\"PCI/6.5.10\"],\"maturity\":\"9\",\"accuracy\":\"9\"}}]}}", + "category": [ + "web" + ], + "type": [ + "access" + ], + "kind": "event" + } + }, + { + "modsec": { + "audit": { + "detail": "Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )" + } + }, + "rule": { + "id": "920350" + }, + "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Singapore", + "location": { + "lon": 103.8, + "lat": 1.3667 + }, + "country_iso_code": "SG" + }, + "as": { + "number": 132203, + "organization": { + "name": "Tencent Building, Kejizhongyi Avenue" + } + }, + "port": 45952, + "ip": "162.62.123.46" + }, + "message": "Host header is a numeric IP address", + "url": { + "path": "/", + "original": "https://34.87.56.16:443/", + "scheme": "https", + "port": 443, + "domain": "34.87.56.16" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-05-14T15:12:18.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.0", + "response": { + "mime_type": "text/html; charset=utf-8", + "status_code": 200 + } + }, + "event": { + "ingested": "2021-09-17T03:51:00.601681843Z", + "original": "{\"transaction\":{\"client_ip\":\"162.62.123.46\",\"time_stamp\":\"Fri May 14 15:12:18 2021\",\"server_id\":\"c06217c4ac0d6f8892d2489cd5d92aaceec2508e\",\"client_port\":45952,\"host_ip\":\"162.62.123.46\",\"host_port\":443,\"id\":\"162100513893.802359\",\"request\":{\"method\":\"GET\",\"http_version\":1.0,\"uri\":\"/\",\"headers\":{\"Host\":\"34.87.56.16\",\"Connection\":\"close\"}},\"response\":{\"http_code\":200,\"headers\":{\"Vary\":\"Accept-Encoding, Origin\",\"X-XSS-Protection\":\"1; mode=block\",\"Set-Cookie\":\"_pmcapi_session=e1e011a4d0188a1453cc4b8b9f3e476c; path=/; HttpOnly\",\"X-Permitted-Cross-Domain-Policies\":\"none\",\"Cache-Control\":\"max-age=0, private, must-revalidate\",\"ETag\":\"W/\\\"f7e5c631964147f2a3458c4f97647883\\\"\",\"Strict-Transport-Security\":\"max-age=31536000; includeSubDomains\",\"Status\":\"200 OK\",\"Connection\":\"close\",\"X-Powered-By\":\"Phusion Passenger 6.0.2\",\"Content-Type\":\"text/html; charset=utf-8\",\"Content-Length\":\"12475\",\"Date\":\"Fri, 14 May 2021 15:12:18 GMT\",\"Server\":\"nginx/1.14.0\",\"X-Request-Id\":\"15fa3f35-b204-4b2a-bbd8-7aec1d8e4417\",\"X-Download-Options\":\"noopen\",\"X-Runtime\":\"0.026203\",\"X-Content-Type-Options\":\"nosniff\",\"X-Frame-Options\":\"SAMEORIGIN\",\"Via\":\"1.1 google\"}},\"producer\":{\"modsecurity\":\"ModSecurity v3.0.2 (Linux)\",\"connector\":\"ModSecurity-nginx v0.1.1-beta\",\"secrules_engine\":\"Enabled\",\"components\":[\"OWASP_CRS/3.0.2\\\"\"]},\"messages\":[{\"message\":\"Host header is a numeric IP address\",\"details\":{\"match\":\"Matched \\\"Operator `Rx' with parameter `^[\\\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `34.87.56.16' )\",\"reference\":\"o0,11v21,11\",\"ruleId\":\"920350\",\"file\":\"/etc/nginx/modsec/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf\",\"lineNumber\":\"733\",\"data\":\"34.87.56.16\",\"severity\":\"4\",\"ver\":\"OWASP_CRS/3.0.0\",\"rev\":\"2\",\"tags\":[\"application-multi\",\"language-multi\",\"platform-multi\",\"attack-protocol\",\"OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST\",\"WASCTC/WASC-21\",\"OWASP_TOP_10/A7\",\"PCI/6.5.10\"],\"maturity\":\"9\",\"accuracy\":\"9\"}}]}}", + "category": [ + "web" + ], + "type": [ + "access" + ], + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-common-config.yml b/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..89a777e6427 --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,6 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tz_offset: "+00:00" + tags: + - preserve_original_event diff --git a/packages/modsecurity/data_stream/auditlog/_dev/test/system/test-logfile-config.yml b/packages/modsecurity/data_stream/auditlog/_dev/test/system/test-logfile-config.yml new file mode 100644 index 00000000000..3591d081550 --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/_dev/test/system/test-logfile-config.yml @@ -0,0 +1,7 @@ +service: modsec-audit-logfile +vars: ~ +input: logfile +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/modsec-audit.log" diff --git a/packages/modsecurity/data_stream/auditlog/agent/stream/stream.yml.hbs b/packages/modsecurity/data_stream/auditlog/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..a6238fb1b9a --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/agent/stream/stream.yml.hbs @@ -0,0 +1,23 @@ +paths: +{{#each paths}} +- {{this}} +{{/each}} +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +fields_under_root: true +fields: + tz_offset: {{tz_offset}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +exclude_files: [".gz$"] +processors: +{{#if processors}} +{{processors}} +{{/if}} +- add_locale: ~ diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..fe85cabc28e --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,145 @@ +--- +description: Pipeline for modsecurity audit log. + +processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: + field: message + target_field: event.original + ignore_missing: true + - json: + field: event.original + target_field: json + ignore_failure: true +# parse date + - set: + field: _temps.date + if: ctx?.tz_offset != null + value: '{{json.transaction.time_stamp}}{{tz_offset}}' + - set: + field: _temps.date + if: ctx?.tz_offset == null + value: '{{json.transaction.time_stamp}}{{event.timezone }}' + - date: + field: _temps.date + formats: + - E MMM d HH:mm:ss yyyyZ + - E MMM d HH:mm:ss yyyyXXX +# rename ecs + - rename: + field: json.transaction.client_ip + target_field: source.ip + ignore_missing: true + - rename: + field: json.transaction.client_port + target_field: source.port + ignore_missing: true + - rename: + field: json.transaction.request.method + target_field: http.request.method + ignore_missing: true + - convert: + field: json.transaction.request.http_version + target_field: http.version + type: string + ignore_missing: true + - set: + field: _temps.url + if: ctx.json.transaction.host_port == 443 + value: "https://{{{json.transaction.request.headers.Host}}}:{{json.transaction.host_port}}{{{json.transaction.request.uri}}}" + - set: + field: _temps.url + if: ctx.json.transaction.host_port == 80 + value: "http://{{{json.transaction.request.headers.Host}}}:{{json.transaction.host_port}}{{{json.transaction.request.uri}}}" + - uri_parts: + field: _temps.url + ignore_failure: true + keep_original: true + remove_if_successful: true + - rename: + field: json.transaction.response.http_code + target_field: http.response.status_code + ignore_missing: true + - rename: + field: json.transaction.response.headers.Content-Type + target_field: http.response.mime_type + ignore_missing: true + - rename: + field: json.transaction.response.Content-Length + target_field: http.response.bytes + ignore_missing: true + - foreach: + field: json.transaction.messages + ignore_missing: true + processor: + rename: + field: _ingest._value.message + target_field: message + - foreach: + field: json.transaction.messages + ignore_missing: true + processor: + rename: + field: _ingest._value.details.match + target_field: modsec.audit.detail + - foreach: + field: json.transaction.messages + ignore_missing: true + processor: + rename: + field: _ingest._value.details.ruleId + target_field: rule.id +# user agent and geoip enrich + - user_agent: + field: json.transaction.request.headers.User-Agent + ignore_missing: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - set: + field: event.kind + value: event + - append: + field: event.category + value: web + - append: + field: event.type + value: access + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + - remove: + field: + - json + - _temps + - tz_offset + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/modsecurity/data_stream/auditlog/fields/agent.yml b/packages/modsecurity/data_stream/auditlog/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/modsecurity/data_stream/auditlog/fields/base-fields.yml b/packages/modsecurity/data_stream/auditlog/fields/base-fields.yml new file mode 100644 index 00000000000..041609421bf --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/fields/base-fields.yml @@ -0,0 +1,23 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: 'message' + type: text + description: human-readable summary of the event +- name: event.module + type: constant_keyword + description: Event module + value: modsecurity +- name: event.dataset + type: constant_keyword + description: Event dataset + value: modsecurity.auditlog diff --git a/packages/modsecurity/data_stream/auditlog/fields/ecs.yml b/packages/modsecurity/data_stream/auditlog/fields/ecs.yml new file mode 100644 index 00000000000..db4c78747d8 --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/fields/ecs.yml @@ -0,0 +1,88 @@ +- external: ecs + name: destination.domain +- external: ecs + name: destination.ip +- external: ecs + name: destination.port +- external: ecs + name: ecs.version +- external: ecs + name: host.ip +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.body.bytes +- external: ecs + name: http.response.status_code +- external: ecs + name: http.response.mime_type +- external: ecs + name: http.version +- external: ecs + name: log.file.path +- external: ecs + name: related.ip +- external: ecs + name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- description: Longitude and latitude. + level: core + name: source.geo.location + type: geo_point +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: source.port +- external: ecs + name: tags +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.fragment +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.scheme +- external: ecs + name: url.port +- external: ecs + name: url.query +- external: ecs + name: user.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version +- external: ecs + name: rule.id diff --git a/packages/modsecurity/data_stream/auditlog/fields/fields.yml b/packages/modsecurity/data_stream/auditlog/fields/fields.yml new file mode 100644 index 00000000000..1ad1b17fe0a --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/fields/fields.yml @@ -0,0 +1,6 @@ +- name: modsec.audit + type: group + fields: + - name: detail + type: keyword + description: Details message of the audit event. diff --git a/packages/modsecurity/data_stream/auditlog/manifest.yml b/packages/modsecurity/data_stream/auditlog/manifest.yml new file mode 100644 index 00000000000..9d76d46436f --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/manifest.yml @@ -0,0 +1,48 @@ +title: Modsecurity Audit Log +type: logs +release: experimental +streams: + - input: logfile + template_path: stream.yml.hbs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/modsec-audit* + - name: tz_offset + type: text + title: Timezone offset (+HH:mm format) + multi: false + required: false + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - modsec-audit + - name: preserve_original_event + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + required: true + show_user: true + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + title: Modsecurity Audit Log + description: Collect modsecurity audit logs diff --git a/packages/modsecurity/data_stream/auditlog/sample_event.json b/packages/modsecurity/data_stream/auditlog/sample_event.json new file mode 100644 index 00000000000..ae90192f65a --- /dev/null +++ b/packages/modsecurity/data_stream/auditlog/sample_event.json @@ -0,0 +1,118 @@ +{ + "@timestamp": "2021-05-14T14:38:37.000Z", + "agent": { + "ephemeral_id": "061dfa96-ca94-49ac-91b6-bdf673019894", + "hostname": "docker-fleet-agent", + "id": "825f840d-2cf2-4972-91e6-99c4735ef994", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.16.0" + }, + "data_stream": { + "dataset": "modsecurity.auditlog", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "825f840d-2cf2-4972-91e6-99c4735ef994", + "snapshot": true, + "version": "7.16.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "dataset": "modsecurity.auditlog", + "ingested": "2021-09-17T03:51:35Z", + "kind": "event", + "timezone": "+00:00", + "type": [ + "access" + ] + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "7018a7d8148499f0598830dd37987dc4", + "ip": [ + "172.18.0.7" + ], + "mac": [ + "02:42:ac:12:00:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "Core", + "family": "redhat", + "kernel": "5.11.0-34-generic", + "name": "CentOS Linux", + "platform": "centos", + "type": "linux", + "version": "7 (Core)" + } + }, + "http": { + "request": { + "method": "PUT" + }, + "response": { + "mime_type": "application/json; charset=utf-8", + "status_code": 400 + }, + "version": "1.1" + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/modsec-audit.log" + }, + "offset": 0 + }, + "source": { + "as": { + "number": 9009, + "organization": { + "name": "M247 Ltd" + } + }, + "geo": { + "city_name": "Montreal", + "continent_name": "North America", + "country_iso_code": "CA", + "country_name": "Canada", + "location": { + "lat": 45.4994, + "lon": -73.5703 + }, + "region_iso_code": "CA-QC", + "region_name": "Quebec" + }, + "ip": "37.120.205.2", + "port": 56047 + }, + "tags": [ + "modsec-audit" + ], + "url": { + "domain": "www.owayride.com", + "original": "https://www.owayride.com:443/orders/2734183/finish", + "path": "/orders/2734183/finish", + "port": 443, + "scheme": "https" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "okhttp", + "original": "okhttp/2.7.5", + "version": "2.7.5" + } +} \ No newline at end of file diff --git a/packages/modsecurity/docs/README.md b/packages/modsecurity/docs/README.md new file mode 100644 index 00000000000..e60d0f6e81f --- /dev/null +++ b/packages/modsecurity/docs/README.md @@ -0,0 +1,104 @@ +# Modsecuriy Integration + +This integration periodically fetches audit logs from [Modsecurity](https://github.com/SpiderLabs/ModSecurity/) servers. It can parse audit logs created by the HTTP server. + +## Compatibility + +The logs were tested with Modsecurity v3 with nginx connector.Change the default modsecurity logging format to json as per configuration + +``` +SecAuditLogType Serial +SecAuditLog /var/log/modsec_audit.json +SecAuditLogFormat JSON +``` + +### Audit Log + +The `Audit Log` dataset collects Modsecurity Audit logs. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.domain | Destination domain. | keyword | +| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | +| destination.port | Port of the destination. | long | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.request.referrer | Referrer for this HTTP request. | keyword | +| http.response.body.bytes | Size in bytes of the response body. | long | +| http.response.mime_type | Mime type of the body of the response. This value must only be populated based on the content of the response body, not on the `Content-Type` header. Comparing the mime type of a response with the response's Content-Type header can be helpful in detecting misconfigured servers. | keyword | +| http.response.status_code | HTTP response status code. | long | +| http.version | HTTP version. | keyword | +| input.type | Input type | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.offset | Log offset | long | +| message | human-readable summary of the event | text | +| modsec.audit.detail | Details message of the audit event. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.path | Path of the request, such as "/search". | wildcard | +| url.port | Port of the request, such as 443. | long | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | +| user.name | Short name or login of the user. | keyword | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | +| user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.os.full | Operating system name, including the version or code name. | keyword | +| user_agent.os.name | Operating system name, without the version. | keyword | +| user_agent.os.version | Operating system version as a raw string. | keyword | +| user_agent.version | Version of the user agent. | keyword | + diff --git a/packages/modsecurity/img/modsec.svg b/packages/modsecurity/img/modsec.svg new file mode 100644 index 00000000000..3001b7e70cb --- /dev/null +++ b/packages/modsecurity/img/modsec.svg @@ -0,0 +1 @@ + diff --git a/packages/modsecurity/manifest.yml b/packages/modsecurity/manifest.yml new file mode 100644 index 00000000000..35fb06a4678 --- /dev/null +++ b/packages/modsecurity/manifest.yml @@ -0,0 +1,28 @@ +format_version: 1.0.0 +name: modsecurity +title: "ModSecurity Audit" +version: 0.1.0 +license: basic +description: "ModSecuirty Audit Log Integration" +type: integration +categories: + - security + - web +release: experimental +conditions: + kibana.version: "^7.16.0" +icons: + - src: /img/modsec.svg + title: ModSecurity + size: 32x32 + type: image/svg+xml +policy_templates: + - name: modsec + title: ModSecurity audit logs + description: Collect modsecurity audit logs + inputs: + - type: logfile + title: Collect logs from modsecurity instances + description: Collecting modsecurity audit logs +owner: + github: elastic/integrations