From 54591696452e6d5b0197d7533eb6e322241c6022 Mon Sep 17 00:00:00 2001 From: Lei Qiu Date: Thu, 14 Nov 2019 16:56:41 -0800 Subject: [PATCH] Add attack_pattern_kql field to MISP threat indicators (#14470) * Add attach_pattern_kql to MISP threat intel input --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 15 +++- x-pack/filebeat/module/misp/fields.go | 2 +- .../module/misp/threat/_meta/fields.yml | 10 ++- .../module/misp/threat/config/pipeline.js | 80 +++++++++++-------- .../misp/threat/test/misp-test.json.log | 1 + .../test/misp-test.json.log-expected.json | 27 ++++++- 7 files changed, 98 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 97cadebd8be9..7d3828b8ea99 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -398,6 +398,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add more filesets to Zeek module. {pull}14150[14150] - Add `index` option to all inputs to directly set a per-input index value. {pull}14010[14010] - Remove beta flag for some filebeat modules. {pull}14374[14374] +- Add attack_pattern_kql field to MISP threat indicators. {pull}14470[14470] *Heartbeat* - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 35e8bb83614d..ed9feecd135b 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -11827,7 +11827,20 @@ The attack_pattern for this indicator is a STIX Pattern as specified in STIX Ver type: keyword -example: [source.ip = '198.51.100.1/32'] +example: [destination:ip = '91.219.29.188/32'] + + +-- + +*`misp.threat_indicator.attack_pattern_kql`*:: ++ +-- +The attack_pattern for this indicator is KQL query that matches the attack_pattern specified in the STIX Pattern format. + + +type: keyword + +example: destination.ip: "91.219.29.188/32" -- diff --git a/x-pack/filebeat/module/misp/fields.go b/x-pack/filebeat/module/misp/fields.go index d29b654ebe44..17dca85aadf6 100644 --- a/x-pack/filebeat/module/misp/fields.go +++ b/x-pack/filebeat/module/misp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetMisp returns asset data. // This is the base64 encoded gzipped contents of module/misp. func AssetMisp() string { - return "eJzkPF1v27qS7/0V89YWkL2nt7jANsAuELT3AnnoadFkz1lgsTBoaizNCUVqSUqp99cv+CFZkmVFduMkxha49ySixJkh5/sjC7jH7RUUZMo3AJaswCv4enP7/Q1AioZrKi0peQX//gYA4KtKK4GwURpyJlNBMgOba2QWSG6ULph7GzZaFX6X5RuADaFIzZX/fgGSFdjCc//stsQryLSqmicjcN2/f/p9dnuPAF7Gl7sgu2CZtYzfr0pmLWrZLreEX/t1+L63bnKl7VWDgs2ZBYEWtqoCY5XGuDPEnU3n0wPUdCgqtaopRTBVWSpt/emaEjlttu58uwfL1qqyQ2DLLiF7pzl2Hp2l5mgo7aEmsEZxBVxp7D0P+9/j9kHp/hcTdALcpCgtbQg1qA3YHHeXlxJnVunlFHLu/8+J3u+swAax/uFOotXZcww7/GlRppiOYGjxp52P3pfd4glY3pMQK54zkqsyZwbNkbgefZp3OXqg4IGCB/rOvAer4CEnnoPNyQwocFep0ZRKpmaEmoYWzoqSUTYiup/3Vx4V2ma3roidV3IbiJcpsw32r0ZWZyH0glI6Cz8m6ASpPA63a+GEjFmq0UM1UBlMnURSuOVtEMpZCG9IG7syiKPneeCeU2bxOBViqcAgtR61RsDhgZmAAjgUEiAJP/75+ePHj58giNsk8oK9NO4Og5NQV+u/kLsrPKd43Tl8ve6BFDck0XhObih465QeFUxvIVNMJDukErcvaUxBVZarAhNQGkgGDgbcbJCP0deqdlVpgyu1WTE+kNVGxfs3nGxdD994VNWPqOJ2twBvOU/zX++hAc6YybgLWHaPEpBsjtrJV6mxRmn9G8HiKf88Gjv3427JI07G8UWpVabRXKahGB7RixoMJ48OxpjKOxLVFzQlM/Dc3bajz273Jehmf+UEyWn2nykxDVDgTILGUqPxAsFtxYQPAGpKKyaMUxgZk/S/Hpj/NTC7SYAZeEAh3H+5YMagcUcx6+OLlKCbvTN+dZIzC8UXlJhZ+DW8vPJcde7DdJ869KJA7ByEVkbCFms0CeAyWybOOOy43DF1l82X8K1ECX8oztawaIlZeGIWqoZpT2iN4lmCQUHGOrK1EmjGqC5ROyVjlj1EAfAnK0qnuIa7fv7HtynSDHKr9LPSFkGOUbdGoWRmwKrhfcm0MlZvF+Hjxy6MK2kZt6vxaPXpZcqRF2H2rMA7XBSMRAJlriSCrIo16gTQ8uV7HwL3yJ8yVNLqypCSK4N2xFo1y3DbWz7BZPW2muvpyf5n3s8LtgNTMOivnaU1auPc4TXmrCalgcnUeXeq0hwNPJDNgauiUN6nK1FbaoSAjOMNwjpo1zWC0jxHYzWzmMJ6CwwMyUxgX+ov0qCN38Drs2qz8XxJ0zYbyVeTXDgG6ekEwwTevxyo9wX+jJmG5yXi5JSDi/HPyzwO/5yyPGwcAAY2H1KSeJzENoEHRyrTjmbcgtU+2WoVpGqSmEYprzyoM1NFppc/6epvJiKxzHYT4v1rs9uSOBNiCw9K35uB5xDyBYuGooXfb6HqKfJj3mZVKEv183gPTapIIzNKJrAD7eOzstKlMuhMJ8l05BTGqd7t8gjFBrmSaZ/m83NzCzWSbbp0my7h5okpb6urTDywnrFsCrt7Cyf4UnGTmV5UfDs4T038c3f3vfWAmDAK7qV6kC64L5ggTqoywFWK3pfaPTJqY91mCWjcoHO2FbCQKWNFJ3lmUDv/iaRfN1tjsUigMpWXJ65q1NbrEeebOen0WUrvz3FVlFoVZEJl2zvAm2DGmCC7Tfy7mfY/OmevZiTYmtxaY51r4paKt8ZpcpYAK0tBvHP7zglk1gEIqMG7b7fv/cIGlM1RP5BxpEvlFZvSkJLRVWkbnAKAi3QD95jn1TmAczB8QddvDnqHI/tzJDOitoE1Ov5sEhjpEgbf9BRa/GjhUV2oeu9tlnpJXzN+nyqlk7WySZoqk6RalSXqBH+WQpFd3JNN7nErVJahTjSTRhWNkiiUxQXjHI1ZWK3+YjJpTWb8PlgjrbIKXRheOcletIpGK2UdADBcI8oFZ6WtNCam3Pr1uGlNujLJg9LF1LVMF+KDmr0CYzXJ7DlTF8NavfFBfMczaXW4NJZJjj6Tuw7yMxHeS2VHbNDv/adnN0DXHmKwPy4O94pee+RCdoNkr1ZfoxdSpxdyFCU4PnFHE42BX3OKWjqnDgqH4u3dzX/CN18IM/Du9ss345Zuf3wz72O1p2kUYGlK0QVkkomtoV0qQCrbIOSNV/QbKSP3eqizjYr9xSh/dxGTistUhfMXz83817DWhJvuS8EWMMckEYnZSD+XMWjScdFTmYUbq2z+TIlQ2e2h8mDfOfaPkdyjmAb+XmncPAu2IyLrfM0gszGUxqAkfJbOqQjvyHmPYULphb62VdvXtq8A70Ln283IG48qw+G3Z240HDbpmSRmN4VPUBbM8tx9+aSdiOf3XhrDFzyYEuWi9m6JP/BwIBTj9sbJ2fUpzi1RfFGFMxt/ujNyAF+TIj6q+bJGbQ7otqfC8I8A4iT0HMizurqRA45G7BG7cAC/Xw4QjsJxg3gM6x2HXLdTL2Ll4E0zGxOUrjZaFUegdWLG1HeS9xJw8cicHq5E6lxcrqShFDWmDrWKrQUOIxWSFoWgDCXHY/OrgdxKWhrNRz4tvcN044BaqUAomaE+RHeX0CXcxGTHjoKmvrV1JlMVZC2miXvJu7IhBySV39pqRs6NCU6usybGxkvphB644+LOlThP7dhzNlij7je+zIm4ni5c9tzfYDHohN4ROdoEDV0rk1OWTxHapKv4WXXi5xZKzGSfj6DXFjQ/3tj+CPUjNBZkNa4s45b4C5L39ebuxz8goGF+4UpvJFliAq594mUG4chzSf9T4cvT3mDyK+R/0VTjYr2Fz00uGSdDtEMzSWdJ2vWh7bo1dgT6RIkPj+IYlIuKG6fY5yX8YuOy/W35G3xn2sLfYdH7jGS2nOkq/1dIyS2phH+Dtx8+/evy7x+WH377bfnhXz7+7e1/T52exGxoDKdPba2UQCbnn9qfzn4Z9Akhqytn4odBAlsbrwt7szmrvdmccdf/QA/MWbi865G1gD1xtAk1mEk9NTJ9c3Y0Y2f0blRnDqIxCGe8H16fHdno5Xq4U5jukh4GdY3pKmWWdWPnkCn4FpfhS3/5hJxpb6uZmdP2G4ed8yxq3Jre3h70AzMtIc6jC0WtkE+RaH35OgFT8dxpEqGysJ/SzSpYzTYb4glUbdnt83aNOmLgnc8ocZwdbNK+mDTogbsYC898I0xzuGeNENaYkfSZ8IaXnTf+QDJVD7uEmL+57oWf1BTzLAShTM9LSuh7PIUYX0JGfRw9AZwniYpoeDwNbUt9p2oQE5ttQNalcxm6U77+x+2dC6d8g7HHB9ZoHxAlfPCy++nTpyT+D0hyURmq5+Rwz5q6u4aU/PxDzNDv6Ykmp+vvue2m9qcSWyk3jNt9zTWhpTWWSo/0p/4YPj9BL4c9Zirk8LLxrU9cCYH+JGK7VMz67EJ02CjuqxpK9qpVVpXEOxqZ9eogagOsZ8eSplYb+h3ioFLjKCeBNVKnOJrqWDdFnKJlJMbDn4tR2MNLGtVr5y+5t4N5YeBsV0+vhJOGkex1039lKi8VbTUmcPR0hT68syvQD94NLLIIbyWxPym6vEnjrSXNRELSxhdJw02N4C18l0zcLnCcVUokdSUk6thaM6mHX7RdZAZvPFeB8LonyJ4dYsXHBMmPwugFtBXWUTU0QkVZrQWZfNzQPWk7q/ui084aWCwal6CznbdIvpmrxQrW4T644yOlB6x+2qjtwWrkk6THb3ui2itI2qbB1fe56TgeEGe0D97VI9FPv/x4zY8sPe7bru5OMy1Y95Ngx8bGAJuhv8G00/60RNtI5zv5do2CoZ/vso3O+Om+tOkZr452/YVpq9LV8wdtC+OWajI24aoo0ZKzCVxTgQuzDXYE/e8kmUhyxu9RJxTqFQvGudfPTLSPUjKZrqQVmCYydM8a6/YwKE3syHbQTLlNLGqt9CPl2pe0OGR6nOFkpPe75+xLN0dH8f+zjLi0PQO+QTeOuexsVO8KBmNdlZmOmvxU5Fmx9/Lr87rDScz+MUMp2NYcIcBumzH5zVDaJCXtxxqd74elo0Q6GVPS+GcbzYzVFbeVxkXQ5PvPmeY5WeS2cRsXobVo6jxfeGale6C/PLJiVJmTsTHtdXaizD0JkbS5Np+/FJhmGJ8xAb5ySzLz1hl/+nlGg8D6jDTYu6iMhZzV6Hshw7BvJ7t7BMf1D2SE96SSmBQkqfAWwKIuMCWn7VlaM8kxTcJ8ZUY8CfgnJKWqe61YY2L6uqeHese/Nzw0db6HZomGJ7tz0RIuqnXiVbaxiUVWJF0Mk0zVqGWBctKKXsY8Us8MzTnH3ozOvmfTeidcoeYOcqqcIyM5umhZCZVtE6ms0oR2m/RvfpExkkmJ2oz85lwZswl/hibxf3Umw6SSpcaUuGVrMVmGvKxZqf8vt9Js86yX0gCdcScjsQpozJhOBRq/PNh/oMm86Rxex8XfXxuIKyVGAvD+01MCb6XE3IBbKRECbYEZWSqYxXaOLkDrDHOEHMMuljNdex1u4sJTuf2je6FoGuGeZNq0xysl3I9HjU+5jw6GzylKYmKhNguDuiaOSW/WqcNPi4y5cMa5c7EW2044cY1x6nEwKdUdquonaReGM19CfK2x84zrv5Tg+DE6HIOsJjq3n7otLoJqTttJHDNGcfJVoHbQ9lG8f/WP1R5v7WZPwTncjx2BGy9itKbgjwPLJ9iE3lazR+N6X4UUQUHGsnsHYtxOhLBebFuDwSCk4JywOdsJQTdAZ/i60+xx2ebjwCm/Oj03H88LUXiPEdSBNrE49cP/BQAA//8merCW" + return "eJzkPG1v28jR3/MrBvclCUDqeZKgwMVACxjJHWC0uaSxe1egKITVckTNebnL213KUX99sS+kSIqiKcWyLTRAezaX3HnZed8Zp3CLmwsoyJQvACxZgRfw6er6ywuADA3XVFpS8gL+8gIA4JPKKoGwVBpWTGaCZA52pZFZILlUumDubVhqVfhdZi8AloQiMxf++xQkK7CB5/7ZTYkXkGtV1U8G4Lp/P/t9tnsPAJ7Fl9sg22CZtYzfzktmLWrZLDeEX/p1+LKzblZK24saBbtiFgRa2KgKjFUa484QdzatT/dQ06Ko1GpNGYKpylJp67lrSuS03Dj+thnLFqqyfWCzNiE73BziR2upZg1lHdQErlFcAFcaO8/D/re4uVO6+8UInQBXGUpLS0INagl2hdvDy4gzq/RsDDn3/6dE7xdWYI1Yl7mjaLX2HMIOv1mUGWYDGFr8Zqej93G7eASWtyTEnK8YyXm5YgbNgbgezM2bFXqg4IGCB/rKvAar4G5FfAV2RaZHgTtKjaZUMjMD1NS0cFaUjPIB1f2wu3Kv0ta7tVXstJpbQzxPna2xfza6OgmhJ9TSSfgxQUdo5WG4XQqnZMzSGj1UA5XBzGkkhVPeBKWchPCStLFzgzjIzz3nnDGLh5kQSwUGrfWo1QoOd8wEFMChkABJ+Przh3fv3r2HoG6jyAv21Lg7DI5CXS1+R+6O8JTqdePw9bYHMlySROMluabgpTN6VDC9gVwxkWyRSty+pDEDVVmuCkxAaSAZJBhwuUQ+RF9j2lWlDc7Vcs54T1drE+/fcLp12X/jXlM/YIqb3QK82TTLf7mDBjhnJuMuYNktSkCyK9ROv0qNa5TWvxE8nvLPo7NzP26XPOJknFyUWuUazXk6ij6LntRhOH10MIZM3oGoPqErmYDn9rQdfXazq0FXuytHaE69/0SNqYECZxI0lhqNVwhuKyZ8ArCmrGLCOIORM0n/8cD8r0HYTQLMwB0K4f7LBTMGjWPFpI/PUoOudnj87DRnEopPqDGT8Ktlee6l6tTMdJ869KJCbAOERkfCFgs0CeAsnyXOOWyl3Al1W8xn8LlECb8qzhaQNsSknphUrWE8ElqgeJRkUJCxjmytBJohqkvUzsiYWQdRAPzGitIZrv6uH376PEaaQW6VflTaIsgh6hYolMwNWNU/L5lVxupNGj6+78C4kpZxOx/OVh9epxx5EWbHC7zCtGAkEihXSiLIqligTgAtn732KXCH/DFHJa2uDCk5N2gHvFW9DNed5SNcVmerqZGe7H7m47zgOzADg/7YWbZGbVw4vMAVW5PSwGTmojtVaY4G7siugKuiUD6mK1FbqpWAjJMNwnWwrgsEpfkKjdXMYgaLDTAwJHOBXa0/S4c2fALPz6tNxvMpXdtkJJ9NceEQpMcLDCN4f3ei3lX4E1YaHpeIo0sOLsc/rfA4/FeUr8LGAWAQ8z4licdJbBK4c6Qy7WjGDVjti61WQaZGiamN8tyDOjFVZDr1k7b9ZiISy2y7IN49NrspiTMhNnCn9K3pRQ6hXpDWFKV+v1Stx8iPdZt5oSytHyd6qEtFGplRMoEtaJ+flZUulUHnOklmA1wYpnq7yz0UG+RKZl2aTy/NDdRItmnTbdqEmwemvLldZeKOdZxlfbG7s3BELBU3mRhFxbdD8FTnPzc3X5oIiAmj4FaqO+mS+4IJ4qQqA1xl6GOp7SOjltZtloDGJbpgWwELlTJWtIpnBrWLn0j6dbMxFosEKlN5feJqjdp6O+JiM6edvkrp4zmuilKrgky42fYB8DK4MSbIbhL/bq79jy7YWzMSbEFurfbOa+KWipfGWXKWACtLQbx1+i4IZNYBCKjBq8/Xr/3CEpRdob4j40iXyhs2pSEjo6vS1jgFAGcZBu4Iz7MLAKdg+ISh3xT09mf2pyhmRGsDC3TyWRcwshn0vukYtPhR6lFN1XrnbZZ5TV8wfpsppZOFskmWKZNkWpUl6gS/lUKRTW/JJre4ESrPUSeaSaOK2kgUymLKOEdjUqvV70wmjcuM3wdvpFVeoUvDK6fZaWNotFLWAQDDNaJMOSttpTEx5cavx03XpCuT3CldjB3L+EV8MLMXYKwmmT9m6aJ/V298Et+KTBobLo1lkqOv5C6C/oyk91LZAR/0S/fpyR3QpYcY/I/Lw72h1x65UN0g2bmrX6NXUmcXVihKcHLiWBOdgV9zhlq6oA4Kh+L1zdU/4bO/CDPw6vrjZ+OWrr9+Nq/jbU/dKMCyjGIIyCQTG0PbUoBUtkbIO68YN1JO7vVwzzao9mdj/N1BjBouUxUuXjy18F/CQhMu2y8FX8CckEQkJiP9WM6gLsfFSGUSbqyyq0cqhMp2D5UH+8qJf8zk7sU0yPdc4/JRsB1QWRdrBp2NqTQGI+GrdM5E+EDORwwjRi/0tc2bvrZdA3gTOt+uBt641xj2vz1xo2G/Sc8ksbopfIGyYJav3JcP2ol4+uildnwhgilRpmsflniGB4ZQzNvrIGfbpzj1iuKjKpzb+M3xyAF8Tob4oObLNWqzx7Y9FIa/BhBHoedAnjTUjRJwMGL3+IU9+H13gnAQjkvEQ0TvMOTanXoRKwdvXNiYoGy+1Ko4AK0jK6a+k7xTgIssc3a4EpkLcbmShjLUmDnUKrYQ2M9USFoUgnKUHA+trwZyK2lpsB75sPT2y409aqUCoWSOeh/dbUJncBWLHVsK6vutjXOZqiBrMUvcSz6UDTUgqfzWVjNyYUwIcp03MTYeSiv1wK0Ut47ERWqH8tngGnW38WVKxvVw6bKX/hqLXif0lsjBJmhoe5kV5asxQutyFT+pTfzQQImV7NMR9NyS5vsb2++hfoDGgqzGuWXcEn9C8j5d3Xz9CQIa5juO9EqSJSbg0hdeJhCOfCXpjwqfnvYak+8h/6OmNaaLDXyoa8k4mqLtm0k6SdGuC23brbEl0BdKfHoUx6BcVlwHxb4u4RfrkO3t7P/hC9MW/gRp5zOS+WxiqPyvDI0l6TOPCyrhz/Dy/ZvZ2zfvZ2/fz978+OP/vXv78t/TeTi//eMQb3o6Pv7173+DPyrUsdPLp0sxr+h93+GvrfPTL9vdnYObyM0WM2dUXsAPfV7+MMZKiXk/uhhn30IpgUxOZ99vLiAw6CtsVlcuZupnXWxhvHPpDDvNd4adhnOpPU1FJzEb7RC3AeyJo2W41Bo1/APjTCdHM7aab2efpiAaqxqMd+sVJ0c2pg0e7him2yqSQb3GbJ4xy1r7xtLL57gMH7vLRxShO1tNLEU33zjsXKi2xo3p7O1B3zHTEOJC5HBLGApUEq3vB0jAVHzlTLNQedhP6XoVrGbLJfEEquYe88NmgTpi4KP5qHGc7e16P5u68p6zGMp3fWdRzdyTplwLzEn6q4Vall16c0cyU3fbCqM/ufaBH9Vl9CgEocxOS0poJD2GGH8nj/owegI4TxIV0fF4GpoZhdY1TKwUNxlum85ZaPf59I/rG5ef+o5tjw8s0N4hSnjjdff9+/dJ/B+Q5KIytJ5SFD9pLfQSMvIDJfHKY8dO1EVyf85Ne7rnSuxNXTJudy3XiJXWWCo90PD7tf/8CLsc9phokMPLxveScSUEek7E/rNYRtvWPGCpuL8mUrJz/WdVSbxlkVnnYkktgXX8WFJffocGkjj5VWceSRCNzBmO+rqxXXPP0DISw/nk2Rjs/iEN2rXT9zA0k45hgm/boFAJVgft3cC0bmgzldeK5norSPR4y0N4Z9vx0Hs3iEga3kpiw1cMeZM6WkvqEY+kSTSSWppqxUt921HcLkicVUok60pI1LFXadQOP2n/zQTZeKwb18uOIntxiFdoJmh+VEavoI2yDpqhASrKaiHIrIYd3YP2B7svWv3BQcSicwk220WL5LvjGqxgEc6DOzlSuifqx80u773efZD7huuOqnZueG3dMewbB3Wct4hD73vP6p7sp3ufe8kPvMvd9V3tnSZ6sPYnwY8NzVXWU5S98bHd8ZOmM9G3Rm47L0OD5Hk7nWHuPrXrGb5ubscL416lbef3+hbGLa3J2ISrokRLzidwTQWmZhP8CPrfSTKRrBi/RZ1QuABKGefePjPRPMrI5LqSVmCWhEJTaqzbw6A0scXdQTPlJrGotdL33H8/pcch05EMpyOd371kn7s7Okj+H2VmqGnC8B3PcW5o66M6R9Cbk6vMeNbkx0xPir3XX18o74+2dtkMpWAbc4ACu22G9DdHaZOMtJ8TdbEflo4S6XRMSeOfLTUzVlfcVhrTYMl3nzPNV2SR2zpsTEOv1hg/n3gIqM3Q754BMqpckbGx7HVyoswtCZE0tTZfvxSY5RifMQH+Kpxk7r0zfvMDogaBdQWpt3dRGQsrtkbfXBqmp1vV3QMkrsuQAdmTSmJSkKTCewCLusCMnLVn2ZpJjlkSBlZz4knAPyEp1brT2zakps97HKvD/p1prDH+7hvO6nN2G6IlXFSLxJtsYxOLrEjaGCa5WqOWBcpRL3oeA14dNzSFj52hp93IpolOuELNHeRMuUBGcnTZshIq3yRSWaUJ7SbpnnyaM5JJidoM/OZCGbMMf9cn8X/GJ8ekkqXGjLhlCzF6r3tew2f/K6dSb/Ooh1IDnXAmA7kKaMyZzgQav9zbv2fJvOvsH8fZn1+TiCslBhLw7tNjEm+lxNSEWykREm2BOVkqmMVmMDFAa03HhBrDNpczbX8dTuLMS7ld1j1RNo1wSzKr5w2UEu7Hg+bR3Ed70+cMJTGRqmVqUK+JY9IZHmvJU5ozl864cC7exTYjY1xjHCPtjZ61p9S6RdrUcOavEJ9r7jzh+M8lOb6PDicg85FW+IfuM4ygam47jWPGKE7+FqiZXL4X7+/967+He7vJY4UO90NnCocvMRpX8Oue5SN8QmerybOGna9CiaAgY9mtAzHsJ0JaLzaNw2AQSnBO2ZzvhGAboDXN3mr2OG/3sYfLz87OTcfzTAzefQS1oI0sjv3w3wAAAP//tGgQig==" } diff --git a/x-pack/filebeat/module/misp/threat/_meta/fields.yml b/x-pack/filebeat/module/misp/threat/_meta/fields.yml index bc0dc7782fea..293cf76b0b08 100644 --- a/x-pack/filebeat/module/misp/threat/_meta/fields.yml +++ b/x-pack/filebeat/module/misp/threat/_meta/fields.yml @@ -409,7 +409,15 @@ description: > The attack_pattern for this indicator is a STIX Pattern as specified in STIX Version 2.0 Part 5 - STIX Patterning. example: > - [source.ip = '198.51.100.1/32'] + [destination:ip = '91.219.29.188/32'] + + - name: attack_pattern_kql + level: core + type: keyword + description: > + The attack_pattern for this indicator is KQL query that matches the attack_pattern specified in the STIX Pattern format. + example: > + destination.ip: "91.219.29.188/32" - name: negate level: core diff --git a/x-pack/filebeat/module/misp/threat/config/pipeline.js b/x-pack/filebeat/module/misp/threat/config/pipeline.js index 598ff57534d2..51b3912bfb08 100644 --- a/x-pack/filebeat/module/misp/threat/config/pipeline.js +++ b/x-pack/filebeat/module/misp/threat/config/pipeline.js @@ -34,6 +34,7 @@ var threat = (function () { var setAttackPattern = function (evt) { var indicator_type = evt.Get("json.type"); var attackPattern; + var attackPatternKQL; var arr; var ip; var filename; @@ -48,36 +49,35 @@ var threat = (function () { } else { asn = v; } - attackPattern = '[' + 'as.number = ' + '\'' + asn + '\'' + ']'; - evt.Put("as.number", asn); + attackPattern = '[' + 'source:as:number = ' + '\'' + asn + '\'' + ' OR destination:as:number = ' + '\'' + asn + '\'' + ']'; + attackPatternKQL = 'source.as.number: ' + asn + ' OR destination.as.number: ' + asn; break; case 'btc': - attackPattern = '[' + 'bitcoin.address = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'bitcoin:address = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'bitcoin.address: ' + '"' + v + '"'; break; case "domain": - attackPattern = '[' + 'dns.question.name = ' + '\'' + v + '\'' + ' OR url.domain = ' + '\'' + v + '\'' + ']'; - evt.Put("dns.question.name", v); - evt.Put("url.domain", v); + attackPattern = '[' + 'dns:question:name = ' + '\'' + v + '\'' + ' OR url:domain = ' + '\'' + v + '\'' + ' OR source:domain = ' + '\'' + v + '\'' + ' OR destination:domain = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'dns.question.name: ' + '"' + v + '"' + ' OR url.domain: ' + '"' + v + '"' + ' OR source.domain: ' + '"' + v + '"' + ' OR destination.domain: ' + '"' + v + '"'; break; case "domain|ip": arr = v.split("|"); if (arr.length == 2) { var domain = arr[0]; ip = arr[1].split("/")[0]; - attackPattern = '[' + '(' + 'dns.question.name = ' + '\'' + domain + '\'' + ' OR url.domain = ' + '\'' + domain + '\'' + ')' + - ' AND ' + '(' + 'source.ip = ' + '\'' + ip + '\'' + ' OR destination.ip = ' + '\'' + ip + '\'' + ')' + ']'; - evt.Put("dns.question.name", domain); - evt.Put("url.domain", domain); - evt.Put("source.ip", ip); - evt.Put("destination.ip", ip); + attackPattern = '[' + '(' + 'dns:question:name = ' + '\'' + domain + '\'' + ' OR url:domain = ' + '\'' + domain + '\'' + ')' + + ' AND ' + '(' + 'source:ip = ' + '\'' + ip + '\'' + ' OR destination:ip = ' + '\'' + ip + '\'' + ')' + ']'; + attackPatternKQL = '(' + 'dns.question.name :' + '"' + domain + '"' + ' OR url.domain: ' + '"' + domain + '"' + ')' + ' AND ' + '(' + 'source.ip: ' + '"' + ip + '"' + ' OR destination.ip: ' + '"' + ip + '"' + ')'; } break; case 'email-src': - attackPattern = '[' + 'user.email = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'user:email = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'user.email: ' + '"' + v + '"'; evt.Put("user.email", v); break; case "filename": - attackPattern = '[' + 'file.path = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'file:path = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'file.path: ' + '"' + v + '"'; evt.Put("file.path", v); break; case "filename|md5": @@ -85,7 +85,8 @@ var threat = (function () { if (arr.length == 2) { filename = arr[0]; var md5 = arr[1]; - attackPattern = '[' + 'file.hash.md5 = ' + '\'' + md5 + '\'' + ' AND file.path = ' + '\'' + filename + '\'' + ']'; + attackPattern = '[' + 'file:hash:md5 = ' + '\'' + md5 + '\'' + ' AND file:path = ' + '\'' + filename + '\'' + ']'; + attackPatternKQL = 'file.hash.md5: ' + '"' + md5 + '"' + ' AND file.path: ' + '"' + filename + '"'; evt.Put("file.hash.md5", md5); evt.Put("file.path", filename); } @@ -95,7 +96,8 @@ var threat = (function () { if (arr.length == 2) { filename = arr[0]; var sha1 = arr[1]; - attackPattern = '[' + 'file.hash.sha1 = ' + '\'' + sha1 + '\'' + ' AND file.path = ' + '\'' + filename + '\'' + ']'; + attackPattern = '[' + 'file:hash:sha1 = ' + '\'' + sha1 + '\'' + ' AND file:path = ' + '\'' + filename + '\'' + ']'; + attackPatternKQL = 'file.hash.sha1: ' + '"' + sha1 + '"' + ' AND file.path: ' + '"' + filename + '"'; evt.Put("file.hash.sha1", sha1); evt.Put("file.path", filename); } @@ -105,71 +107,85 @@ var threat = (function () { if (arr.length == 2) { filename = arr[0]; var sha256 = arr[1]; - attackPattern = '[' + 'file.hash.sha256 = ' + '\'' + sha256 + '\'' + ' AND file.path = ' + '\'' + filename + '\'' + ']'; + attackPattern = '[' + 'file:hash:sha256 = ' + '\'' + sha256 + '\'' + ' AND file:path = ' + '\'' + filename + '\'' + ']'; + attackPatternKQL = 'file.hash.sha256: ' + '"' + sha256 + '"' + ' AND file.path: ' + '"' + filename + '"'; evt.Put("file.hash.sha256", sha256); evt.Put("file.path", filename); } break; case 'github-username': - attackPattern = '[' + 'github.username = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'user:name = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'user.name: ' + '"' + v + '"'; break; case "hostname": - attackPattern = '[' + 'source.domain = ' + '\'' + v + '\'' + ' OR destination.domain = ' + '\'' + v + '\'' + ']'; - evt.Put("source.domain", v); - evt.Put("destination.domain", v); + attackPattern = '[' + 'source:domain = ' + '\'' + v + '\'' + ' OR destination:domain = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'source.domain: ' + '"' + v + '"' + ' OR destination.domain: ' + '"' + v + '"'; break; case "ip-dst": ip = v.split("/")[0]; - attackPattern = '[destination.ip = ' + '\'' + ip + '\'' + ']'; + attackPattern = '[destination:ip = ' + '\'' + ip + '\'' + ']'; + attackPatternKQL = 'destination.ip: ' + '"' + ip + '"'; evt.Put("destination.ip", ip); break; case "ip-dst|port": arr = v.split("|"); if (arr.length == 2) { - attackPattern = '[destination.ip = ' + '\'' + arr[0] + '\'' + ' AND destination.port = ' + '\'' + arr[1] + '\'' + ']'; + attackPattern = '[destination:ip = ' + '\'' + arr[0] + '\'' + ' AND destination:port = ' + '\'' + arr[1] + '\'' + ']'; + attackPatternKQL = 'destination.ip: ' + '"' + arr[0] + '"' + ' AND destination.port: ' + arr[1]; evt.Put("destination.ip", arr[0]); + evt.Put("destination.port", arr[1]); } break; case "ip-src": ip = v.split("/")[0]; - attackPattern = '[' + 'source.ip = ' + '\'' + ip + '\'' + ']'; + attackPattern = '[' + 'source:ip = ' + '\'' + ip + '\'' + ']'; + attackPatternKQL = 'source.ip: ' + '"' + ip + '"'; evt.Put("source.ip", ip); break; case "link": - attackPattern = '[' + 'url.full = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'url:full = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'url.full: ' + '"' + v + '"'; evt.Put("url.full", v); break; case "md5": - attackPattern = '[' + 'file.hash.md5 = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'file:hash:md5 = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'file.hash.md5: ' + '"' + v + '"'; evt.Put("file.hash.md5", v); break; case 'regkey': attackPattern = '[' + 'regkey = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'regkey: ' + '"' + v + '"'; break; case "sha1": - attackPattern = '[' + 'file.hash.sha1 = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'file:hash:sha1 = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'file.hash.sha1: ' + '"' + v + '"'; evt.Put("file.hash.sha1", v); break; case "sha256": - attackPattern = '[' + 'file.hash.sha256 = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'file:hash:sha256 = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'file.hash.sha256: ' + '"' + v + '"'; evt.Put("file.hash.sha256", v); break; case "sha512": - attackPattern = '[' + 'file.hash.sha512 = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'file:hash:sha512 = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'file.hash.sha512: ' + '"' + v + '"'; evt.Put("file.hash.sha512", v); break; case "url": - attackPattern = '[' + 'url.full = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'url:full = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'url.full: ' + '"' + v + '"'; evt.Put("url.full", v); break; case 'yara': - attackPattern = '[' + 'yara.rule = ' + '\'' + v + '\'' + ']'; + attackPattern = '[' + 'yara:rule = ' + '\'' + v + '\'' + ']'; + attackPatternKQL = 'yara.rule: ' + '"' + v + '"'; break; } - if (attackPattern == undefined) { + if (attackPattern == undefined || attackPatternKQL == undefined) { evt.Put("error.message", 'Unsupported type: ' + indicator_type); } evt.Put("misp.threat_indicator.attack_pattern", attackPattern); + evt.Put("misp.threat_indicator.attack_pattern_kql", attackPatternKQL); }; var pipeline = new processor.Chain() diff --git a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log index 3fcb49fda429..cba6c830428f 100644 --- a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log +++ b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log @@ -1,3 +1,4 @@ {"id":"1","event_id":"1","object_id":"0","object_relation":null,"category":"Network activity","type":"ip-dst","to_ids":false,"uuid":"5d2cb906-eff4-40f0-9f1d-10eb7d6a0c26","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"98.235.162.24","Event":{"org_id":"1","distribution":"3","id":"1","info":"Tor exit nodes feed","orgc_id":"2","uuid":"58dcfe62-ed84-4e5e-b293-4991950d210f"}} {"id":"2","event_id":"2","object_id":"0","object_relation":null,"category":"Payload delivery","type":"md5","to_ids":true,"uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"89357a1b2e32f2b9bddff94b8136810b","Event":{"org_id":"1","distribution":"3","id":"1","info":"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass","orgc_id":"2","uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81"}} {"id":"3","event_id":"3","object_id":"0","object_relation":null,"category":"Payload delivery","type":"filename","to_ids":true,"uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de","Event":{"org_id":"1","distribution":"3","id":"1","info":"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass","orgc_id":"2","uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81"}} +{"id":"4","event_id":"4","object_id":"0","object_relation":null,"category":"Bad Domain","type":"domain","to_ids":true,"uuid":"563b3ea6-b26c-401f-a68b-4d84950d210b","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de","Event":{"org_id":"4","distribution":"3","id":"4","info":"OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman","orgc_id":"2","uuid":"563b3ea6-b26c-401f-a68b-4d84950d210b"}} diff --git a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json index 67e35fd4bed0..f0823c4e027a 100644 --- a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json +++ b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json @@ -17,7 +17,8 @@ "input.type": "log", "log.offset": 0, "message": "98.235.162.24", - "misp.threat_indicator.attack_pattern": "[destination.ip = '98.235.162.24']", + "misp.threat_indicator.attack_pattern": "[destination:ip = '98.235.162.24']", + "misp.threat_indicator.attack_pattern_kql": "destination.ip: \"98.235.162.24\"", "misp.threat_indicator.description": "Tor exit nodes feed", "misp.threat_indicator.feed": "misp", "misp.threat_indicator.id": "58dcfe62-ed84-4e5e-b293-4991950d210f", @@ -35,7 +36,8 @@ "input.type": "log", "log.offset": 460, "message": "89357a1b2e32f2b9bddff94b8136810b", - "misp.threat_indicator.attack_pattern": "[file.hash.md5 = '89357a1b2e32f2b9bddff94b8136810b']", + "misp.threat_indicator.attack_pattern": "[file:hash:md5 = '89357a1b2e32f2b9bddff94b8136810b']", + "misp.threat_indicator.attack_pattern_kql": "file.hash.md5: \"89357a1b2e32f2b9bddff94b8136810b\"", "misp.threat_indicator.description": "OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass", "misp.threat_indicator.feed": "misp", "misp.threat_indicator.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81", @@ -53,11 +55,30 @@ "input.type": "log", "log.offset": 987, "message": "f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de", - "misp.threat_indicator.attack_pattern": "[file.path = 'f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de']", + "misp.threat_indicator.attack_pattern": "[file:path = 'f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de']", + "misp.threat_indicator.attack_pattern_kql": "file.path: \"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\"", "misp.threat_indicator.description": "OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass", "misp.threat_indicator.feed": "misp", "misp.threat_indicator.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81", "misp.threat_indicator.type": "filename", "service.type": "misp" + }, + { + "@timestamp": "2017-03-30T12:54:26.000Z", + "event.category": "threat-intel", + "event.dataset": "misp.threat", + "event.module": "misp", + "event.type": "indicator", + "fileset.name": "threat", + "input.type": "log", + "log.offset": 1551, + "message": "f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de", + "misp.threat_indicator.attack_pattern": "[dns:question:name = 'f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de' OR url:domain = 'f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de' OR source:domain = 'f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de' OR destination:domain = 'f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de']", + "misp.threat_indicator.attack_pattern_kql": "dns.question.name: \"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\" OR url.domain: \"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\" OR source.domain: \"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\" OR destination.domain: \"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\"", + "misp.threat_indicator.description": "OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman", + "misp.threat_indicator.feed": "misp", + "misp.threat_indicator.id": "563b3ea6-b26c-401f-a68b-4d84950d210b", + "misp.threat_indicator.type": "domain", + "service.type": "misp" } ] \ No newline at end of file