From 0f88063cefc5c88e6a58d37c854f097c77e260e3 Mon Sep 17 00:00:00 2001 From: Josef Hlawatschek <43576722+hlawatjt@users.noreply.github.com> Date: Fri, 8 Nov 2019 13:50:35 +0200 Subject: [PATCH 01/27] Release to https://pypi.org/project/maltego-trx/ --- maltego_trx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maltego_trx/__init__.py b/maltego_trx/__init__.py index 136f674..33b4ad5 100644 --- a/maltego_trx/__init__.py +++ b/maltego_trx/__init__.py @@ -1 +1 @@ -VERSION = "1.3.4" \ No newline at end of file +VERSION = "1.3.5" From fb1ae7ce4feeeb18710df811fc85b932457cec85 Mon Sep 17 00:00:00 2001 From: Philipp Dowling Date: Tue, 21 Jan 2020 15:27:19 +0100 Subject: [PATCH 02/27] make trailing slash optional on handler and fix dockerfile --- demo/apache/Dockerfile | 2 +- demo/gunicorn/Dockerfile | 2 +- maltego_trx/handler.py | 2 +- maltego_trx/server.py | 8 +++++++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/demo/apache/Dockerfile b/demo/apache/Dockerfile index 900d19c..802c9a6 100644 --- a/demo/apache/Dockerfile +++ b/demo/apache/Dockerfile @@ -21,4 +21,4 @@ COPY . /var/www/TRX/ RUN chown -R www-data:www-data /var/www/TRX/ -CMD ["python", "project.py", "runserver"] +CMD ["python3", "project.py", "runserver"] diff --git a/demo/gunicorn/Dockerfile b/demo/gunicorn/Dockerfile index 48360a8..66f56f7 100644 --- a/demo/gunicorn/Dockerfile +++ b/demo/gunicorn/Dockerfile @@ -18,4 +18,4 @@ COPY . /var/www/TRX/ RUN chown -R www-data:www-data /var/www/TRX/ -CMD ["python", "project.py", "runserver"] +CMD ["python3", "project.py", "runserver"] diff --git a/maltego_trx/handler.py b/maltego_trx/handler.py index 3437cef..9605cb2 100644 --- a/maltego_trx/handler.py +++ b/maltego_trx/handler.py @@ -15,7 +15,7 @@ def handle_run(name, args, app): if command == "runserver": print("\n=== Maltego Transform Server: v%s ===\n" % VERSION) print_transforms() - app.run(host="0.0.0.0", port=8080, debug=True) + app.run(host="0.0.0.0", port=8080, debug=False) elif command == "list": print_transforms() elif command == "local" and len(args) > 3: diff --git a/maltego_trx/server.py b/maltego_trx/server.py index e0f1546..84474b3 100644 --- a/maltego_trx/server.py +++ b/maltego_trx/server.py @@ -9,6 +9,7 @@ URL_TEMPLATE = '/run//' +URL_TEMPLATE_NO_SLASH = '/run/' def get_exception_message(msg="An exception occurred with the transform. Check the logs for more details."): @@ -53,7 +54,7 @@ def run_transform(transform_name, client_msg): app = Flask(__name__) application = app # application variable for usage with apache mod wsgi -@app.route(URL_TEMPLATE, methods=['GET', 'POST']) + def transform_runner(transform_name): transform_name = transform_name.lower() if transform_name in mapping: @@ -68,6 +69,11 @@ def transform_runner(transform_name): return "No transform found with the name '%s'." % transform_name, 404 +# Add the route with and without the slash, since POSTs can't be redirected +app.route(URL_TEMPLATE_NO_SLASH, methods=['GET', 'POST'])(transform_runner) +app.route(URL_TEMPLATE, methods=['GET', 'POST'])(transform_runner) + + @app.route('/', methods=['GET', 'POST']) def index(): return "You have reached a Maltego Transform Server.", 200 From fa3a6fde922211d182642d4c0975a6554453f03a Mon Sep 17 00:00:00 2001 From: Philipp Dowling Date: Tue, 21 Jan 2020 15:27:57 +0100 Subject: [PATCH 03/27] bump version --- maltego_trx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maltego_trx/__init__.py b/maltego_trx/__init__.py index 33b4ad5..288108b 100644 --- a/maltego_trx/__init__.py +++ b/maltego_trx/__init__.py @@ -1 +1 @@ -VERSION = "1.3.5" +VERSION = "1.3.6" From ca27adcea2f35bde3f32ce98be35a2d650cf2ac5 Mon Sep 17 00:00:00 2001 From: PatervaMaltego Date: Wed, 29 Jan 2020 17:25:54 +0200 Subject: [PATCH 04/27] Type fixed. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cde7e0a..aa2e13a 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ The request/maltego msg object given to the transform contains the information a **Methods:** - `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object created by the method. - - `addUIMessagte(msg: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message type constant. + - `addUIMessage(msg: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message type constant. ### Entity From 907e6c8a98c7441a129a912d23c402fa2eb3f307 Mon Sep 17 00:00:00 2001 From: Philipp Dowling Date: Wed, 29 Jan 2020 16:29:20 +0100 Subject: [PATCH 05/27] Update README.md parameter name is "message" not "msg" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa2e13a..5cd19a8 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ The request/maltego msg object given to the transform contains the information a **Methods:** - `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object created by the method. - - `addUIMessage(msg: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message type constant. + - `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message type constant. ### Entity From 09565706a05320d1fac7d11a93ee64458734b65d Mon Sep 17 00:00:00 2001 From: Paul Richards Date: Mon, 29 Jun 2020 12:56:37 +0200 Subject: [PATCH 06/27] Added MaltegoCrypto from Tendai's maltego_crypto_test library. --- demo/gunicorn/project.py | 5 +- maltego_trx/oauth.py | 156 +++++++++++++++++++++++++++++++++++++++ setup.py | 3 +- 3 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 maltego_trx/oauth.py diff --git a/demo/gunicorn/project.py b/demo/gunicorn/project.py index 09ab5f6..410e445 100644 --- a/demo/gunicorn/project.py +++ b/demo/gunicorn/project.py @@ -1,11 +1,10 @@ import sys import transforms -from maltego_trx.registry import register_transform_function, register_transform_classes -from maltego_trx.server import app, application +from maltego_trx.registry import register_transform_classes +from maltego_trx.server import app from maltego_trx.handler import handle_run -# register_transform_function(transform_func) register_transform_classes(transforms) handle_run(__name__, sys.argv, app) diff --git a/maltego_trx/oauth.py b/maltego_trx/oauth.py new file mode 100644 index 0000000..99ac5ff --- /dev/null +++ b/maltego_trx/oauth.py @@ -0,0 +1,156 @@ +"""Maltego OAuth Crypto Helper""" +import base64 +from Crypto import Random +from Crypto.Cipher import PKCS1_v1_5, AES +from Crypto.Hash import SHA +from Crypto.PublicKey import RSA +from requests.auth import AuthBase + + +class MaltegoOauth: + """ + A Crypto Helper for Maltego OAuth Secrets received from the Transform Distribution Server + The TDS will send back an encrypted combination of the following : + 1. Token + 2. Token Secret + 3. Refresh Token + 4. Expires In + + Contains 1 Methods: + 1. decrypt_secrets(private_key_path="pem file", ciphertext="request.getTransformSetting('name from TDS')") + """ + + @staticmethod + def _rsa_decrypt(private_key_path=None, ciphertext=None): + """ + RSA Decryption function, returns decrypted plaintext in b64 encoding + """ + dsize = SHA.digest_size + sentinel = Random.new().read(20 + dsize) + ciphertext = base64.b64decode(ciphertext) + private_key = RSA.import_key(open(private_key_path).read()) + cipher = PKCS1_v1_5.new(private_key) + plaintext = cipher.decrypt(ciphertext, sentinel) + return plaintext + + @staticmethod + def _aes_decrypt(key=None, ciphertext=None): + """ + AES Decryption function, returns decrypted plaintext value + """ + unpad = lambda s: s[:-ord(s[len(s) - 1:])] + key = base64.b64decode(key) + ciphertext = base64.b64decode(ciphertext) + cipher = AES.new(key, AES.MODE_ECB) + plaintext = unpad(cipher.decrypt(ciphertext)).decode('utf8') + return plaintext + + @classmethod + def decrypt_secrets(cls, private_key_path=None, encoded_ciphertext=None): + """ + The TDS will send back an encrypted combination of the following : + 1. Token + 2. Token Secret + 3. Refresh Token + 4. Expires In + + This function decodes the combinations and decrypts as required and returns a dictionary with the following keys + {"token":"", + "token_secret": "", + "refresh_token": "", + "expires_in": ""} + """ + + encrypted_fields = encoded_ciphertext.split("$") + + if len(encrypted_fields) == 1: + token = cls._rsa_decrypt(private_key_path, encrypted_fields[0]) + token_fields = { + "token": token + } + + elif len(encrypted_fields) == 2: + token = cls._rsa_decrypt(private_key_path, encrypted_fields[0]) + token_secret = cls._rsa_decrypt(private_key_path, encrypted_fields[1]) + token_fields = { + "token": token, + "token_secret": token_secret + } + + elif len(encrypted_fields) == 3: + aes_key = cls._rsa_decrypt(private_key_path, encrypted_fields[2]) + token = cls._aes_decrypt(aes_key, encrypted_fields[0]) + token_secret = cls._aes_decrypt(aes_key, encrypted_fields[1]) + token_fields = { + "token": token, + "token_secret": token_secret + } + elif len(encrypted_fields) == 4: + token = cls._rsa_decrypt(private_key_path, encrypted_fields[0]) + token_secret = cls._rsa_decrypt(private_key_path, encrypted_fields[1]) + refresh_token = cls._rsa_decrypt(private_key_path, encrypted_fields[2]) + expires_in = cls._rsa_decrypt(private_key_path, encrypted_fields[3]) + token_fields = { + "token": token, + "token_secret": token_secret, + "refresh_token": refresh_token, + "expires_in": expires_in + } + elif len(encrypted_fields) == 5: + aes_key = cls._rsa_decrypt(private_key_path, encrypted_fields[4]) + token = cls._rsa_decrypt(private_key_path, encrypted_fields[0]) + token_secret = cls._rsa_decrypt(private_key_path, encrypted_fields[1]) + refresh_token = cls._rsa_decrypt(private_key_path, encrypted_fields[2]) + expires_in = cls._rsa_decrypt(private_key_path, encrypted_fields[3]) + token_fields = { + "token": token, + "token_secret": token_secret, + "refresh_token": refresh_token, + "expires_in": expires_in + } + else: + token_fields = { + "token": "", + "token_secret": "", + "refresh_token": "", + "expires_in": "" + } + + return token_fields + + +class OAuth2BearerToken(AuthBase): + """Implements OAuth2 Bearer access token authentication. + + Pass this object via the `auth` parameter to a request or a + session object in order to authenticate your requests. + + Example usage, once you have the `access_token`: + + class GreetPerson(DiscoverableTransform): + + @classmethod + def create_entities(cls, request, response): + person_name = request.Value + + private_key_path = "private_key.pem" + + encrypted_secrets = request.getTransformSetting('maltego.web.api.key.linkedin') + + token_fields = MaltegoCrypto.decrypt_secrets(private_key_path,encrypted_secrets) + + api_url = ("https://api.linkedin.com/v2/emailAddress?q=members&projection=(elements*(handle~))") + auth = OAuth2BearerToken(token_fields['token']) + result = requests.get(api_url,auth=auth) + + response.addEntity(Phrase, result.text) + """ + + def __init__(self, access_token): + self.access_token = access_token + + def __call__(self, request): + request.headers['Authorization'] = 'Bearer {}'.format( + self.access_token + ) + return request diff --git a/setup.py b/setup.py index 2ac37d7..6092967 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,8 @@ license='MIT', install_requires=[ 'flask>=1', - 'six>=1' + 'six>=1', + 'pycryptodome>=3.9.7' ], packages=[ 'maltego_trx', From 6b1a8a826be397b999c20d70ffbd809a7602279e Mon Sep 17 00:00:00 2001 From: Paul Richards Date: Mon, 29 Jun 2020 12:59:05 +0200 Subject: [PATCH 07/27] If there is a symmetric key included in encypted message, it should be used to decrypt other message parameters. --- maltego_trx/oauth.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maltego_trx/oauth.py b/maltego_trx/oauth.py index 99ac5ff..c745e6b 100644 --- a/maltego_trx/oauth.py +++ b/maltego_trx/oauth.py @@ -98,10 +98,10 @@ def decrypt_secrets(cls, private_key_path=None, encoded_ciphertext=None): } elif len(encrypted_fields) == 5: aes_key = cls._rsa_decrypt(private_key_path, encrypted_fields[4]) - token = cls._rsa_decrypt(private_key_path, encrypted_fields[0]) - token_secret = cls._rsa_decrypt(private_key_path, encrypted_fields[1]) - refresh_token = cls._rsa_decrypt(private_key_path, encrypted_fields[2]) - expires_in = cls._rsa_decrypt(private_key_path, encrypted_fields[3]) + token = cls._aes_decrypt(aes_key, encrypted_fields[0]) + token_secret = cls._aes_decrypt(aes_key, encrypted_fields[1]) + refresh_token = cls._aes_decrypt(aes_key, encrypted_fields[2]) + expires_in = cls._aes_decrypt(aes_key, encrypted_fields[3]) token_fields = { "token": token, "token_secret": token_secret, From 8d5343a2a1b0d8e1db7fab3ef1e19c7492f7ca62 Mon Sep 17 00:00:00 2001 From: Paul Richards Date: Tue, 30 Jun 2020 14:37:59 +0200 Subject: [PATCH 08/27] Added tests --- tests/__init__.py | 0 tests/test.py | 33 ++++++++++++++++++++++++ tests/test_private_key.pem | 52 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/test.py create mode 100644 tests/test_private_key.pem diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..c03f577 --- /dev/null +++ b/tests/test.py @@ -0,0 +1,33 @@ +from maltego_trx.oauth import MaltegoOauth + +okta_ciphertoken = "pZ598ZEZ7EwpBQSOJSvCZJKkcWhtbX95K7Q0f0hwbk93O+xaUB4/NegK3r54PH1NReis/Jgt4UbGc5oCuU+R7UDM1icoDUmyCmV1U78iqjHElPdDBHlxPrl2zoXBwcU1iFbukDNy49Xghy3cwwqhmEXg/FKnYUlBQl6jdf06kE1pwfHiRgF5NrJISAD7eCmUybqiAVRDW4lbLeMGk/uSrrCpDQBxd2Em/sYKBzqO87pQRPLll23G9KNxyVinHQksGgc+dI0OPYv9EQcUs7g1JFraUKUeFjHAf/OyGGrp6WH84nGqG35afkH9xsDmySBb5DZWRjef8DzSd7oRagoGq3wKBHDh700mMKf/YkR8L9cK4l1w4yNh7EfIVlCWD5yWJUx4a1lN9CvJmFs7N9A903spVVGeP8avz50raWFb5g/4XhtIpei4ylVFi49dDeVjgb0BR0I4vuU7VDmcwFweAtSfclPb5Xfp98zI9yXnr8DB31gmzf3DIkVFIIFLxZe0gG+BSIEmC7/z0L7J+YvIqoGDq8jC0Ehe9IVXYfK1yluWAsbwj62rNAzkPVSe5l+FMQFrQRXpxZFBMXzphqRAhyYKfGNX/tocoMJoLFp8O3PBRC0kIHjTNaSX8KpftNYWreEdZx6wjk2n2eeOXM5nVeSOx4uoIifRe4NUV7VYzbGm3FSRc0k1HZIGHwn1WTvn95U3crvwFN86N3k41JU+0kzSpsLxy+Y63VfPxYTbRUET/wIOf0NTfG96QCPtVKkdVh9aTqnhaN0TpGoaZQEjUqpURxbj+8mT3g6MI4vOC9Qr6vw2wIkvzehr/yKxh6vsnfUUw8UJ1YtlZI7exPsHyFLUr2J3VvpDsaemRcn1KY/uHJZ3MMHK9GNciLA6Skw8SL38mw1tJJZ1h1bMMk1IYgBZLe2ZSXXyv7JaDPq/WJmHWDaIalmXE1cKMA56qMgYGdHxs3Pai5nwEtzIGVS1OcuMnBF6sOMMBTKSFAJmlcYo8NGwjKqe/yy1cVJZe7Ucv/jy/IS3AdE1my8cDya92jDsxifsaONQMm6YIs54SIA=$f7TU33wTTxYFtbVnwJeGGw==$c64l75qukq+980EkB1tQZKEicryt6HBB0IOjeMSRQmLOYCQpTcaXlPRJ3QdS+DfRocor7OL36wLAooRqz1IaN4QQ5+6Wx5reEXFVgKtaDX777I046DM5QU3Jf+iibSbm3mXYMw0z+OknbXyGLnc7ceSaPTJdP1LkCGQ75fLMXHCLWSpqwOKHOBhqyQGwrYlj2WxPzmOfAMaIkjKZIQWzoGjrYRzwkNCQbxykJcwD5TVuVDwAHyp84zcvW0WWoUZ+rrrooJwuJJQEdiTwLZsseqklXRNso4e5eFQwH49T9IDPHkKfVusu6rLiMNgFyc18rFR1d/BYLXBu7uzMAuvQ8wZdcOtJYx+JLJmOaPI65ymGNFTpwTHYnDBTXmpW0qX2dtEglAERw1nrdl94fwsa+I/iw3H7VIivqRF3pAchfvdNF75MIEm6XH2UXY/sZD7zjMdxxwKQiaEg2bpLd2mEtqsLc0mTq/zd03ZLEUnzXsXlp8brCfDgjoVsZAJH+ElQ6wr3dxzRMlDIxWWBYFcv8LnGVQk/Vciqtee780Yh/lgttv06kyXz81dIWjumz8TqV2eV9ZpP/YxTnNzAa91fleGNah/IKhMOV8PsGy2uOnHRiUL218p1T6+Cm9B2kmL1C/2MM3080NjJVWIfqYsyTGPbR+hURK0RB/oteG5QWPs=" +private_key_path = "test_private_key.pem" + + +def testOktaDecrypt(): + decrypted_linkedIn_token = MaltegoOauth.decrypt_secrets(private_key_path, okta_ciphertoken)['token'] + expected_token = "eyJraWQiOiJ4ZzEyMmNZMXQ0NU5GM2l5YlUzUF9tWGluYkxEXzVrRHFwMkQ0VEl2RGd3IiwiYWxnIjoiUlMyNTYifQ.eyJ2ZXIiOjEsImp0aSI6IkFULklldnVNcXNJNDFqMURldEdPU1N2Q2VoNlI5eUxEQ0EydFBJdl9FZ3hDU3ciLCJpc3MiOiJodHRwczovL2Rldi02MTcyNDUub2t0YS5jb20vb2F1dGgyL2RlZmF1bHQiLCJhdWQiOiJhcGk6Ly9kZWZhdWx0IiwiaWF0IjoxNTg5NzgzNjU0LCJleHAiOjE1ODk3ODcyNTQsImNpZCI6IjBvYTR1YTZ1YlJIeFRDTGg4NHg2IiwidWlkIjoiMDB1NHU0ZzA4SDFqYXQwVnA0eDYiLCJzY3AiOlsib3BlbmlkIl0sInN1YiI6InRtQG1hbHRlZ28uY29tIn0.dAZrIZ9NCIqIx3fr8_0EexYCzarj8CC4CvWpVgZQCvfhtV08tGnMdWN8yuzADYvJUSzDz_meqIMMUCaVOpYZ5vzepr3LZfT-Xn00KoaaRcHGLPowphvsEPkpimvJqgnRmWw0e0VTH5Pfg9eyl3o2UUUsDofM-RkJNjxB4Uf0D6IyZaCyl0s_KhcXGZuh7hDGoR76UcKaCqRXqmqnOZs_GoMPIoDS5NHIze0MOK6sgKr8uiLikIhdh481n8PyWzPX2XZLUNcjogqX280so6Ki24VBloyxt5VgAJ1gV-e9SDj-9QSdQohQNDDSHbgiO4s1TUlhFKIm5UQWUImdhSuv1w" + assert decrypted_linkedIn_token == expected_token + + +def testReadKey(): + private_key = open(private_key_path).read() + assert private_key.__sizeof__() == 3292 + + +def testCipherSplit(): + encrypted_fields = okta_ciphertoken.split("$") + assert len(encrypted_fields) == 3 + + +def testToken_Field_Creation(): + encrypted_fields = okta_ciphertoken.split("$") + aes_key = MaltegoOauth._rsa_decrypt(private_key_path, encrypted_fields[2]) + token = MaltegoOauth._aes_decrypt(aes_key, encrypted_fields[0]) + token_secret = MaltegoOauth._aes_decrypt(aes_key, encrypted_fields[1]) + token_fields = { + "token": token, + "token_secret": token_secret + } + decrypted_azure_token_dict = MaltegoOauth.decrypt_secrets(private_key_path, okta_ciphertoken) + assert token_fields == decrypted_azure_token_dict diff --git a/tests/test_private_key.pem b/tests/test_private_key.pem new file mode 100644 index 0000000..d3f357f --- /dev/null +++ b/tests/test_private_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCk6q9Ra8EYERVk +lBLZP3Kt0Eqy1Kcd6d5CQ+aE7l8A07aUIednqHaKuU/3m6i6ODBqCD84K35dVHS3 +Xq4DHOFPj6M+WPR9hb8FFkLuqHSHqYdeRAI4NUsb3n9xF/ebrLP4ZM6gou4ONbgL +sR5x/SkBKfhWfHlOk7iALTdqaWjBXShLYEYAAVkauoElqr6KIoeCE0zCg6KJ6IcC +SvGCpl87kFgchHQWrkQXJCI4ABznWUuTCnQpVKg5dtnDPxTaHLsoQ4ahdLEcjaDW +lOScQ1WNbcWaDKg5sd/DnuyNm1e6PZWMf5df+OHcbMz9HHmtgIz5SHJZFyYimDyX +w7iEPjaunwia2hWkE20j+AjIqJ2smjkvpVtMCnDAhiH0bugE/yorDkRHkL0EZ5v1 +x6hC6IhbHidP3j57GnDLRji2VNGzU5N+WA4j9ZU/UpdC9Vrp81Xm3Fea8R1LpkAq +oRioLSuAxr/W37m00Mbq7Ijl9Ez3rrkiWlRoQ1E1E2VK3iSMNXXIoc9rhj3U2T+B +upy5vj97J1m3mX0/ufccN57e0KH/GUELetPXU21oIcJckUpwkzqJa/YHt2JSEFWl +SQW5rs2jaBcflAzJhQj3UGL6YSmZDw3HKEGn/wK33U1PE8XH7bJJdcy322Zzqj1q +kzmyKkKsn96Xn7CvjyHLD7u6jLOFSQIDAQABAoICAGqdTbnVb3+fi7T6BTVtTzYO +8juqPl+YUZeFTgGiGMjwFZiuUmsw/XGxW4E3oFzC9omVy0kE1SyA7POeweBBS2ej +9GTaHTUIwfUH7z1aqfsKHflS/hxYV7YsoTb7x5dcjvyGLw6qRjvpfpIQbx5CC8A0 +4dcHoWSrGxvCH5ErlA1trB8OnjJirLga2mL/fy7OI8xzrawSbYG6UY2p5XgRFn/r +UQsele4TuvE66uRJLmZh0/m7SF1v3VFJBH60yUY4TMY64U5/ogBTjycqGqDq5uQH +kzeD9z1VQNO2ajchthUwuv2ZfsMMovddXyhCwGbqNDj0HPh7fqvev01dumvDzJUN +fnDHKDwDPRO9oxV7769eZP82hxH4rxWA8g1uOAXDx6x6GomtdbcR4F77f6zrMp0i +hqSdeCQvfF5Byhk835U83qwJRnR2ltbFXXMJGXkuW9W+9GYVmof/8jZt/+rAK+z/ +aSrNXe/FjSYSVFjGDaGebWw44dacIDX4jNERH+dL4gjgnPWgE8iLCeP4QEu0/d3A +KRabyymz93iJGq/3tK8ttcBS0bMoJRNsQ4K1VP3cDSG62BOLnCFfbcrxyhp8bUVW +2JIRllkHXFlLYUkyI1UhkIDNIqrn+0DfD/i+Qq/fvvjgSJfRwtE79wO8700XNo10 +y1+DnizLz8wbodiVH9gBAoIBAQDSK6F+v3GK1nx9EezE0QnTJR8Mr6d0Ua6es6Z3 +hiC45kKdQFMaFY4L2MJeZrahq/niw3/zvtOa0uVa98hjs0INumDdbmXbKZH4AIDs +kA/UFZVxeoAVypGRKYCCWUby9UVl/FmJT7jabAiMLfeeihOw1s5hcDY1hnZJGo5j +TPoL4o/+6Sr+d1YbEyx/YDg8e5RfIbm+elVNrZ45HDQSNs15R1MG9rT8uoIxZmlS +IPC6uNpe0oNEcXEMljedOBJOxQjvJJMfHRHA0jWL1rAdVJAhDPfCu+wXGSCu4zQG +vc/EU2Jk9L5AbayDRuQikqQJGkPI/pwP0gALlo0sHH1zVOm9AoIBAQDI4Nn/7Dh1 +S36FdfhRl8g0AbXwzTwtUUGQw0sQfEKoBVtIXmvoWYN16ryGEuYe1upinilp8C0P +RmtYD6eE4mS57ukIPYheaamz/Uf9HZfypyhDvrfMn+OvzmiSbmiTBHXlRSsGsxbR +P/AbjhpClKH8PxLGJdeTpeYVrtXSVCLptkk/rH+U3da5zto+4mEWlu4kNrGlDhjV +RVsv5IuyNZfk+Q4/76jzKzDa86Rm2YdIlc5TjoxarVH1vBQmpiKx9/be23xY2WsE +2673PUcNz5O6V/+OTIypzwwxywZ4LTOBpL9uzffmN/yGbgT3Tg1RFmZB1702KvYH +C3vi9SiofzR9AoIBAAoNukD06XqJvhTBicD0evLVwMF7mZgP3DmNQHZRPTl7Ek6x +aAhEZbIdYVbgtPXQ4zg8v98qDrdGRWBvn+9dANjlRILzJ/4u4+OoKoKmdYtgqPBv +urbQJNx7zsDtgl5W60Xwp1vRK3ePWW1TOZgk5MI91EuG8aDn2LqwgYUwhnmREfBQ +uRTJIp5S8Xr6YFZMVxGh7F+3PGNl3b6/oaIJaxTVG5ymqou4ZEf2rS0XlExqUU/d +5BefEZhXizuDFiUcecvuxPblDhdaNuOElpIgnHBoTWXMVYPZWN3k0nVMGSc8EeXg +a0Vruafh+UHKH/yre/iebVq4YfYr8n7csgeVVUUCggEAP6ZvrRYOdawsNOHCgygS ++deo7No7PSjIG7Sl7l1RSagY2n+AtajXbN+qSNloLVFwBzuSZ80Amhx4Gvkq3YJW +5Et9b2z/7tqQOUYCL4PXB75LlduypZXsMWK34940KJF7QeB+16qbikY2MKUAUSSD +h0f9DOgkvNYOZ8R0YCbkwSVPZGumKWd5iHqw0Mgud1fvsW3bMC+dUsadNDm4wgkV +TipUh5HK+PIwktAswaIfqbI+JF/AvWK526FyySRPThECGm91oTmTHYD2mcTC5O9n +Id6MTWyYDZ5bgNOSAzZfYa7wMY32BO6sh3QJAsuqkI0GbcqMW8OVHXpYEPwZm/pi +iQKCAQBrOiqj9sgl1Z2JFzR/OnpFYuqFmiKaz3qeZLOLDEFOlhJ+VPVu+Bza6E7I +TFDoXU+IG4mu0FmCffANEVnnD39MF7iQ3quk8frpFxajcu3EC3gG9U0Nn/AAgCkm +dqZWg4vb3uIl0VY6Gxzg6BfoRK9F250YgsWyTVi34tF/o9DtLsGjre8uxGNXoag0 +DmwwWZsH+IdCRk8UDJ7AMIEGDpQxObKQRiSsZxPGz38PZd49nL9bMz0rSC45IQ3Q +qor9aQnBB8+XDYZUGNUJAPGNOY8rQdPqWO0N3KUGp18hhZCncjQQFSuPohFAgEPO +e+tSC0NMYgwe0MkkVArhPdYdAbQS +-----END PRIVATE KEY----- \ No newline at end of file From 4ced532cd3fe575eabb52691f014dfdff95ffdbb Mon Sep 17 00:00:00 2001 From: Paul Richards Date: Tue, 30 Jun 2020 14:39:49 +0200 Subject: [PATCH 09/27] Bumped version and decoded decrypted message to string from bytes. --- .gitignore | 3 ++- maltego_trx/__init__.py | 2 +- maltego_trx/oauth.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index c5a6848..dcf5b77 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ build/ dist/ *.pyc __pycache__ -maltego_trx.egg-info \ No newline at end of file +maltego_trx.egg-info +.pytest_cache diff --git a/maltego_trx/__init__.py b/maltego_trx/__init__.py index 288108b..bf730e4 100644 --- a/maltego_trx/__init__.py +++ b/maltego_trx/__init__.py @@ -1 +1 @@ -VERSION = "1.3.6" +VERSION = "1.3.7" diff --git a/maltego_trx/oauth.py b/maltego_trx/oauth.py index c745e6b..e0d3464 100644 --- a/maltego_trx/oauth.py +++ b/maltego_trx/oauth.py @@ -30,7 +30,7 @@ def _rsa_decrypt(private_key_path=None, ciphertext=None): ciphertext = base64.b64decode(ciphertext) private_key = RSA.import_key(open(private_key_path).read()) cipher = PKCS1_v1_5.new(private_key) - plaintext = cipher.decrypt(ciphertext, sentinel) + plaintext = cipher.decrypt(ciphertext, sentinel).decode('utf8') return plaintext @staticmethod From c65da8f77b2dbaa281d0b4862c003828ea0835b5 Mon Sep 17 00:00:00 2001 From: Maltego Staff Date: Tue, 21 Jul 2020 16:29:21 +0200 Subject: [PATCH 10/27] Documentation Update. --- README.md | 145 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 88 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 5cd19a8..abff59c 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,18 @@ # Maltego TRX Python Library ## Getting Started -The library can be used in either Python2 or Python 3. + +The library can be used in either Python2 or Python 3 (recommended). To install the trx library run the following command: -``` + +``` bash pip install maltego-trx ``` After installing you can create a new project by running the following command: -``` + +``` bash maltego-trx start new_project ``` @@ -21,11 +24,11 @@ Add a new transform by creating a new python file in the "transforms" folder of Any file in the folder where the **class name matches the filename** and the class inherits from Transform, will automatically be discovered and added to your server. - A simple transform would look like the following: `new_project/transforms/GreetPerson.py` -```python + +``` python from maltego_trx.entities import Phrase from maltego_trx.transform import DiscoverableTransform @@ -43,88 +46,105 @@ class GreetPerson(DiscoverableTransform): ``` ## Running The Transform Server + ### For Development You can start the development server, by running the following command: -``` + +``` bash python project.py runserver ``` This will startup a development server that automatically reloads every time the code is changed. ### For Production + You can run a gunicorn transform server, after installing gunicorn on the host machine and then running the command: -``` + +``` bash gunicorn --bind=0.0.0.0:8080 --threads=25 --workers=2 project:app ``` *For publicly accessible servers, it is recommended to run your Gunicorn server behind proxy servers such as Nginx.* ## Run a Docker Transform server + The `demo` folder provides an example project. The Docker files given can be used to setup and run your project in Docker. The Dockerfile and docker-compose file can be used to easily setup and run a development transform server. -If you have copied the `docker-compose.yml`, `Dockerfile` and `prod.yml` files into your project, -then you can use the following commands to run the server in Docker. +If you have copied the `docker-compose.yml`, `Dockerfile` and `prod.yml` files into your project, then you can use the following commands to run the server in Docker. Run the following to start the development server: -``` + +``` bash docker-compose up ``` Run the following command to run a production gunicorn server: -``` + +``` bash docker-compose -f prod.yml up --build ``` *For publicly accessible servers, it is recommended to run your Gunicorn server behind proxy servers such as Nginx.* ## Local Transforms + [Documentation](https://docs.maltego.com/support/solutions/articles/15000017605-writing-local-transforms-in-python) Transforms written using this library can be used as either local or server transforms. To run a local transform from your project, you will need to pass the following arguments: -``` + +``` bash project.py local ``` You can find the correct transform_name to use by running `python project.py list`. ### Caveats + The following values are not passed to local transforms, and will have dummy values in their place: + - `type`: `local.Unknown` - `weight`: 100 - `slider`: 100 - `transformSettings`: {} ## Legacy Transforms + [Documentation](https://docs.maltego.com/support/solutions/articles/15000018299-porting-old-trx-transforms-to-the-latest-version) -If you have old TRX transforms that are written as functions, +If you have old TRX transforms that are written as functions, they can be registered with the server using the `maltego_trx.registry.register_transform_function` method. In order to port your old transforms, make two changes: + 1. Import the MaltegoTransform class from the `maltego_trx` package instead of from a local file. 2. Call the `register_transform_function` in order for the transform to be registered in your project. For example In the legacy transform file, change: -```python + +``` python from Maltego import * def old_transform(m): ``` + To: + ```python + from maltego_trx.maltego import MaltegoTransform def old_transform(m): ``` In the `project.py` file add the following: + ```python from maltego_trx.registry import register_transform_function from legacy_transform import trx_DNS2IP @@ -132,86 +152,97 @@ from legacy_transform import trx_DNS2IP register_transform_function(trx_DNS2IP) ``` - ## CLI + The following commands can be run using the project.py file. ### Run Server -``` + +``` bash python project.py runserver ``` Start a development server that you can use to develop new transforms. ### List -``` + +``` bash python project.py list ``` List the available transforms together with their transform server URLs and local transform names. ## Reference + ### Constants + The following constants can be imported from `maltego_trx.maltego`. **Message Types:** - - `UIM_FATAL` - - `UIM_PARTIAL` - - `UIM_INFORM` - - `UIM_DEBUG` - + +- `UIM_FATAL` +- `UIM_PARTIAL` +- `UIM_INFORM` +- `UIM_DEBUG` + **Bookmark Colors:** - - `BOOKMARK_COLOR_NONE` - - `BOOKMARK_COLOR_BLUE` - - `BOOKMARK_COLOR_GREEN` - - `BOOKMARK_COLOR_YELLOW` - - `BOOKMARK_COLOR_PURPLE` - - `BOOKMARK_COLOR_RED` -**Link Styles:** - - `LINK_STYLE_NORMAL` - - `LINK_STYLE_DASHED` - - `LINK_STYLE_DOTTED` - - `LINK_STYLE_DASHDOT` +- `BOOKMARK_COLOR_NONE` +- `BOOKMARK_COLOR_BLUE` +- `BOOKMARK_COLOR_GREEN` +- `BOOKMARK_COLOR_YELLOW` +- `BOOKMARK_COLOR_PURPLE` +- `BOOKMARK_COLOR_RED` +**Link Styles:** +- `LINK_STYLE_NORMAL` +- `LINK_STYLE_DASHED` +- `LINK_STYLE_DOTTED` +- `LINK_STYLE_DASHDOT` ### Request/MaltegoMsg + The request/maltego msg object given to the transform contains the information about the input entity. **Attributes:** - - `Value: str`: The display value of the input entity on the graph - - `Weight: int`: The weight of the input entity - - `Slider: int`: Results slider setting in the client - - `Type: str`: The input entity type - - `Properties: dict(str: str)`: A key-value dictionary of the input entity properties - - `TransformSettings: dict(str: str)`: A key-value dictionary of the transform settings + +- `Value: str`: The display value of the input entity on the graph +- `Weight: int`: The weight of the input entity +- `Slider: int`: Results slider setting in the client +- `Type: str`: The input entity type +- `Properties: dict(str: str)`: A key-value dictionary of the input entity properties +- `TransformSettings: dict(str: str)`: A key-value dictionary of the transform settings **Methods:** - - `getProperty(name: str)`: get a property value of the input entity - - `getTransformSetting(name: str)`: get a transform setting value + +- `getProperty(name: str)`: get a property value of the input entity +- `getTransformSetting(name: str)`: get a transform setting value ### Response/MaltegoTransform **Methods:** - - `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object created by the method. - - `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message type constant. - + +- `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object created by the method. +- `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message type constant. + ### Entity **Methods:** - - `setType(type: str)`: Set the entity type (e.g. "Phrase" for maltego.Phrase entity) - - `setValue(value: str)`: Set the entity value - - `setWeight(weight: int)`: Set the entity weight - - `addDisplayInformation(content: str, title: str)`: Add display information for the entity. - - `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. Matching rule can be `strict` or `loose`. - - `setIconURL(url: str)`: Set the entity icon URL - - `setBookmark(bookmark: int)`: Set bookmark color index (e.g. -1 for BOOKMARK_COLOR_NONE, 3 for BOOKMARK_COLOR_PURPLE) - - `setNote(note: str)`: Set note content + +- `setType(type: str)`: Set the entity type (e.g. "Phrase" for maltego.Phrase entity) +- `setValue(value: str)`: Set the entity value +- `setWeight(weight: int)`: Set the entity weight +- `addDisplayInformation(content: str, title: str)`: Add display information for the entity. +- `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. Matching rule can be `strict` or `loose`. +- `setIconURL(url: str)`: Set the entity icon URL +- `setBookmark(bookmark: int)`: Set bookmark color index (e.g. -1 for BOOKMARK_COLOR_NONE, 3 for BOOKMARK_COLOR_PURPLE) +- `setNote(note: str)`: Set note content **Link Methods:** - - `setLinkColor(color: str)`: Set the link color (e.g. hex "#0000FF" for blue) - - `setLinkStyle(style: int)`: Set the link style index (e.g. 0 for LINK_STYLE_NORMAL, 2 for LINK_STYLE_DOTTED) - - `setLinkThickness(thick: int)`: Set link thickness (default is 1) - - `setLinkLabel(label: str)`: Set the label of the link - - `reverseLink()`: Reverse the link direction + +- `setLinkColor(color: str)`: Set the link color (e.g. hex "#0000FF" for blue) +- `setLinkStyle(style: int)`: Set the link style index (e.g. 0 for LINK_STYLE_NORMAL, 2 for LINK_STYLE_DOTTED) +- `setLinkThickness(thick: int)`: Set link thickness (default is 1) +- `setLinkLabel(label: str)`: Set the label of the link +- `reverseLink()`: Reverse the link direction From ea801f29ce49bf1c2e484f62caeefc08f1d96c1f Mon Sep 17 00:00:00 2001 From: MaltegoThinus <52233693+MaltegoThinus@users.noreply.github.com> Date: Thu, 29 Oct 2020 09:48:37 +0200 Subject: [PATCH 11/27] Update maltego.py --- maltego_trx/maltego.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/maltego_trx/maltego.py b/maltego_trx/maltego.py index 0560217..4383b83 100644 --- a/maltego_trx/maltego.py +++ b/maltego_trx/maltego.py @@ -93,6 +93,9 @@ def setLinkLabel(self, label): def reverseLink(self): self.addProperty('link#maltego.link.direction', 'link#maltego.link.direction', 'loose', 'output-to-input') + + def addCustomLinkProperty(self, fieldName=None, displayName=None, value=None): + self.addProperty('link#' + fieldName, displayName, '', value) def setBookmark(self, bookmark): self.addProperty('bookmark#', 'Bookmark', '', bookmark) From 3124b9d19c8a98df1f7f4e3128b05d2e6a6cb440 Mon Sep 17 00:00:00 2001 From: Chris Dietrich Date: Sun, 10 Jan 2021 13:24:47 +0100 Subject: [PATCH 12/27] add missing entity types and expose handler properties as initially implemented by tlansec in paterva/maltego-trx/pull/3 --- maltego_trx/entities.py | 8 ++++++-- maltego_trx/handler.py | 7 ++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/maltego_trx/entities.py b/maltego_trx/entities.py index 858481d..e129248 100644 --- a/maltego_trx/entities.py +++ b/maltego_trx/entities.py @@ -1,7 +1,9 @@ Alias = "maltego.Alias" ASNumber = "maltego.AS" -BuiltwithTechnology = "maltego.Builtwith.Technology" +BuiltwithTechnology = "maltego.BuiltWithTechnology" +BuiltWithRelationship = "maltego.BuiltWithRelationship" CircularArea = "maltego.CircularArea" +Company = "maltego.Company" Device = "maltego.Device" DNS = "maltego.DNSName" Document = "maltego.Document" @@ -9,6 +11,7 @@ Email = "maltego.EmailAddress" FlickrAffiliation = "maltego.AffiliationFlickr" GPS = "maltego.GPS" +Hash = "maltego.Hash" Hashtag = "maltego.Hashtag" Image = "maltego.Image" IPAddress = "maltego.IPv4Address" @@ -18,9 +21,11 @@ Namechk = "maltego.Namechk" Netblock = "maltego.Netblock" NS = "maltego.NSRecord" +Organization = "maltego.Organization" Person = "maltego.Person" PhoneNumber = "maltego.PhoneNumber" Phrase = "maltego.Phrase" +Port = "maltego.Port" Sentiment = "maltego.Sentiment" StockSymbol = "maltego.StockSymbol" Tweet = "maltego.Twit" @@ -30,4 +35,3 @@ URL = "maltego.URL" Website = "maltego.Website" WebTitle = "maltego.WebTitle" - diff --git a/maltego_trx/handler.py b/maltego_trx/handler.py index 9605cb2..1b13203 100644 --- a/maltego_trx/handler.py +++ b/maltego_trx/handler.py @@ -8,14 +8,14 @@ Receive commands run inside a project folder. """ -def handle_run(name, args, app): +def handle_run(name, args, app, port=8080, ssl_context=None, debug=False): if name == "__main__": if len(args) >= 2: command = args[1].lower() if command == "runserver": print("\n=== Maltego Transform Server: v%s ===\n" % VERSION) print_transforms() - app.run(host="0.0.0.0", port=8080, debug=False) + app.run(host="0.0.0.0", port=port, debug=debug, ssl_context=ssl_context) elif command == "list": print_transforms() elif command == "local" and len(args) > 3: @@ -27,5 +27,6 @@ def handle_run(name, args, app): print(get_exception_message(msg="Unable to find a transform matching '%s'." % transform_name)) else: - print("Command not recognised. Available commands are: \n 'runserver'") + commands = ["list", "local", "runserver"] + print("Command not recognised. Available commands are:\r\n{0}".format("\r\n".join(commands))) From d6445df916d0fdacfd527c3ccbdc0b0f511279ed Mon Sep 17 00:00:00 2001 From: Tendai Marengereke Date: Thu, 14 Jan 2021 22:36:53 +0200 Subject: [PATCH 13/27] refactor(test): replace OS dependent test --- tests/test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test.py b/tests/test.py index c03f577..bfb5767 100644 --- a/tests/test.py +++ b/tests/test.py @@ -1,4 +1,5 @@ from maltego_trx.oauth import MaltegoOauth +import os okta_ciphertoken = "pZ598ZEZ7EwpBQSOJSvCZJKkcWhtbX95K7Q0f0hwbk93O+xaUB4/NegK3r54PH1NReis/Jgt4UbGc5oCuU+R7UDM1icoDUmyCmV1U78iqjHElPdDBHlxPrl2zoXBwcU1iFbukDNy49Xghy3cwwqhmEXg/FKnYUlBQl6jdf06kE1pwfHiRgF5NrJISAD7eCmUybqiAVRDW4lbLeMGk/uSrrCpDQBxd2Em/sYKBzqO87pQRPLll23G9KNxyVinHQksGgc+dI0OPYv9EQcUs7g1JFraUKUeFjHAf/OyGGrp6WH84nGqG35afkH9xsDmySBb5DZWRjef8DzSd7oRagoGq3wKBHDh700mMKf/YkR8L9cK4l1w4yNh7EfIVlCWD5yWJUx4a1lN9CvJmFs7N9A903spVVGeP8avz50raWFb5g/4XhtIpei4ylVFi49dDeVjgb0BR0I4vuU7VDmcwFweAtSfclPb5Xfp98zI9yXnr8DB31gmzf3DIkVFIIFLxZe0gG+BSIEmC7/z0L7J+YvIqoGDq8jC0Ehe9IVXYfK1yluWAsbwj62rNAzkPVSe5l+FMQFrQRXpxZFBMXzphqRAhyYKfGNX/tocoMJoLFp8O3PBRC0kIHjTNaSX8KpftNYWreEdZx6wjk2n2eeOXM5nVeSOx4uoIifRe4NUV7VYzbGm3FSRc0k1HZIGHwn1WTvn95U3crvwFN86N3k41JU+0kzSpsLxy+Y63VfPxYTbRUET/wIOf0NTfG96QCPtVKkdVh9aTqnhaN0TpGoaZQEjUqpURxbj+8mT3g6MI4vOC9Qr6vw2wIkvzehr/yKxh6vsnfUUw8UJ1YtlZI7exPsHyFLUr2J3VvpDsaemRcn1KY/uHJZ3MMHK9GNciLA6Skw8SL38mw1tJJZ1h1bMMk1IYgBZLe2ZSXXyv7JaDPq/WJmHWDaIalmXE1cKMA56qMgYGdHxs3Pai5nwEtzIGVS1OcuMnBF6sOMMBTKSFAJmlcYo8NGwjKqe/yy1cVJZe7Ucv/jy/IS3AdE1my8cDya92jDsxifsaONQMm6YIs54SIA=$f7TU33wTTxYFtbVnwJeGGw==$c64l75qukq+980EkB1tQZKEicryt6HBB0IOjeMSRQmLOYCQpTcaXlPRJ3QdS+DfRocor7OL36wLAooRqz1IaN4QQ5+6Wx5reEXFVgKtaDX777I046DM5QU3Jf+iibSbm3mXYMw0z+OknbXyGLnc7ceSaPTJdP1LkCGQ75fLMXHCLWSpqwOKHOBhqyQGwrYlj2WxPzmOfAMaIkjKZIQWzoGjrYRzwkNCQbxykJcwD5TVuVDwAHyp84zcvW0WWoUZ+rrrooJwuJJQEdiTwLZsseqklXRNso4e5eFQwH49T9IDPHkKfVusu6rLiMNgFyc18rFR1d/BYLXBu7uzMAuvQ8wZdcOtJYx+JLJmOaPI65ymGNFTpwTHYnDBTXmpW0qX2dtEglAERw1nrdl94fwsa+I/iw3H7VIivqRF3pAchfvdNF75MIEm6XH2UXY/sZD7zjMdxxwKQiaEg2bpLd2mEtqsLc0mTq/zd03ZLEUnzXsXlp8brCfDgjoVsZAJH+ElQ6wr3dxzRMlDIxWWBYFcv8LnGVQk/Vciqtee780Yh/lgttv06kyXz81dIWjumz8TqV2eV9ZpP/YxTnNzAa91fleGNah/IKhMOV8PsGy2uOnHRiUL218p1T6+Cm9B2kmL1C/2MM3080NjJVWIfqYsyTGPbR+hURK0RB/oteG5QWPs=" private_key_path = "test_private_key.pem" @@ -11,9 +12,7 @@ def testOktaDecrypt(): def testReadKey(): - private_key = open(private_key_path).read() - assert private_key.__sizeof__() == 3292 - + assert os.path.exists(private_key_path) def testCipherSplit(): encrypted_fields = okta_ciphertoken.split("$") From 9f4b76ccbd073bfdbb21b39afa5ba4212f42e9d1 Mon Sep 17 00:00:00 2001 From: Tendai Marengereke Date: Thu, 14 Jan 2021 22:44:38 +0200 Subject: [PATCH 14/27] feat(MaltegoOAuth): replace pycryptodome lib with cryptography for OAuth crypto pycryptodome requires a large dependency on windows. --- maltego_trx/oauth.py | 32 +++++++++++++++++++------------- setup.py | 2 +- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/maltego_trx/oauth.py b/maltego_trx/oauth.py index e0d3464..8f88f98 100644 --- a/maltego_trx/oauth.py +++ b/maltego_trx/oauth.py @@ -1,9 +1,10 @@ """Maltego OAuth Crypto Helper""" import base64 -from Crypto import Random -from Crypto.Cipher import PKCS1_v1_5, AES -from Crypto.Hash import SHA -from Crypto.PublicKey import RSA + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from requests.auth import AuthBase @@ -16,7 +17,7 @@ class MaltegoOauth: 3. Refresh Token 4. Expires In - Contains 1 Methods: + Contains Methods: 1. decrypt_secrets(private_key_path="pem file", ciphertext="request.getTransformSetting('name from TDS')") """ @@ -25,12 +26,14 @@ def _rsa_decrypt(private_key_path=None, ciphertext=None): """ RSA Decryption function, returns decrypted plaintext in b64 encoding """ - dsize = SHA.digest_size - sentinel = Random.new().read(20 + dsize) ciphertext = base64.b64decode(ciphertext) - private_key = RSA.import_key(open(private_key_path).read()) - cipher = PKCS1_v1_5.new(private_key) - plaintext = cipher.decrypt(ciphertext, sentinel).decode('utf8') + + with open(private_key_path, "rb") as key_file: + private_key = serialization.load_pem_private_key(key_file.read(), + password=None, + backend=None) + plaintext = private_key.decrypt(ciphertext, padding.PKCS1v15()) + return plaintext @staticmethod @@ -38,11 +41,14 @@ def _aes_decrypt(key=None, ciphertext=None): """ AES Decryption function, returns decrypted plaintext value """ - unpad = lambda s: s[:-ord(s[len(s) - 1:])] key = base64.b64decode(key) + cipher = Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + decryptor = cipher.decryptor(); ciphertext = base64.b64decode(ciphertext) - cipher = AES.new(key, AES.MODE_ECB) - plaintext = unpad(cipher.decrypt(ciphertext)).decode('utf8') + padded_b64_plaintext = decryptor.update(ciphertext) + decryptor.finalize() + + unpad = lambda data: data[:-ord(data[len(data) - 1:])] + plaintext = unpad(padded_b64_plaintext).decode('utf8') return plaintext @classmethod diff --git a/setup.py b/setup.py index 6092967..edc23fc 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ install_requires=[ 'flask>=1', 'six>=1', - 'pycryptodome>=3.9.7' + 'cryptography>=3.3.1' ], packages=[ 'maltego_trx', From 7e6c2ab6be2af29eb07b7e4d7fee4ead8021764c Mon Sep 17 00:00:00 2001 From: Thinus Prinsloo Date: Tue, 12 Jan 2021 11:20:27 +0200 Subject: [PATCH 15/27] Added missing Link method to documentation --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index abff59c..47b9981 100644 --- a/README.md +++ b/README.md @@ -246,3 +246,4 @@ The request/maltego msg object given to the transform contains the information a - `setLinkThickness(thick: int)`: Set link thickness (default is 1) - `setLinkLabel(label: str)`: Set the label of the link - `reverseLink()`: Reverse the link direction +- `addCustomLinkProperty(fieldname=None, displayName=None, value=None)`: Set a custom property for the link From 11ac443c1126a755768425379eda9c0b422e6e7a Mon Sep 17 00:00:00 2001 From: Tendai Marengereke Date: Tue, 19 Jan 2021 14:39:57 +0200 Subject: [PATCH 16/27] refactor(MaltegoOAuth): expose password api for private key (PEM file) --- maltego_trx/oauth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maltego_trx/oauth.py b/maltego_trx/oauth.py index 8f88f98..73d77ce 100644 --- a/maltego_trx/oauth.py +++ b/maltego_trx/oauth.py @@ -22,7 +22,7 @@ class MaltegoOauth: """ @staticmethod - def _rsa_decrypt(private_key_path=None, ciphertext=None): + def _rsa_decrypt(private_key_path=None, ciphertext=None, password=None): """ RSA Decryption function, returns decrypted plaintext in b64 encoding """ @@ -30,7 +30,7 @@ def _rsa_decrypt(private_key_path=None, ciphertext=None): with open(private_key_path, "rb") as key_file: private_key = serialization.load_pem_private_key(key_file.read(), - password=None, + password, backend=None) plaintext = private_key.decrypt(ciphertext, padding.PKCS1v15()) From 1885caa32fd9b50fce1fdce8ec95ddf7c71cabe3 Mon Sep 17 00:00:00 2001 From: Tendai Marengereke Date: Tue, 19 Jan 2021 16:09:02 +0200 Subject: [PATCH 17/27] refactor(MaltegoOAuth): use inbuilt primitives_padding.PKCS7() unpadding --- maltego_trx/oauth.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/maltego_trx/oauth.py b/maltego_trx/oauth.py index 73d77ce..9a9bd38 100644 --- a/maltego_trx/oauth.py +++ b/maltego_trx/oauth.py @@ -2,8 +2,8 @@ import base64 from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.primitives import serialization, padding as primitives_padding +from cryptography.hazmat.primitives.asymmetric import padding as asymmetric_padding from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from requests.auth import AuthBase @@ -32,7 +32,7 @@ def _rsa_decrypt(private_key_path=None, ciphertext=None, password=None): private_key = serialization.load_pem_private_key(key_file.read(), password, backend=None) - plaintext = private_key.decrypt(ciphertext, padding.PKCS1v15()) + plaintext = private_key.decrypt(ciphertext, asymmetric_padding.PKCS1v15()) return plaintext @@ -46,9 +46,8 @@ def _aes_decrypt(key=None, ciphertext=None): decryptor = cipher.decryptor(); ciphertext = base64.b64decode(ciphertext) padded_b64_plaintext = decryptor.update(ciphertext) + decryptor.finalize() - - unpad = lambda data: data[:-ord(data[len(data) - 1:])] - plaintext = unpad(padded_b64_plaintext).decode('utf8') + unpadder = primitives_padding.PKCS7(128).unpadder() + plaintext = (unpadder.update(padded_b64_plaintext) + unpadder.finalize()).decode('utf8') return plaintext @classmethod From 0f4727ffa0fcb433bbba654d1a9a5c1737d13a54 Mon Sep 17 00:00:00 2001 From: Tendai Marengereke Date: Tue, 2 Mar 2021 16:34:49 +0200 Subject: [PATCH 18/27] feat: add Genealogy element parsing and overlay functionality --- README.md | 47 +++++++++++++++++++ demo/apache/transforms/OverlayExample.py | 30 ++++++++++++ demo/gunicorn/transforms/OverlayExample.py | 30 ++++++++++++ maltego_trx/maltego.py | 37 +++++++++++++-- maltego_trx/overlays.py | 19 ++++++++ .../template_dir/transforms/OverlayExample.py | 30 ++++++++++++ maltego_trx/test_hierarchical_entity.xml | 13 +++++ 7 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 demo/apache/transforms/OverlayExample.py create mode 100644 demo/gunicorn/transforms/OverlayExample.py create mode 100644 maltego_trx/overlays.py create mode 100644 maltego_trx/template_dir/transforms/OverlayExample.py create mode 100644 maltego_trx/test_hierarchical_entity.xml diff --git a/README.md b/README.md index 47b9981..54cae8a 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,28 @@ The following constants can be imported from `maltego_trx.maltego`. - `LINK_STYLE_DOTTED` - `LINK_STYLE_DASHDOT` +### Enums + +**Overlays:** + +Overlays Enums are imported from `maltego_trx.overlays` + +*Overlay Position:* +- `NORTH = "N"` +- `NORTH_EAST = "NE"` +- `NORTH_WEST = "NW"` +- `EAST = "E"` +- `CENTER = "C"` +- `WEST = "W"` +- `SOUTH = "S"` +- `SOUTH_EAST = "SE"` +- `SOUTH_WEST = "SW"` + +*Overlay Type* +- `IMAGE = "image"` +- `COLOUR = "colour"` +- `TEXT = "text"` + ### Request/MaltegoMsg The request/maltego msg object given to the transform contains the information about the input entity. @@ -213,6 +235,8 @@ The request/maltego msg object given to the transform contains the information a - `Type: str`: The input entity type - `Properties: dict(str: str)`: A key-value dictionary of the input entity properties - `TransformSettings: dict(str: str)`: A key-value dictionary of the transform settings +- `Genealogy: list(dict(str: str))`: A key-value dictionary of the Entity genealogy, + this is only applicable for extended entities e.g. Website Entity **Methods:** @@ -235,9 +259,32 @@ The request/maltego msg object given to the transform contains the information a - `setWeight(weight: int)`: Set the entity weight - `addDisplayInformation(content: str, title: str)`: Add display information for the entity. - `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. Matching rule can be `strict` or `loose`. +- `addOverlay(property_name: str, position:Position, overlay_type:OverlayType)`: Add an overlay to the entity. `Position` and `Type` are defined in the `maltego_tx.overlays` + +Overlay can be added as Text, Image or Color + +```python + + # references the icon name `Champion` from the Maltego Desktop Client and this is will show up as an overlay on the graph + entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + + # add a dynamic property + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + + # add the text value of the property `exampleDynamicPropertyName` as an overlay, any existing property name will work + entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + + # add a color overlay + entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + + # add a flag overlay - DE is an icon on the Maltego Desktop Client + entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) +``` + - `setIconURL(url: str)`: Set the entity icon URL - `setBookmark(bookmark: int)`: Set bookmark color index (e.g. -1 for BOOKMARK_COLOR_NONE, 3 for BOOKMARK_COLOR_PURPLE) - `setNote(note: str)`: Set note content +- `setGenealogy(genealogy: dict)`: Set genealogy **Link Methods:** diff --git a/demo/apache/transforms/OverlayExample.py b/demo/apache/transforms/OverlayExample.py new file mode 100644 index 0000000..34577a5 --- /dev/null +++ b/demo/apache/transforms/OverlayExample.py @@ -0,0 +1,30 @@ +from maltego_trx.entities import Phrase +from maltego_trx.overlays import Position, OverlayType + +from maltego_trx.transform import DiscoverableTransform + + +class OverlayExample(DiscoverableTransform): + """ + Returns a phrase with overlays on the graph. + """ + + @classmethod + def create_entities(cls, request, response): + person_name = request.Value + entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) + + # references the icon name `Champion` and this is will show up as an overlay on the graph + entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + + # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + + # add the text of the property `exampleDynamicPropertyName` as an overlay + entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + + # add a color overlay + entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + + # add a flag overlay + entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/demo/gunicorn/transforms/OverlayExample.py b/demo/gunicorn/transforms/OverlayExample.py new file mode 100644 index 0000000..34577a5 --- /dev/null +++ b/demo/gunicorn/transforms/OverlayExample.py @@ -0,0 +1,30 @@ +from maltego_trx.entities import Phrase +from maltego_trx.overlays import Position, OverlayType + +from maltego_trx.transform import DiscoverableTransform + + +class OverlayExample(DiscoverableTransform): + """ + Returns a phrase with overlays on the graph. + """ + + @classmethod + def create_entities(cls, request, response): + person_name = request.Value + entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) + + # references the icon name `Champion` and this is will show up as an overlay on the graph + entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + + # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + + # add the text of the property `exampleDynamicPropertyName` as an overlay + entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + + # add a color overlay + entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + + # add a flag overlay + entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/maltego_trx/maltego.py b/maltego_trx/maltego.py index 4383b83..522e386 100644 --- a/maltego_trx/maltego.py +++ b/maltego_trx/maltego.py @@ -1,7 +1,9 @@ import uuid; + from xml.dom import minidom -from .entities import Phrase +from .entities import Phrase, translate_legacy_properties +from .overlays import Position, OverlayType from .utils import remove_invalid_xml_chars BOOKMARK_COLOR_NONE = "-1" @@ -44,7 +46,7 @@ ADD_FIELD_TEMPLATE = "" DISP_INFO_TEMPLATE = "" UIM_TEMPLATE = "%(text)s" - +OVERLAY_TEMPLATE = "" class MaltegoEntity(object): def __init__(self, type=None, value=None): @@ -55,6 +57,7 @@ def __init__(self, type=None, value=None): self.additionalFields = [] self.displayInformation = [] self.iconURL = "" + self.overlays = [] def setType(self, type=None): if type: @@ -93,7 +96,7 @@ def setLinkLabel(self, label): def reverseLink(self): self.addProperty('link#maltego.link.direction', 'link#maltego.link.direction', 'loose', 'output-to-input') - + def addCustomLinkProperty(self, fieldName=None, displayName=None, value=None): self.addProperty('link#' + fieldName, displayName, '', value) @@ -103,6 +106,9 @@ def setBookmark(self, bookmark): def setNote(self, note): self.addProperty('notes#', 'Notes', '', note) + def addOverlay(self, property_name=None, position=Position, overlay_type=OverlayType): + self.overlays.append([property_name, position.value, overlay_type.value]) + def add_field_to_xml(self, additional_field): name, display, matching, value = additional_field matching = "strict" if matching.lower().strip() == "strict" else "loose" @@ -139,6 +145,16 @@ def returnEntity(self): lines.append(self.add_field_to_xml(additional_field)) lines.append("") + if self.overlays: + lines.append("") + for overlay in self.overlays: + overlay_tag = OVERLAY_TEMPLATE % { + "property_name": overlay[0], + "position": overlay[1], + "type": overlay[2], + } + lines.append(overlay_tag) + lines.append("") if self.iconURL: lines.append("%s" % self.iconURL) @@ -226,6 +242,17 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): self.Weight = self._get_int(entity, "Weight") self.Slider = self._get_int(maltego_msg, "Limits", attr_name="SoftLimit") + self._entity_types = [] + self.Genealogy = [] + genealogy_tag = maltego_msg.getElementsByTagName("Genealogy") + genealogy_types = genealogy_tag[0].getElementsByTagName("Type") if genealogy_tag else [] + for genealogy_type_tag in genealogy_types: + entity_type_name = genealogy_type_tag.getAttribute("Name") + entity_type_old_name = genealogy_type_tag.getAttribute("OldName") + entity_type = {"Name": entity_type_name, + "OldName": entity_type_old_name if entity_type_old_name else None} + self._entity_types.append(entity_type_name) + self.Genealogy.append(entity_type) # Additional Fields self.Properties = {} @@ -235,6 +262,10 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): name = field.getAttribute("Name") value = self._get_text(field) self.Properties[name] = value + for entity_type in self._entity_types: + v3_property = translate_legacy_properties(entity_type, name) + if v3_property: + self.Properties[v3_property] = value # Transform Settings self.TransformSettings = {} diff --git a/maltego_trx/overlays.py b/maltego_trx/overlays.py new file mode 100644 index 0000000..9cc2747 --- /dev/null +++ b/maltego_trx/overlays.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class Position(Enum): + NORTH = "N" + NORTH_EAST = "NE" + NORTH_WEST = "NW" + EAST = "E" + CENTER = "C" + WEST = "W" + SOUTH = "S" + SOUTH_EAST = "SE" + SOUTH_WEST = "SW" + + +class OverlayType(Enum): + IMAGE = "image" + COLOUR = "colour" + TEXT = "text" diff --git a/maltego_trx/template_dir/transforms/OverlayExample.py b/maltego_trx/template_dir/transforms/OverlayExample.py new file mode 100644 index 0000000..34577a5 --- /dev/null +++ b/maltego_trx/template_dir/transforms/OverlayExample.py @@ -0,0 +1,30 @@ +from maltego_trx.entities import Phrase +from maltego_trx.overlays import Position, OverlayType + +from maltego_trx.transform import DiscoverableTransform + + +class OverlayExample(DiscoverableTransform): + """ + Returns a phrase with overlays on the graph. + """ + + @classmethod + def create_entities(cls, request, response): + person_name = request.Value + entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) + + # references the icon name `Champion` and this is will show up as an overlay on the graph + entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + + # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + + # add the text of the property `exampleDynamicPropertyName` as an overlay + entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + + # add a color overlay + entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + + # add a flag overlay + entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/maltego_trx/test_hierarchical_entity.xml b/maltego_trx/test_hierarchical_entity.xml new file mode 100644 index 0000000..5b828fe --- /dev/null +++ b/maltego_trx/test_hierarchical_entity.xml @@ -0,0 +1,13 @@ + + + + + + + www.paterva.com + false + 80 + + www.paterva.com + 0 + From 5f6e0755d392c25a69a4e7c99a29a0d2674b5f79 Mon Sep 17 00:00:00 2001 From: Tendai Marengereke Date: Tue, 2 Mar 2021 16:38:25 +0200 Subject: [PATCH 19/27] feat: add version 2 -> version 3 property translation --- maltego_trx/entities.py | 30 +++++++++++++++++++ tests/test_property_mapping.py | 20 +++++++++++++ tests/test_request.xml | 18 +++++++++++ .../transforms/TestRequestPropertyMapping.py | 15 ++++++++++ tests/transforms/__init__.py | 0 5 files changed, 83 insertions(+) create mode 100644 tests/test_property_mapping.py create mode 100644 tests/test_request.xml create mode 100644 tests/transforms/TestRequestPropertyMapping.py create mode 100644 tests/transforms/__init__.py diff --git a/maltego_trx/entities.py b/maltego_trx/entities.py index e129248..4446430 100644 --- a/maltego_trx/entities.py +++ b/maltego_trx/entities.py @@ -35,3 +35,33 @@ URL = "maltego.URL" Website = "maltego.Website" WebTitle = "maltego.WebTitle" + +# {entityName: {version2PropertyName: version3PropertyName,...}} +entity_property_map = { + "maltego.Person": {"firstname": "person.firstnames", "lastname": "person.lastname"}, + "maltego.Domain": {"whois": "whois-info"}, + "maltego.IPv4Address": {"whois": "whois-info"}, + "maltego.URL": {"maltego.v2.value.property": "short-title", "theurl": "url", "fulltitle": "title"}, + "maltego.Document": {"maltego.v2.value.property": "title", "link": "url", "metainfo": "document.meta-data"}, + "maltego.Location": {"area": "location.area", "countrysc": "url", "long": "longitude", "lat": "latitude"}, + "maltego.PhoneNumber": {"countrycode": "phonenumber.countrycode", "citycode": "phonenumber.citycode", + "areacode": "phonenumber.areacode", "lastnumbers": "phonenumber.lastnumbers"}, + "maltego.affiliation.Spock": {"network": "affiliation.network", "uid": "affiliation.uid", + "profile_url": "affiliation.profile-url", "spock_websites": "spock.websites"}, + "maltego.affiliation": {"network": "affiliation.network", "uid": "affiliation.uid", + "profile_url": "affiliation.profile-url"}, + "maltego.Service": {"banner": "banner.text", "port": "port.number"}, + "maltego.Alias": {"properties.alias": "alias"}, + "maltego.Device": {"properties.device": "device"}, + "maltego.GPS": {"properties.gps": "gps.coordinate"}, + "maltego.CircularArea": {"area": "radius"}, + "maltego.Image": {"properties.image": "description", "fullImage": "url"}, + "maltego.NominatimLocation": {"properties.nominatimlocation": "nominatimlocation"}, + "maltego.BuiltWithTechnology": {"properties.builtwithtechnology": "builtwith.technology"}, + "maltego.FacebookObject": {"properties.facebookobject": "facebook.object"} +} + + +def translate_legacy_properties(entity_type, v2_property): + """Function maps a legacy version 2 entity property name to version 3 entity property name""" + return entity_property_map .get(entity_type, {}).get(v2_property) diff --git a/tests/test_property_mapping.py b/tests/test_property_mapping.py new file mode 100644 index 0000000..12af79e --- /dev/null +++ b/tests/test_property_mapping.py @@ -0,0 +1,20 @@ +from maltego_trx.registry import register_transform_classes +from maltego_trx.server import app +from tests import transforms + + +def test_request_property_mapping(): + register_transform_classes(transforms) + app.testing = True + + with app.test_client() as test_app: + response = make_transform_call(test_app, "/run/testrequestpropertymapping/") + assert response.status_code == 200 + data = response.data.decode('utf8') + assert "whois-info found" in data + + +def make_transform_call(test_app=None, run_endpoint=""): + with open('test_request.xml') as requestMsg: + response = test_app.post(run_endpoint, data=requestMsg.read()) + return response diff --git a/tests/test_request.xml b/tests/test_request.xml new file mode 100644 index 0000000..77c5f40 --- /dev/null +++ b/tests/test_request.xml @@ -0,0 +1,18 @@ + + + + + + + + + paterva.com + whois-info found + + paterva.com + 0 + + + + + diff --git a/tests/transforms/TestRequestPropertyMapping.py b/tests/transforms/TestRequestPropertyMapping.py new file mode 100644 index 0000000..c82426a --- /dev/null +++ b/tests/transforms/TestRequestPropertyMapping.py @@ -0,0 +1,15 @@ +from maltego_trx.entities import Phrase + +from maltego_trx.transform import DiscoverableTransform + + +class TestRequestPropertyMapping(DiscoverableTransform): + """ + Test if the automatic mapping of v2 propertyname `whois` -> `whois-info` has been done by the library. Original input + contains only whois property name. see test_request.xml + """ + + @classmethod + def create_entities(cls, request, response): + v3_property_value = request.Properties['whois-info'] + response.addEntity(Phrase, "%s" % v3_property_value) diff --git a/tests/transforms/__init__.py b/tests/transforms/__init__.py new file mode 100644 index 0000000..e69de29 From ae0a9ffdbfe0747d2374109818ff9d4a6cd81e2b Mon Sep 17 00:00:00 2001 From: Philipp Dowling Date: Tue, 2 Mar 2021 22:20:10 +0100 Subject: [PATCH 20/27] minor code style changes, add convenience method for clearing old property names from input entity --- maltego_trx/__init__.py | 2 +- maltego_trx/entities.py | 21 +++++++++++++-------- maltego_trx/maltego.py | 24 +++++++++++++++++------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/maltego_trx/__init__.py b/maltego_trx/__init__.py index bf730e4..3ba22d0 100644 --- a/maltego_trx/__init__.py +++ b/maltego_trx/__init__.py @@ -1 +1 @@ -VERSION = "1.3.7" +VERSION = "1.3.8" diff --git a/maltego_trx/entities.py b/maltego_trx/entities.py index 4446430..0f48e06 100644 --- a/maltego_trx/entities.py +++ b/maltego_trx/entities.py @@ -37,19 +37,24 @@ WebTitle = "maltego.WebTitle" # {entityName: {version2PropertyName: version3PropertyName,...}} -entity_property_map = { +entity_property_map = { "maltego.Person": {"firstname": "person.firstnames", "lastname": "person.lastname"}, "maltego.Domain": {"whois": "whois-info"}, "maltego.IPv4Address": {"whois": "whois-info"}, "maltego.URL": {"maltego.v2.value.property": "short-title", "theurl": "url", "fulltitle": "title"}, "maltego.Document": {"maltego.v2.value.property": "title", "link": "url", "metainfo": "document.meta-data"}, "maltego.Location": {"area": "location.area", "countrysc": "url", "long": "longitude", "lat": "latitude"}, - "maltego.PhoneNumber": {"countrycode": "phonenumber.countrycode", "citycode": "phonenumber.citycode", - "areacode": "phonenumber.areacode", "lastnumbers": "phonenumber.lastnumbers"}, - "maltego.affiliation.Spock": {"network": "affiliation.network", "uid": "affiliation.uid", - "profile_url": "affiliation.profile-url", "spock_websites": "spock.websites"}, - "maltego.affiliation": {"network": "affiliation.network", "uid": "affiliation.uid", - "profile_url": "affiliation.profile-url"}, + "maltego.PhoneNumber": { + "countrycode": "phonenumber.countrycode", "citycode": "phonenumber.citycode", + "areacode": "phonenumber.areacode", "lastnumbers": "phonenumber.lastnumbers" + }, + "maltego.affiliation.Spock": { + "network": "affiliation.network", "uid": "affiliation.uid", "profile_url": "affiliation.profile-url", + "spock_websites": "spock.websites" + }, + "maltego.affiliation": { + "network": "affiliation.network", "uid": "affiliation.uid", "profile_url": "affiliation.profile-url" + }, "maltego.Service": {"banner": "banner.text", "port": "port.number"}, "maltego.Alias": {"properties.alias": "alias"}, "maltego.Device": {"properties.device": "device"}, @@ -62,6 +67,6 @@ } -def translate_legacy_properties(entity_type, v2_property): +def translate_legacy_property_name(entity_type, v2_property): """Function maps a legacy version 2 entity property name to version 3 entity property name""" return entity_property_map .get(entity_type, {}).get(v2_property) diff --git a/maltego_trx/maltego.py b/maltego_trx/maltego.py index 522e386..a2d5666 100644 --- a/maltego_trx/maltego.py +++ b/maltego_trx/maltego.py @@ -2,7 +2,7 @@ from xml.dom import minidom -from .entities import Phrase, translate_legacy_properties +from .entities import Phrase, translate_legacy_property_name, entity_property_map from .overlays import Position, OverlayType from .utils import remove_invalid_xml_chars @@ -48,6 +48,7 @@ UIM_TEMPLATE = "%(text)s" OVERLAY_TEMPLATE = "" + class MaltegoEntity(object): def __init__(self, type=None, value=None): self.entityType = type if type else Phrase @@ -242,7 +243,6 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): self.Weight = self._get_int(entity, "Weight") self.Slider = self._get_int(maltego_msg, "Limits", attr_name="SoftLimit") - self._entity_types = [] self.Genealogy = [] genealogy_tag = maltego_msg.getElementsByTagName("Genealogy") genealogy_types = genealogy_tag[0].getElementsByTagName("Type") if genealogy_tag else [] @@ -251,7 +251,6 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): entity_type_old_name = genealogy_type_tag.getAttribute("OldName") entity_type = {"Name": entity_type_name, "OldName": entity_type_old_name if entity_type_old_name else None} - self._entity_types.append(entity_type_name) self.Genealogy.append(entity_type) # Additional Fields @@ -262,10 +261,10 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): name = field.getAttribute("Name") value = self._get_text(field) self.Properties[name] = value - for entity_type in self._entity_types: - v3_property = translate_legacy_properties(entity_type, name) - if v3_property: - self.Properties[v3_property] = value + for entity_type in self.Genealogy: + v3_property_name = translate_legacy_property_name(entity_type["Name"], name) + if v3_property_name is not None: + self.Properties[v3_property_name] = value # Transform Settings self.TransformSettings = {} @@ -278,6 +277,7 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): elif LocalArgs: self.Value = LocalArgs[0] self.Type = "local.Unknown" + self.Genealogy = None self.Weight = 100 self.Slider = 100 @@ -294,6 +294,16 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): self.buildProperties(text.split("#"), hash_rnd, equals_rnd, bslash_rnd) self.TransformSettings = {} + def clearLegacyProperties(self): + to_clear = set() + for entity_type in self.Genealogy or []: + for prop_name in entity_property_map.get(entity_type["Name"], []): + to_clear.add(prop_name) + + for field_name in to_clear: + if field_name in self.Properties: + del self.Properties[field_name] + def buildProperties(self, key_value_array, hash_rnd, equals_rnd, bslash_rnd): self.Properties = {} for property_section in key_value_array: From 3afb3b6ac6e65a6aa39893b02ab49830b10e7d75 Mon Sep 17 00:00:00 2001 From: Philipp Dowling Date: Fri, 5 Mar 2021 15:51:08 +0100 Subject: [PATCH 21/27] adjusted code style, README, requirements, setup.py and demos ahead of release --- README.md | 80 ++++++++++++------- demo/apache/transforms/OverlayExample.py | 28 ++++--- demo/gunicorn/Dockerfile | 3 +- demo/gunicorn/prod-ssl.yml | 7 ++ demo/gunicorn/prod.yml | 2 +- demo/gunicorn/requirements.txt | 2 +- demo/gunicorn/transforms/OverlayExample.py | 28 ++++--- maltego_trx/maltego.py | 8 +- maltego_trx/overlays.py | 11 +-- .../template_dir/transforms/OverlayExample.py | 27 ++++--- setup.py | 47 ++++++----- 11 files changed, 148 insertions(+), 95 deletions(-) create mode 100644 demo/gunicorn/prod-ssl.yml diff --git a/README.md b/README.md index 54cae8a..91d7c1d 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ maltego-trx start new_project This will create a folder new_project with the recommend project structure. +Alternatively, you can copy either the `gunicorn` or `apache` example projects from the `demo` directory. +These also include Dockerfile and corresponding docker-compose configuration files for production deployment. + **Adding a Transform:** Add a new transform by creating a new python file in the "transforms" folder of your directory. @@ -207,16 +210,13 @@ The following constants can be imported from `maltego_trx.maltego`. Overlays Enums are imported from `maltego_trx.overlays` -*Overlay Position:* +*Overlay OverlayPosition:* - `NORTH = "N"` -- `NORTH_EAST = "NE"` -- `NORTH_WEST = "NW"` -- `EAST = "E"` -- `CENTER = "C"` -- `WEST = "W"` - `SOUTH = "S"` -- `SOUTH_EAST = "SE"` +- `WEST = "W"` +- `NORTH_WEST = "NW"` - `SOUTH_WEST = "SW"` +- `CENTER = "C"` *Overlay Type* - `IMAGE = "image"` @@ -240,15 +240,23 @@ The request/maltego msg object given to the transform contains the information a **Methods:** -- `getProperty(name: str)`: get a property value of the input entity -- `getTransformSetting(name: str)`: get a transform setting value +- `getProperty(name: str)`: Get a property value of the input entity +- `getTransformSetting(name: str)`: Get a transform setting value +- `clearLegacyProperties()`: Delete (duplicate) legacy properties from the input entity. This will not result in +property information being lost, it will simply clear out some properties that the TRX library duplicates on all +incoming Transform requests. In older versions of TRX, these Entity properties would have a different internal ID when +sent the server than what the Maltego client would advertise in the Entity Manager UI. For a list of Entities with such +properties and their corresponding legacy and actual IDs, see `entity_property_map` in `maltego_trx/entities.py`. For +the majority of projects this distinction can be safely ignored. ### Response/MaltegoTransform **Methods:** -- `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object created by the method. -- `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message type constant. +- `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object +created by the method. +- `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message +type constant. ### Entity @@ -258,27 +266,45 @@ The request/maltego msg object given to the transform contains the information a - `setValue(value: str)`: Set the entity value - `setWeight(weight: int)`: Set the entity weight - `addDisplayInformation(content: str, title: str)`: Add display information for the entity. -- `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. Matching rule can be `strict` or `loose`. -- `addOverlay(property_name: str, position:Position, overlay_type:OverlayType)`: Add an overlay to the entity. `Position` and `Type` are defined in the `maltego_tx.overlays` +- `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. +Matching rule can be `strict` or `loose`. +- `addOverlay(propertyName: str, position: OverlayPosition, overlay_type: OverlayType)`: Add an overlay to the entity. +`OverlayPosition` and `OverlayType` are defined in the `maltego_tx.overlays` Overlay can be added as Text, Image or Color ```python - # references the icon name `Champion` from the Maltego Desktop Client and this is will show up as an overlay on the graph - entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) - - # add a dynamic property - entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") - - # add the text value of the property `exampleDynamicPropertyName` as an overlay, any existing property name will work - entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) - - # add a color overlay - entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) - - # add a flag overlay - DE is an icon on the Maltego Desktop Client - entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) + person_name = request.Value + entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) + + # Normally, when we create an overlay, we would reference a property name so that Maltego can then use the + # value of that property to create the overlay. Sometimes that means creating a dynamic property, but usually + # it's better to either use an existing property, or, if you created the Entity yourself, and only need the + # property for the overlay, to use a hidden property. Here's an example of using a dynamic property: + entity.addProperty( + 'dynamic_overlay_icon_name', + displayName="Name for overlay image", + value="Champion" # references an icon in the Maltego client + ) + entity.addOverlay('dynamic_overlay_icon_name', OverlayPosition.WEST, OverlayType.IMAGE) + + # DISCOURAGED: + # You *can* also directly supply the string value of the property, however this is not recommended. Why? If + # the entity already has a property of the same ID (in this case, "DE"), then you would in fact be assigning the + # value of that property, not the string "DE", which is not the intention. Nevertheless, here's an example: + entity.addOverlay( + 'DE', # name of an icon, however, could also accidentally be a property name + OverlayPosition.SOUTH_WEST, + OverlayType.IMAGE + ) + + # Overlays can also be used to display extra text on an entity: + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Overlay Testing") + entity.addOverlay('exampleDynamicPropertyName', OverlayPosition.NORTH, OverlayType.TEXT) + + # Or a small color indicator: + entity.addOverlay('#45e06f', OverlayPosition.NORTH_WEST, OverlayType.COLOUR) ``` - `setIconURL(url: str)`: Set the entity icon URL diff --git a/demo/apache/transforms/OverlayExample.py b/demo/apache/transforms/OverlayExample.py index 34577a5..7bd8ecc 100644 --- a/demo/apache/transforms/OverlayExample.py +++ b/demo/apache/transforms/OverlayExample.py @@ -1,5 +1,5 @@ from maltego_trx.entities import Phrase -from maltego_trx.overlays import Position, OverlayType +from maltego_trx.overlays import OverlayPosition, OverlayType from maltego_trx.transform import DiscoverableTransform @@ -14,17 +14,23 @@ def create_entities(cls, request, response): person_name = request.Value entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) - # references the icon name `Champion` and this is will show up as an overlay on the graph - entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + # Normally, when we create an overlay, we would reference a property name so that Maltego can then use the + # value of that property to create the overlay. Sometimes that means creating a dynamic property, but usually + # it's better to either use an existing property, or, if you created the Entity yourself, and only need the + # property for the overlay, to use a hidden property. Here's an example of using a dynamic property: + entity.addProperty('dynamic_overlay_icon_name', displayName="Name for overlay image", value="Champion") + entity.addOverlay('dynamic_overlay_icon_name', OverlayPosition.WEST, OverlayType.IMAGE) - # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): - entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + # DISCOURAGED: + # You *can* also directly supply the string value of the property, however this is not recommended. Why? If + # the entity already has a property of the same ID (in this case, "DE"), then you would in fact be assigning the + # value of that property, not the string "DE", which is not the intention. Nevertheless, here's an example: + entity.addOverlay('DE', OverlayPosition.SOUTH_WEST, OverlayType.IMAGE) - # add the text of the property `exampleDynamicPropertyName` as an overlay - entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + # Overlays can also be used to display extra text on an entity: + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Overlay Testing") + entity.addOverlay('exampleDynamicPropertyName', OverlayPosition.NORTH, OverlayType.TEXT) - # add a color overlay - entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + # Or a small color indicator: + entity.addOverlay('#45e06f', OverlayPosition.NORTH_WEST, OverlayType.COLOUR) - # add a flag overlay - entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/demo/gunicorn/Dockerfile b/demo/gunicorn/Dockerfile index 66f56f7..42edd11 100644 --- a/demo/gunicorn/Dockerfile +++ b/demo/gunicorn/Dockerfile @@ -5,7 +5,7 @@ RUN mkdir /var/www/TRX/ WORKDIR /var/www/TRX/ # System dependencies -RUN apt-get update +RUN apt-get update -y RUN apt-get install python3-pip -y COPY requirements.txt requirements.txt @@ -18,4 +18,5 @@ COPY . /var/www/TRX/ RUN chown -R www-data:www-data /var/www/TRX/ +# for running a production server, use docker-compose with prod.yml or prod-ssl.yml CMD ["python3", "project.py", "runserver"] diff --git a/demo/gunicorn/prod-ssl.yml b/demo/gunicorn/prod-ssl.yml new file mode 100644 index 0000000..554c9a7 --- /dev/null +++ b/demo/gunicorn/prod-ssl.yml @@ -0,0 +1,7 @@ +version: '3' +services: + python: + build: . + command: "gunicorn --certfile=server.crt --keyfile=server.key --bind=0.0.0.0:8443 --threads=25 --workers=2 project:app" + ports: + - "8443:8443" diff --git a/demo/gunicorn/prod.yml b/demo/gunicorn/prod.yml index c243cc3..0a4c6dc 100644 --- a/demo/gunicorn/prod.yml +++ b/demo/gunicorn/prod.yml @@ -1,7 +1,7 @@ version: '3' services: python: - build: .. + build: . command: "gunicorn --bind=0.0.0.0:8080 --threads=25 --workers=2 project:app" ports: - "8080:8080" diff --git a/demo/gunicorn/requirements.txt b/demo/gunicorn/requirements.txt index c58373f..e9f1c7c 100644 --- a/demo/gunicorn/requirements.txt +++ b/demo/gunicorn/requirements.txt @@ -1 +1 @@ -maltego-trx \ No newline at end of file +maltego-trx>=1.3.8 diff --git a/demo/gunicorn/transforms/OverlayExample.py b/demo/gunicorn/transforms/OverlayExample.py index 34577a5..7bd8ecc 100644 --- a/demo/gunicorn/transforms/OverlayExample.py +++ b/demo/gunicorn/transforms/OverlayExample.py @@ -1,5 +1,5 @@ from maltego_trx.entities import Phrase -from maltego_trx.overlays import Position, OverlayType +from maltego_trx.overlays import OverlayPosition, OverlayType from maltego_trx.transform import DiscoverableTransform @@ -14,17 +14,23 @@ def create_entities(cls, request, response): person_name = request.Value entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) - # references the icon name `Champion` and this is will show up as an overlay on the graph - entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + # Normally, when we create an overlay, we would reference a property name so that Maltego can then use the + # value of that property to create the overlay. Sometimes that means creating a dynamic property, but usually + # it's better to either use an existing property, or, if you created the Entity yourself, and only need the + # property for the overlay, to use a hidden property. Here's an example of using a dynamic property: + entity.addProperty('dynamic_overlay_icon_name', displayName="Name for overlay image", value="Champion") + entity.addOverlay('dynamic_overlay_icon_name', OverlayPosition.WEST, OverlayType.IMAGE) - # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): - entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + # DISCOURAGED: + # You *can* also directly supply the string value of the property, however this is not recommended. Why? If + # the entity already has a property of the same ID (in this case, "DE"), then you would in fact be assigning the + # value of that property, not the string "DE", which is not the intention. Nevertheless, here's an example: + entity.addOverlay('DE', OverlayPosition.SOUTH_WEST, OverlayType.IMAGE) - # add the text of the property `exampleDynamicPropertyName` as an overlay - entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + # Overlays can also be used to display extra text on an entity: + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Overlay Testing") + entity.addOverlay('exampleDynamicPropertyName', OverlayPosition.NORTH, OverlayType.TEXT) - # add a color overlay - entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + # Or a small color indicator: + entity.addOverlay('#45e06f', OverlayPosition.NORTH_WEST, OverlayType.COLOUR) - # add a flag overlay - entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/maltego_trx/maltego.py b/maltego_trx/maltego.py index a2d5666..c5dc7b0 100644 --- a/maltego_trx/maltego.py +++ b/maltego_trx/maltego.py @@ -3,7 +3,7 @@ from xml.dom import minidom from .entities import Phrase, translate_legacy_property_name, entity_property_map -from .overlays import Position, OverlayType +from .overlays import OverlayPosition, OverlayType from .utils import remove_invalid_xml_chars BOOKMARK_COLOR_NONE = "-1" @@ -107,8 +107,10 @@ def setBookmark(self, bookmark): def setNote(self, note): self.addProperty('notes#', 'Notes', '', note) - def addOverlay(self, property_name=None, position=Position, overlay_type=OverlayType): - self.overlays.append([property_name, position.value, overlay_type.value]) + def addOverlay( + self, propertyName, position: OverlayPosition, overlayType: OverlayType + ): + self.overlays.append([propertyName, position.value, overlayType.value]) def add_field_to_xml(self, additional_field): name, display, matching, value = additional_field diff --git a/maltego_trx/overlays.py b/maltego_trx/overlays.py index 9cc2747..1c897b0 100644 --- a/maltego_trx/overlays.py +++ b/maltego_trx/overlays.py @@ -1,16 +1,13 @@ from enum import Enum -class Position(Enum): +class OverlayPosition(Enum): NORTH = "N" - NORTH_EAST = "NE" - NORTH_WEST = "NW" - EAST = "E" - CENTER = "C" - WEST = "W" SOUTH = "S" - SOUTH_EAST = "SE" + WEST = "W" + NORTH_WEST = "NW" SOUTH_WEST = "SW" + CENTER = "C" class OverlayType(Enum): diff --git a/maltego_trx/template_dir/transforms/OverlayExample.py b/maltego_trx/template_dir/transforms/OverlayExample.py index 34577a5..e4478d2 100644 --- a/maltego_trx/template_dir/transforms/OverlayExample.py +++ b/maltego_trx/template_dir/transforms/OverlayExample.py @@ -1,5 +1,5 @@ from maltego_trx.entities import Phrase -from maltego_trx.overlays import Position, OverlayType +from maltego_trx.overlays import OverlayPosition, OverlayType from maltego_trx.transform import DiscoverableTransform @@ -14,17 +14,22 @@ def create_entities(cls, request, response): person_name = request.Value entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) - # references the icon name `Champion` and this is will show up as an overlay on the graph - entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + # Normally, when we create an overlay, we would reference a property name so that Maltego can then use the + # value of that property to create the overlay. Sometimes that means creating a dynamic property, but usually + # it's better to either use an existing property, or, if you created the Entity yourself, and only need the + # property for the overlay, to use a hidden property. Here's an example of using a dynamic property: + entity.addProperty('dynamic_overlay_icon_name', displayName="Name for overlay image", value="Champion") + entity.addOverlay('dynamic_overlay_icon_name', OverlayPosition.WEST, OverlayType.IMAGE) - # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): - entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + # You *can* also directly supply the string value of the property, however this is not recommended. Why? If + # the entity already has a property of the same ID (in this case, "DE"), then you would in fact be assigning the + # value of that property, not the string "DE", which is not the intention. Nevertheless, here's an example: + entity.addOverlay('DE', OverlayPosition.SOUTH_WEST, OverlayType.IMAGE) - # add the text of the property `exampleDynamicPropertyName` as an overlay - entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + # Overlays can also be an additional field of text displayed on the entity: + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Overlay Testing") + entity.addOverlay('exampleDynamicPropertyName', OverlayPosition.NORTH, OverlayType.TEXT) - # add a color overlay - entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + # Or a small color indicator + entity.addOverlay('#45e06f', OverlayPosition.NORTH_WEST, OverlayType.COLOUR) - # add a flag overlay - entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/setup.py b/setup.py index edc23fc..be1ee8b 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,28 @@ from setuptools import setup from maltego_trx import VERSION -setup(name='maltego-trx', - version=VERSION, - description='Python library used to develop Maltego transforms', - url='https://github.com/paterva/maltego-trx/', - author='Maltego Staff', - author_email='support@maltego.com', - license='MIT', - install_requires=[ - 'flask>=1', - 'six>=1', - 'cryptography>=3.3.1' - ], - packages=[ - 'maltego_trx', - 'maltego_trx/template_dir', - 'maltego_trx/template_dir/transforms' - ], - entry_points={'console_scripts': [ - 'maltego-trx = maltego_trx.commands:execute_from_command_line', - ]}, - zip_safe=False - ) +setup( + name='maltego-trx', + version=VERSION, + description='Python library used to develop Maltego transforms', + url='https://github.com/paterva/maltego-trx/', + author='Maltego Staff', + author_email='support@maltego.com', + license='MIT', + install_requires=[ + 'flask>=1', + 'six>=1', + 'cryptography==3.3.2' # pinned for now as newer versions require setuptools_rust + ], + packages=[ + 'maltego_trx', + 'maltego_trx/template_dir', + 'maltego_trx/template_dir/transforms' + ], + entry_points={ + 'console_scripts': [ + 'maltego-trx = maltego_trx.commands:execute_from_command_line', + ] + }, + zip_safe=False +) From 0330cae950928b2f94fae7c5280987ee4c2f3570 Mon Sep 17 00:00:00 2001 From: Matt Coates Date: Mon, 8 Mar 2021 10:29:27 -0500 Subject: [PATCH 22/27] Exception XML message typo: wrong closing tag tag was closed by --- maltego_trx/maltego.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maltego_trx/maltego.py b/maltego_trx/maltego.py index c5dc7b0..6fc8b41 100644 --- a/maltego_trx/maltego.py +++ b/maltego_trx/maltego.py @@ -189,7 +189,7 @@ def throwExceptions(self): lines.append("") for exception in self.exceptions: - lines.append("%s" % remove_invalid_xml_chars(exception)) + lines.append("%s" % remove_invalid_xml_chars(exception)) lines.append("") lines.append("") From d51f7641062c71f93af1ce6cc2499a4ad0cea7d3 Mon Sep 17 00:00:00 2001 From: Maltego Staff <43576722+hlawatjt@users.noreply.github.com> Date: Mon, 14 Jun 2021 16:19:23 +0200 Subject: [PATCH 23/27] fix(readme.md): Debug message update. Included a note regarding debug filter settings in the desktop client. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 91d7c1d..8c0cca9 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,9 @@ The following constants can be imported from `maltego_trx.maltego`. - `UIM_INFORM` - `UIM_DEBUG` +**Please take note:** +You need to enable the `debug` filter option in the Desktop client Output window to view `debug` transform messages. + **Bookmark Colors:** - `BOOKMARK_COLOR_NONE` From b03ff58a5851c4806ff50293a8bf35bf67d58315 Mon Sep 17 00:00:00 2001 From: felixhertrampf Date: Tue, 29 Jun 2021 12:13:42 +0200 Subject: [PATCH 24/27] feat: added venv to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dcf5b77..ade784c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ __pycache__ maltego_trx.egg-info .pytest_cache +venv \ No newline at end of file From 1d5c15674cedd2bcfce3123b250e51b2a4e5f123 Mon Sep 17 00:00:00 2001 From: Felix Hertrampf Date: Thu, 8 Jul 2021 13:04:26 +0200 Subject: [PATCH 25/27] feat: added registry addon to auto-generate CSV config files for pTDS and iTDS" --- README.md | 214 ++++++++++++++++-- maltego_trx/decorator_registry.py | 135 +++++++++++ maltego_trx/template_dir/extensions.py | 23 ++ maltego_trx/template_dir/project.py | 4 + maltego_trx/template_dir/settings.py | 13 ++ .../transforms/GreetPersonLocalized.py | 28 +++ maltego_trx/utils.py | 69 +++++- tests/requirements.txt | 3 + tests/test_registry.py | 165 ++++++++++++++ 9 files changed, 628 insertions(+), 26 deletions(-) create mode 100644 maltego_trx/decorator_registry.py create mode 100644 maltego_trx/template_dir/extensions.py create mode 100644 maltego_trx/template_dir/settings.py create mode 100644 maltego_trx/template_dir/transforms/GreetPersonLocalized.py create mode 100644 tests/requirements.txt create mode 100644 tests/test_registry.py diff --git a/README.md b/README.md index 8c0cca9..f975d26 100644 --- a/README.md +++ b/README.md @@ -10,22 +10,23 @@ To install the trx library run the following command: pip install maltego-trx ``` -After installing you can create a new project by running the following command: +After installing, you can create a new project by running the following command: ``` bash maltego-trx start new_project ``` -This will create a folder new_project with the recommend project structure. +This will create a folder new_project with the recommended project structure. -Alternatively, you can copy either the `gunicorn` or `apache` example projects from the `demo` directory. -These also include Dockerfile and corresponding docker-compose configuration files for production deployment. +Alternatively, you can copy either the `gunicorn` or `apache` example projects from the `demo` directory. These also +include Dockerfile and corresponding docker-compose configuration files for production deployment. **Adding a Transform:** Add a new transform by creating a new python file in the "transforms" folder of your directory. -Any file in the folder where the **class name matches the filename** and the class inherits from Transform, will automatically be discovered and added to your server. +Any file in the folder where the **class name matches the filename**, and the class inherits from Transform, will +automatically be discovered and added to your server. A simple transform would look like the following: @@ -72,11 +73,13 @@ gunicorn --bind=0.0.0.0:8080 --threads=25 --workers=2 project:app ## Run a Docker Transform server -The `demo` folder provides an example project. The Docker files given can be used to setup and run your project in Docker. +The `demo` folder provides an example project. The Docker files given can be used to set up and run your project in +Docker. The Dockerfile and docker-compose file can be used to easily setup and run a development transform server. -If you have copied the `docker-compose.yml`, `Dockerfile` and `prod.yml` files into your project, then you can use the following commands to run the server in Docker. +If you have copied the `docker-compose.yml`, `Dockerfile` and `prod.yml` files into your project, then you can use the +following commands to run the server in Docker. Run the following to start the development server: @@ -115,12 +118,170 @@ The following values are not passed to local transforms, and will have dummy val - `slider`: 100 - `transformSettings`: {} +## Using the Transform Registry + +###### Added in 1.4.0 (July 2021) + +The Transform Registry enables you to annotate Transforms with metadata like display name, description, input and output +entities as well as settings. The Transform Registry will automatically generate CSV files that you can import into the +pTDS and/or your iTDS. + +### Configuring the Registry + +You can configure your registry with all the info you would normally add for every transform/seed on a TDS. We recommend +creating your registry in an extra file, traditionally called `extensions.py`, to avoid circular imports. + +```python +# extensions.py +from settings import api_key_setting + +from maltego_trx.decorator_registry import TransformRegistry + +registry = TransformRegistry( + owner="ACME Corporation", + author="John Doe ", + host_url="https://transforms.acme.org", + seed_ids=["demo"] +) + +# The rest of these attributes are optional + +# metadata +registry.version = "0.1" + +# global settings +registry.global_settings = [api_key_setting] + +# transform suffix to indicate datasource +registry.display_name_suffix = " [ACME]" + +# reference OAuth settings +registry.oauth_settings_id = ['github-oauth'] + +``` + +### Annotating Transforms + +```python +# transforms/GreetPerson.py +... +from maltego_trx.server import registry + + +@registry.register_transform(display_name='Greet Person', + input_entity='maltego.Phrase', + description='Returns a phrase greeting a person on the graph.', + output_entities=['maltego.Phrase'], + disclaimer='This disclaimer is optional and has to be accepted before this transform is run') +class GreetPerson(DiscoverableTransform): + + @classmethod + def create_entities(cls, request, response): + ... +``` + +### Transform Settings + +You can declare transform settings in a central location and add them to the registry. + +#### Configuring Global Settings + +These settings will apply to all transforms which can be very helpful for api keys. + +```python +# settings.py +from maltego_trx.decorator_registry import TransformSetting + +api_key_setting = TransformSetting(name='api_key', + display_name='API Key', + setting_type='string', + global_setting=True) +``` + +```python +# extensions.py +from maltego_trx.template_dir.settings import api_key_setting + +from maltego_trx.decorator_registry import TransformRegistry + +registry = TransformRegistry( + owner="ACME Corporation", + author="John Doe ", + host_url="https://transforms.acme.org", + seed_ids=["demo"] +) + +registry.global_settings = [api_key_setting] +``` + +#### Configuring Settings per Transform + +Settings that aren't required for every transform have to be added to the `register_transform` decorator explicitly. + +```python +# settings.py +... + +language_setting = TransformSetting(name='language', + display_name="Language", + setting_type='string', + default_value='en', + optional=True, + popup=True) +``` + +```python +# transforms/GreetPerson.py +... +from maltego_trx.template_dir.settings import language_setting + +from maltego_trx.transform import DiscoverableTransform + + +@registry.register_transform(display_name="Greet Person", + input_entity="maltego.Phrase", + description='Returns a phrase greeting a person on the graph.', + settings=[language_setting]) +class GreetPerson(DiscoverableTransform): + + @classmethod + def create_entities(cls, request: MaltegoMsg, response: MaltegoTransform): + language = request.getTransformSetting(language_setting.name) + ... +``` + +### Exporting the TDS Configuration + +To export the configurations, use the registry methods `write_transforms_config()` and `write_settings_config()`. These +methods have to executed after they have been registered with the TRX server. + +```python +# project.py + +import sys +import transforms + +from maltego_trx.registry import register_transform_function, register_transform_classes +from maltego_trx.server import app, application +from maltego_trx.handler import handle_run + +# register_transform_function(transform_func) +from maltego_trx.template_dir.extensions import registry + +register_transform_classes(transforms) + +registry.write_transforms_config() +registry.write_settings_config() + +handle_run(__name__, sys.argv, app) +``` + ## Legacy Transforms [Documentation](https://docs.maltego.com/support/solutions/articles/15000018299-porting-old-trx-transforms-to-the-latest-version) -If you have old TRX transforms that are written as functions, -they can be registered with the server using the `maltego_trx.registry.register_transform_function` method. +If you have old TRX transforms that are written as functions, they can be registered with the server using +the `maltego_trx.registry.register_transform_function` method. In order to port your old transforms, make two changes: @@ -143,6 +304,7 @@ To: from maltego_trx.maltego import MaltegoTransform + def old_transform(m): ``` @@ -214,6 +376,7 @@ You need to enable the `debug` filter option in the Desktop client Output window Overlays Enums are imported from `maltego_trx.overlays` *Overlay OverlayPosition:* + - `NORTH = "N"` - `SOUTH = "S"` - `WEST = "W"` @@ -222,6 +385,7 @@ Overlays Enums are imported from `maltego_trx.overlays` - `CENTER = "C"` *Overlay Type* + - `IMAGE = "image"` - `COLOUR = "colour"` - `TEXT = "text"` @@ -238,28 +402,28 @@ The request/maltego msg object given to the transform contains the information a - `Type: str`: The input entity type - `Properties: dict(str: str)`: A key-value dictionary of the input entity properties - `TransformSettings: dict(str: str)`: A key-value dictionary of the transform settings -- `Genealogy: list(dict(str: str))`: A key-value dictionary of the Entity genealogy, - this is only applicable for extended entities e.g. Website Entity +- `Genealogy: list(dict(str: str))`: A key-value dictionary of the Entity genealogy, this is only applicable for + extended entities e.g. Website Entity **Methods:** - `getProperty(name: str)`: Get a property value of the input entity - `getTransformSetting(name: str)`: Get a transform setting value -- `clearLegacyProperties()`: Delete (duplicate) legacy properties from the input entity. This will not result in -property information being lost, it will simply clear out some properties that the TRX library duplicates on all -incoming Transform requests. In older versions of TRX, these Entity properties would have a different internal ID when -sent the server than what the Maltego client would advertise in the Entity Manager UI. For a list of Entities with such -properties and their corresponding legacy and actual IDs, see `entity_property_map` in `maltego_trx/entities.py`. For -the majority of projects this distinction can be safely ignored. +- `clearLegacyProperties()`: Delete (duplicate) legacy properties from the input entity. This will not result in + property information being lost, it will simply clear out some properties that the TRX library duplicates on all + incoming Transform requests. In older versions of TRX, these Entity properties would have a different internal ID when + sent the server than what the Maltego client would advertise in the Entity Manager UI. For a list of Entities with + such properties and their corresponding legacy and actual IDs, see `entity_property_map` in `maltego_trx/entities.py`. + For the majority of projects this distinction can be safely ignored. ### Response/MaltegoTransform **Methods:** -- `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object -created by the method. -- `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message -type constant. +- `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object + created by the method. +- `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message + type constant. ### Entity @@ -269,10 +433,10 @@ type constant. - `setValue(value: str)`: Set the entity value - `setWeight(weight: int)`: Set the entity weight - `addDisplayInformation(content: str, title: str)`: Add display information for the entity. -- `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. -Matching rule can be `strict` or `loose`. -- `addOverlay(propertyName: str, position: OverlayPosition, overlay_type: OverlayType)`: Add an overlay to the entity. -`OverlayPosition` and `OverlayType` are defined in the `maltego_tx.overlays` +- `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. Matching + rule can be `strict` or `loose`. +- `addOverlay(propertyName: str, position: OverlayPosition, overlay_type: OverlayType)`: Add an overlay to the entity. + `OverlayPosition` and `OverlayType` are defined in the `maltego_tx.overlays` Overlay can be added as Text, Image or Color diff --git a/maltego_trx/decorator_registry.py b/maltego_trx/decorator_registry.py new file mode 100644 index 0000000..fcf3b14 --- /dev/null +++ b/maltego_trx/decorator_registry.py @@ -0,0 +1,135 @@ +import os +from dataclasses import dataclass, field +from itertools import chain +from typing import List, Literal, Optional, Dict, Iterable + +from maltego_trx.utils import filter_unique, pascal_case_to_title, escape_csv_fields, export_as_csv, serialize_bool, \ + name_to_path + +TRANSFORMS_CSV_HEADER = "Owner,Author,Disclaimer,Description,Version," \ + "Name,UIName,URL,entityName," \ + "oAuthSettingId,transformSettingIDs,seedIDs" +SETTINGS_CSV_HEADER = "Name,Type,Display,DefaultValue,Optional,Popup" + + +@dataclass() +class TransformMeta: + class_name: str + display_name: str + input_entity: str + description: str + output_entities: List[str] + disclaimer: str + + +@dataclass() +class TransformSetting: + name: str + display_name: str + setting_type: Literal['string', 'boolean', 'date', 'datetime', 'daterange', 'url', 'double', 'int'] + + default_value: Optional[str] = "" + optional: bool = False + popup: bool = False + global_setting: bool = False + + @property + def id(self) -> str: + """this setting's full id for reference""" + if self.global_setting: + return "global#" + self.name + return self.name + + +@dataclass(eq=False) +class TransformRegistry: + owner: str + author: str + + host_url: str + seed_ids: List[str] + + version: str = '0.1' + display_name_suffix: str = "" + + global_settings: List[TransformSetting] = field(default_factory=list) + oauth_settings_id: Optional[str] = "" + + transform_metas: Dict[str, TransformMeta] = field(init=False, default_factory=dict) + transform_settings: Dict[str, List[TransformSetting]] = field(init=False, default_factory=dict) + + def register_transform(self, display_name: str, input_entity: str, description: str, + settings: List[TransformSetting] = None, output_entities: List[str] = None, + disclaimer: str = ""): + """ This method can be used as a decorator on transform classes. The data will be used to fill out csv config + files to be imported into a TDS. + """ + + def decorated(transform_callable: object): + cleaned_transform_name = name_to_path(transform_callable.__name__) + display = display_name or pascal_case_to_title(transform_callable.__name__) + + meta = TransformMeta(cleaned_transform_name, + display, input_entity, + description, + output_entities or [], + disclaimer) + self.transform_metas[cleaned_transform_name] = meta + + if settings: + self.transform_settings[cleaned_transform_name] = settings + + return transform_callable + + return decorated + + def write_transforms_config(self, config_path: str = "./transforms.csv", csv_line_limit: int = 100): + """Exports the collected transform meta data as a csv-file to config_path""" + global_settings_full_names = [gs.id for gs in self.global_settings] + + csv_lines = [] + for transform_name, transform_meta in self.transform_metas.items(): + meta_settings = [setting.id for setting in + self.transform_settings.get(transform_name, [])] + + transform_row = [ + self.owner, + self.author, + transform_meta.disclaimer, + transform_meta.description, + self.version, + transform_name, + transform_meta.display_name + self.display_name_suffix, + os.path.join(self.host_url, "run", transform_name), + transform_meta.input_entity, + ";".join(self.oauth_settings_id), + # combine global and transform scoped settings + ";".join(chain(meta_settings, global_settings_full_names)), + ";".join(self.seed_ids) + ] + + escaped_fields = escape_csv_fields(*transform_row) + csv_lines.append(",".join(escaped_fields)) + + export_as_csv(TRANSFORMS_CSV_HEADER, csv_lines, config_path, csv_line_limit) + + def write_settings_config(self, config_path: str = "./settings.csv", csv_line_limit: int = 100): + """Exports the collected settings meta data as a csv-file to config_path""" + chained_settings = chain(self.global_settings, *list(self.transform_settings.values())) + unique_settings: Iterable[TransformSetting] = filter_unique(lambda s: s.name, chained_settings) + + csv_lines = [] + for setting in unique_settings: + setting_row = [ + setting.id, + setting.setting_type, + setting.display_name, + setting.default_value or "", + serialize_bool(setting.optional, 'True', 'False'), + serialize_bool(setting.popup, 'Yes', 'No') + ] + + escaped_fields = escape_csv_fields(*setting_row) + csv_lines.append(",".join(escaped_fields)) + + export_as_csv(SETTINGS_CSV_HEADER, csv_lines, config_path, csv_line_limit) diff --git a/maltego_trx/template_dir/extensions.py b/maltego_trx/template_dir/extensions.py new file mode 100644 index 0000000..2ff99fa --- /dev/null +++ b/maltego_trx/template_dir/extensions.py @@ -0,0 +1,23 @@ +from maltego_trx.decorator_registry import TransformRegistry + +registry = TransformRegistry( + owner="ACME Corporation", + author="John Doe ", + host_url="https://transforms.acme.com", + seed_ids=["demo"] +) + +# The rest of these attributes are optional + +# metadata +registry.version = "0.1" + +# global settings +# from maltego_trx.template_dir.settings import api_key_setting +# registry.global_settings = [api_key_setting] + +# transform suffix to indicate datasource +# registry.display_name_suffix = " [ACME]" + +# reference OAuth settings +# registry.oauth_settings_id = ['github-oauth'] diff --git a/maltego_trx/template_dir/project.py b/maltego_trx/template_dir/project.py index 09ab5f6..fb841ca 100644 --- a/maltego_trx/template_dir/project.py +++ b/maltego_trx/template_dir/project.py @@ -4,8 +4,12 @@ from maltego_trx.registry import register_transform_function, register_transform_classes from maltego_trx.server import app, application from maltego_trx.handler import handle_run +from maltego_trx.template_dir.extensions import registry # register_transform_function(transform_func) register_transform_classes(transforms) +registry.write_transforms_config() +registry.write_settings_config() + handle_run(__name__, sys.argv, app) diff --git a/maltego_trx/template_dir/settings.py b/maltego_trx/template_dir/settings.py new file mode 100644 index 0000000..1373702 --- /dev/null +++ b/maltego_trx/template_dir/settings.py @@ -0,0 +1,13 @@ +from maltego_trx.decorator_registry import TransformSetting + +api_key_setting = TransformSetting(name='api_key', + display_name='API Key', + setting_type='string', + global_setting=True) + +language_setting = TransformSetting(name='language', + display_name="Language", + setting_type='string', + default_value='en', + optional=True, + popup=True) diff --git a/maltego_trx/template_dir/transforms/GreetPersonLocalized.py b/maltego_trx/template_dir/transforms/GreetPersonLocalized.py new file mode 100644 index 0000000..3be6df3 --- /dev/null +++ b/maltego_trx/template_dir/transforms/GreetPersonLocalized.py @@ -0,0 +1,28 @@ +from maltego_trx.entities import Phrase +from maltego_trx.maltego import MaltegoTransform, MaltegoMsg +from maltego_trx.template_dir.extensions import registry +from maltego_trx.template_dir.settings import language_setting + +from maltego_trx.transform import DiscoverableTransform + + +@registry.register_transform(display_name="Greet Person (localized)", input_entity="maltego.Phrase", + description='Returns a localized phrase greeting a person on the graph.', + settings=[language_setting], + output_entities=["maltego.Phrase"]) +class GreetPersonLocalized(DiscoverableTransform): + + @classmethod + def create_entities(cls, request: MaltegoMsg, response: MaltegoTransform): + person_name = request.Value + + language: str = request.getTransformSetting(language_setting.id).lower() + + if language == 'af': + greeting = f"Hallo {person_name}, lekker om jou te ontmoet!" + elif language == "de": + greeting = f"Moin {person_name}, schön dich kennen zu lernen!" + else: + greeting = f"Hello {person_name}, nice to meet you!" + + response.addEntity(Phrase, greeting) diff --git a/maltego_trx/utils.py b/maltego_trx/utils.py index e5b349a..272d87d 100644 --- a/maltego_trx/utils.py +++ b/maltego_trx/utils.py @@ -1,4 +1,7 @@ import re +from typing import TypeVar, Callable, Hashable, Iterable, Generator, List, Sequence + +import math from six import text_type, binary_type @@ -45,4 +48,68 @@ def remove_invalid_xml_chars(val): """ val = make_utf8(val) val = re.sub(u'[^\u0020-\uD7FF\u0009\u000A\u000D\uE000-\uFFFD\U00010000-\U0010FFFF]+', '?', val) - return val \ No newline at end of file + return val + + +T = TypeVar('T') + + +def filter_unique(get_identifier: Callable[[T], Hashable], collection: Iterable[T]) -> Generator[T, None, None]: + seen = set() + for item in collection: + identifier = get_identifier(item) + + if identifier in seen: + continue + + seen.add(identifier) + + yield item + + +def chunk_list(data: Sequence[T], max_chunk_size: int) -> Generator[Sequence[T], None, None]: + # math.ceil: + # number_of_chunks: decimal-places == 0 -> perfect split + # number_of_chunks: decimal-places > 0 -> need one more list to keep len(chunk) <= max_chunk_size + + number_of_chunks = math.ceil(len(data) / max_chunk_size) + chunk_size = math.ceil(len(data) / number_of_chunks) + + for idx in range(0, len(data), chunk_size): + yield data[idx:idx + chunk_size] + + +def pascal_case_to_title(name: str) -> str: + # https://stackoverflow.com/a/1176023 + name = re.sub('(.)([A-Z][a-z]+)', r'\1 \2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1 \2', name) + + +def escape_csv_fields(*fields: str, separator: str = ',') -> Generator[str, None, None]: + """if a field contains the separator, it will be quoted""" + for f in fields: + yield f'"{f}"' if separator in f else f + + +def export_as_csv(header: str, lines: Sequence[str], export_file_path: str, csv_line_limit: int = -1): + """export a file in as many files as needed to stay below the csv_line_limit (plus header)""" + if csv_line_limit == -1 or len(lines) <= csv_line_limit: + with open(export_file_path, "w+") as csv_file: + csv_file.write(header + "\n") + csv_file.writelines(map(lambda x: x + "\n", lines)) + + return + + # split file to speed-up import into pTDS, iTDS + chunks = list(chunk_list(lines, csv_line_limit)) + for idx, chunk in enumerate(chunks, 1): + path, extension = export_file_path.rsplit(".", 1) + chunked_config_path = f"{path}_{idx}-{len(chunks)}.{extension}" + + with open(chunked_config_path, "w+") as csv_file: + csv_file.write(header + "\n") + csv_file.writelines(map(lambda x: x + "\n", chunk)) + + +def serialize_bool(boolean: bool, serialized_true: str, serialized_false: str) -> str: + return serialized_true if boolean else serialized_false diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..76377c7 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +pytest==6.2.4 + +petname==2.6 \ No newline at end of file diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 0000000..2200e97 --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,165 @@ +import os +import random +from typing import NamedTuple, List + +import petname +import pytest as pytest + +from maltego_trx.decorator_registry import TransformSetting, TransformRegistry, TRANSFORMS_CSV_HEADER, \ + SETTINGS_CSV_HEADER +from maltego_trx.server import app +from maltego_trx.utils import name_to_path, serialize_bool + + +@pytest.fixture +def client(): + with app.test_client() as client: + yield client + + +@pytest.fixture +def registry(): + registry: TransformRegistry = TransformRegistry(owner="Maltego Technologies GmbH", + author="Maltego Support", + host_url="localhost", + seed_ids=["demo"]) + return registry + + +def make_transform_setting(): + name = petname.generate() + setting_type = random.choice(['string', 'boolean', 'date', 'datetime', 'daterange', 'url', 'double', 'int']) + + return TransformSetting(name=name, + display_name=name.title(), + setting_type=random.choice(setting_type), + default_value=petname.generate(), + optional=random.choice([True, False]), + popup=random.choice([True, False]), + global_setting=random.choice([True, False])) + + +def make_transform(registry: TransformRegistry, settings: List[TransformSetting] = None): + display_name = petname.generate(separator=" ") + input_entity = petname.generate(separator=".") + description = petname.generate(words=10, separator=" ").title() + "." + settings = settings or [make_transform_setting(), make_transform_setting()] + output_entities = petname.generate(3).split("-") + disclaimer = petname.generate(words=10, separator=" ").title() + "." + + @registry.register_transform(display_name, input_entity, description, settings, output_entities, disclaimer) + class TestClass: + pass + + return TestClass + + +def test_register_transform_decorator(registry): + test_settings = [make_transform_setting(), make_transform_setting()] + + display_name = petname.generate(separator=" ") + input_entity = petname.generate(separator=".") + description = petname.generate(words=10, separator=" ").title() + "." + output_entities = petname.generate(3).split("-") + disclaimer = petname.generate(words=10, separator=" ").title() + "." + + @registry.register_transform(display_name, input_entity, description, test_settings, output_entities, disclaimer) + class TestClass: + pass + + path_name = name_to_path(TestClass.__name__) + + tx_meta = registry.transform_metas.get(path_name) + + assert tx_meta + assert tx_meta.display_name == display_name + assert tx_meta.input_entity == input_entity + assert tx_meta.description == description + assert tx_meta.disclaimer == disclaimer + + assert test_settings == registry.transform_settings[path_name] + + +class TransformCsvLine(NamedTuple): + owner: str + author: str + disclaimer: str + description: str + version: str + name: str + display_name: str + host: str + input_entity: str + oauth_id: str + settings_ids: str + seed_ids: str + + +class SettingCsvLine(NamedTuple): + name: str + setting_type: str + display_name: str + default: str + optional: str + popup: str + + +def test_transform_to_csv(registry): + random_class = make_transform(registry) + + path_name = name_to_path(random_class.__name__) + + tx_meta = registry.transform_metas.get(path_name) + tx_settings = registry.transform_settings.get(path_name, []) + + registry.write_transforms_config() + + with open("./transforms.csv") as transforms_csv: + header = next(transforms_csv) + assert header.rstrip("\n") == TRANSFORMS_CSV_HEADER + + line = next(transforms_csv).rstrip("\n") + data: TransformCsvLine = TransformCsvLine(*line.split(',')) + + assert data.owner == registry.owner + assert data.author == registry.author + assert data.disclaimer == tx_meta.disclaimer + assert data.description == tx_meta.description + assert data.version == registry.version + assert data.name == tx_meta.class_name + assert data.display_name == tx_meta.display_name + assert data.host == os.path.join(registry.host_url, "run", path_name) + assert data.input_entity == tx_meta.input_entity + assert data.oauth_id == registry.oauth_settings_id + assert data.settings_ids.split(";") == [s.id for s in tx_settings] + assert data.seed_ids.split(";") == registry.seed_ids + + +def test_setting_to_csv(registry): + local_setting = make_transform_setting() + local_setting.global_setting = False + + global_setting = make_transform_setting() + global_setting.global_setting = True + + registry.global_settings.append(global_setting) + + @registry.register_transform("", "", "", settings=[local_setting]) + class TestClass: + pass + + registry.write_settings_config() + with open("./settings.csv") as settings_csv: + header = next(settings_csv) + assert header.rstrip("\n") == SETTINGS_CSV_HEADER + + for line, setting in zip(settings_csv.readlines(), [global_setting, local_setting]): + line = line.rstrip("\n") + data: SettingCsvLine = SettingCsvLine(*line.split(',')) + + assert data.name == setting.id + assert data.setting_type == setting.setting_type + assert data.display_name == setting.display_name + assert data.default == setting.default_value + assert data.optional == serialize_bool(setting.optional, 'True', 'False') + assert data.popup == serialize_bool(setting.popup, 'Yes', 'No') From 0f3105034313c49dc7e14205f04380ab7695aaca Mon Sep 17 00:00:00 2001 From: Felix Hertrampf Date: Thu, 8 Jul 2021 13:33:57 +0200 Subject: [PATCH 26/27] feat: bumped version to 1.4.0 --- maltego_trx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maltego_trx/__init__.py b/maltego_trx/__init__.py index 3ba22d0..af63e4a 100644 --- a/maltego_trx/__init__.py +++ b/maltego_trx/__init__.py @@ -1 +1 @@ -VERSION = "1.3.8" +VERSION = "1.4.0" From 8c4f38c841aaf11633c3569d0ef77bebccf355d3 Mon Sep 17 00:00:00 2001 From: Felix Hertrampf Date: Thu, 8 Jul 2021 14:22:35 +0200 Subject: [PATCH 27/27] feat: added pro tip about auto `display_name` from transform class name --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f975d26..898eefb 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,12 @@ class GreetPerson(DiscoverableTransform): ... ``` +**Pro Tip:** If the `display_name` is either `None` or `""`, the registry will try to create a display name from the class +name: + +- `DNSToIP` 'DNS To IP' +- `GreetPerson` 'Greet Person' + ### Transform Settings You can declare transform settings in a central location and add them to the registry.