From 0f4727ffa0fcb433bbba654d1a9a5c1737d13a54 Mon Sep 17 00:00:00 2001 From: Tendai Marengereke Date: Tue, 2 Mar 2021 16:34:49 +0200 Subject: [PATCH 1/4] feat: add Genealogy element parsing and overlay functionality --- README.md | 47 +++++++++++++++++++ demo/apache/transforms/OverlayExample.py | 30 ++++++++++++ demo/gunicorn/transforms/OverlayExample.py | 30 ++++++++++++ maltego_trx/maltego.py | 37 +++++++++++++-- maltego_trx/overlays.py | 19 ++++++++ .../template_dir/transforms/OverlayExample.py | 30 ++++++++++++ maltego_trx/test_hierarchical_entity.xml | 13 +++++ 7 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 demo/apache/transforms/OverlayExample.py create mode 100644 demo/gunicorn/transforms/OverlayExample.py create mode 100644 maltego_trx/overlays.py create mode 100644 maltego_trx/template_dir/transforms/OverlayExample.py create mode 100644 maltego_trx/test_hierarchical_entity.xml diff --git a/README.md b/README.md index 47b9981..54cae8a 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,28 @@ The following constants can be imported from `maltego_trx.maltego`. - `LINK_STYLE_DOTTED` - `LINK_STYLE_DASHDOT` +### Enums + +**Overlays:** + +Overlays Enums are imported from `maltego_trx.overlays` + +*Overlay Position:* +- `NORTH = "N"` +- `NORTH_EAST = "NE"` +- `NORTH_WEST = "NW"` +- `EAST = "E"` +- `CENTER = "C"` +- `WEST = "W"` +- `SOUTH = "S"` +- `SOUTH_EAST = "SE"` +- `SOUTH_WEST = "SW"` + +*Overlay Type* +- `IMAGE = "image"` +- `COLOUR = "colour"` +- `TEXT = "text"` + ### Request/MaltegoMsg The request/maltego msg object given to the transform contains the information about the input entity. @@ -213,6 +235,8 @@ The request/maltego msg object given to the transform contains the information a - `Type: str`: The input entity type - `Properties: dict(str: str)`: A key-value dictionary of the input entity properties - `TransformSettings: dict(str: str)`: A key-value dictionary of the transform settings +- `Genealogy: list(dict(str: str))`: A key-value dictionary of the Entity genealogy, + this is only applicable for extended entities e.g. Website Entity **Methods:** @@ -235,9 +259,32 @@ The request/maltego msg object given to the transform contains the information a - `setWeight(weight: int)`: Set the entity weight - `addDisplayInformation(content: str, title: str)`: Add display information for the entity. - `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. Matching rule can be `strict` or `loose`. +- `addOverlay(property_name: str, position:Position, overlay_type:OverlayType)`: Add an overlay to the entity. `Position` and `Type` are defined in the `maltego_tx.overlays` + +Overlay can be added as Text, Image or Color + +```python + + # references the icon name `Champion` from the Maltego Desktop Client and this is will show up as an overlay on the graph + entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + + # add a dynamic property + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + + # add the text value of the property `exampleDynamicPropertyName` as an overlay, any existing property name will work + entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + + # add a color overlay + entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + + # add a flag overlay - DE is an icon on the Maltego Desktop Client + entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) +``` + - `setIconURL(url: str)`: Set the entity icon URL - `setBookmark(bookmark: int)`: Set bookmark color index (e.g. -1 for BOOKMARK_COLOR_NONE, 3 for BOOKMARK_COLOR_PURPLE) - `setNote(note: str)`: Set note content +- `setGenealogy(genealogy: dict)`: Set genealogy **Link Methods:** diff --git a/demo/apache/transforms/OverlayExample.py b/demo/apache/transforms/OverlayExample.py new file mode 100644 index 0000000..34577a5 --- /dev/null +++ b/demo/apache/transforms/OverlayExample.py @@ -0,0 +1,30 @@ +from maltego_trx.entities import Phrase +from maltego_trx.overlays import Position, OverlayType + +from maltego_trx.transform import DiscoverableTransform + + +class OverlayExample(DiscoverableTransform): + """ + Returns a phrase with overlays on the graph. + """ + + @classmethod + def create_entities(cls, request, response): + person_name = request.Value + entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) + + # references the icon name `Champion` and this is will show up as an overlay on the graph + entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + + # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + + # add the text of the property `exampleDynamicPropertyName` as an overlay + entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + + # add a color overlay + entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + + # add a flag overlay + entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/demo/gunicorn/transforms/OverlayExample.py b/demo/gunicorn/transforms/OverlayExample.py new file mode 100644 index 0000000..34577a5 --- /dev/null +++ b/demo/gunicorn/transforms/OverlayExample.py @@ -0,0 +1,30 @@ +from maltego_trx.entities import Phrase +from maltego_trx.overlays import Position, OverlayType + +from maltego_trx.transform import DiscoverableTransform + + +class OverlayExample(DiscoverableTransform): + """ + Returns a phrase with overlays on the graph. + """ + + @classmethod + def create_entities(cls, request, response): + person_name = request.Value + entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) + + # references the icon name `Champion` and this is will show up as an overlay on the graph + entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + + # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + + # add the text of the property `exampleDynamicPropertyName` as an overlay + entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + + # add a color overlay + entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + + # add a flag overlay + entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/maltego_trx/maltego.py b/maltego_trx/maltego.py index 4383b83..522e386 100644 --- a/maltego_trx/maltego.py +++ b/maltego_trx/maltego.py @@ -1,7 +1,9 @@ import uuid; + from xml.dom import minidom -from .entities import Phrase +from .entities import Phrase, translate_legacy_properties +from .overlays import Position, OverlayType from .utils import remove_invalid_xml_chars BOOKMARK_COLOR_NONE = "-1" @@ -44,7 +46,7 @@ ADD_FIELD_TEMPLATE = "" DISP_INFO_TEMPLATE = "" UIM_TEMPLATE = "%(text)s" - +OVERLAY_TEMPLATE = "" class MaltegoEntity(object): def __init__(self, type=None, value=None): @@ -55,6 +57,7 @@ def __init__(self, type=None, value=None): self.additionalFields = [] self.displayInformation = [] self.iconURL = "" + self.overlays = [] def setType(self, type=None): if type: @@ -93,7 +96,7 @@ def setLinkLabel(self, label): def reverseLink(self): self.addProperty('link#maltego.link.direction', 'link#maltego.link.direction', 'loose', 'output-to-input') - + def addCustomLinkProperty(self, fieldName=None, displayName=None, value=None): self.addProperty('link#' + fieldName, displayName, '', value) @@ -103,6 +106,9 @@ def setBookmark(self, bookmark): def setNote(self, note): self.addProperty('notes#', 'Notes', '', note) + def addOverlay(self, property_name=None, position=Position, overlay_type=OverlayType): + self.overlays.append([property_name, position.value, overlay_type.value]) + def add_field_to_xml(self, additional_field): name, display, matching, value = additional_field matching = "strict" if matching.lower().strip() == "strict" else "loose" @@ -139,6 +145,16 @@ def returnEntity(self): lines.append(self.add_field_to_xml(additional_field)) lines.append("") + if self.overlays: + lines.append("") + for overlay in self.overlays: + overlay_tag = OVERLAY_TEMPLATE % { + "property_name": overlay[0], + "position": overlay[1], + "type": overlay[2], + } + lines.append(overlay_tag) + lines.append("") if self.iconURL: lines.append("%s" % self.iconURL) @@ -226,6 +242,17 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): self.Weight = self._get_int(entity, "Weight") self.Slider = self._get_int(maltego_msg, "Limits", attr_name="SoftLimit") + self._entity_types = [] + self.Genealogy = [] + genealogy_tag = maltego_msg.getElementsByTagName("Genealogy") + genealogy_types = genealogy_tag[0].getElementsByTagName("Type") if genealogy_tag else [] + for genealogy_type_tag in genealogy_types: + entity_type_name = genealogy_type_tag.getAttribute("Name") + entity_type_old_name = genealogy_type_tag.getAttribute("OldName") + entity_type = {"Name": entity_type_name, + "OldName": entity_type_old_name if entity_type_old_name else None} + self._entity_types.append(entity_type_name) + self.Genealogy.append(entity_type) # Additional Fields self.Properties = {} @@ -235,6 +262,10 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): name = field.getAttribute("Name") value = self._get_text(field) self.Properties[name] = value + for entity_type in self._entity_types: + v3_property = translate_legacy_properties(entity_type, name) + if v3_property: + self.Properties[v3_property] = value # Transform Settings self.TransformSettings = {} diff --git a/maltego_trx/overlays.py b/maltego_trx/overlays.py new file mode 100644 index 0000000..9cc2747 --- /dev/null +++ b/maltego_trx/overlays.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class Position(Enum): + NORTH = "N" + NORTH_EAST = "NE" + NORTH_WEST = "NW" + EAST = "E" + CENTER = "C" + WEST = "W" + SOUTH = "S" + SOUTH_EAST = "SE" + SOUTH_WEST = "SW" + + +class OverlayType(Enum): + IMAGE = "image" + COLOUR = "colour" + TEXT = "text" diff --git a/maltego_trx/template_dir/transforms/OverlayExample.py b/maltego_trx/template_dir/transforms/OverlayExample.py new file mode 100644 index 0000000..34577a5 --- /dev/null +++ b/maltego_trx/template_dir/transforms/OverlayExample.py @@ -0,0 +1,30 @@ +from maltego_trx.entities import Phrase +from maltego_trx.overlays import Position, OverlayType + +from maltego_trx.transform import DiscoverableTransform + + +class OverlayExample(DiscoverableTransform): + """ + Returns a phrase with overlays on the graph. + """ + + @classmethod + def create_entities(cls, request, response): + person_name = request.Value + entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) + + # references the icon name `Champion` and this is will show up as an overlay on the graph + entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + + # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + + # add the text of the property `exampleDynamicPropertyName` as an overlay + entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + + # add a color overlay + entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + + # add a flag overlay + entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/maltego_trx/test_hierarchical_entity.xml b/maltego_trx/test_hierarchical_entity.xml new file mode 100644 index 0000000..5b828fe --- /dev/null +++ b/maltego_trx/test_hierarchical_entity.xml @@ -0,0 +1,13 @@ + + + + + + + www.paterva.com + false + 80 + + www.paterva.com + 0 + From 5f6e0755d392c25a69a4e7c99a29a0d2674b5f79 Mon Sep 17 00:00:00 2001 From: Tendai Marengereke Date: Tue, 2 Mar 2021 16:38:25 +0200 Subject: [PATCH 2/4] feat: add version 2 -> version 3 property translation --- maltego_trx/entities.py | 30 +++++++++++++++++++ tests/test_property_mapping.py | 20 +++++++++++++ tests/test_request.xml | 18 +++++++++++ .../transforms/TestRequestPropertyMapping.py | 15 ++++++++++ tests/transforms/__init__.py | 0 5 files changed, 83 insertions(+) create mode 100644 tests/test_property_mapping.py create mode 100644 tests/test_request.xml create mode 100644 tests/transforms/TestRequestPropertyMapping.py create mode 100644 tests/transforms/__init__.py diff --git a/maltego_trx/entities.py b/maltego_trx/entities.py index e129248..4446430 100644 --- a/maltego_trx/entities.py +++ b/maltego_trx/entities.py @@ -35,3 +35,33 @@ URL = "maltego.URL" Website = "maltego.Website" WebTitle = "maltego.WebTitle" + +# {entityName: {version2PropertyName: version3PropertyName,...}} +entity_property_map = { + "maltego.Person": {"firstname": "person.firstnames", "lastname": "person.lastname"}, + "maltego.Domain": {"whois": "whois-info"}, + "maltego.IPv4Address": {"whois": "whois-info"}, + "maltego.URL": {"maltego.v2.value.property": "short-title", "theurl": "url", "fulltitle": "title"}, + "maltego.Document": {"maltego.v2.value.property": "title", "link": "url", "metainfo": "document.meta-data"}, + "maltego.Location": {"area": "location.area", "countrysc": "url", "long": "longitude", "lat": "latitude"}, + "maltego.PhoneNumber": {"countrycode": "phonenumber.countrycode", "citycode": "phonenumber.citycode", + "areacode": "phonenumber.areacode", "lastnumbers": "phonenumber.lastnumbers"}, + "maltego.affiliation.Spock": {"network": "affiliation.network", "uid": "affiliation.uid", + "profile_url": "affiliation.profile-url", "spock_websites": "spock.websites"}, + "maltego.affiliation": {"network": "affiliation.network", "uid": "affiliation.uid", + "profile_url": "affiliation.profile-url"}, + "maltego.Service": {"banner": "banner.text", "port": "port.number"}, + "maltego.Alias": {"properties.alias": "alias"}, + "maltego.Device": {"properties.device": "device"}, + "maltego.GPS": {"properties.gps": "gps.coordinate"}, + "maltego.CircularArea": {"area": "radius"}, + "maltego.Image": {"properties.image": "description", "fullImage": "url"}, + "maltego.NominatimLocation": {"properties.nominatimlocation": "nominatimlocation"}, + "maltego.BuiltWithTechnology": {"properties.builtwithtechnology": "builtwith.technology"}, + "maltego.FacebookObject": {"properties.facebookobject": "facebook.object"} +} + + +def translate_legacy_properties(entity_type, v2_property): + """Function maps a legacy version 2 entity property name to version 3 entity property name""" + return entity_property_map .get(entity_type, {}).get(v2_property) diff --git a/tests/test_property_mapping.py b/tests/test_property_mapping.py new file mode 100644 index 0000000..12af79e --- /dev/null +++ b/tests/test_property_mapping.py @@ -0,0 +1,20 @@ +from maltego_trx.registry import register_transform_classes +from maltego_trx.server import app +from tests import transforms + + +def test_request_property_mapping(): + register_transform_classes(transforms) + app.testing = True + + with app.test_client() as test_app: + response = make_transform_call(test_app, "/run/testrequestpropertymapping/") + assert response.status_code == 200 + data = response.data.decode('utf8') + assert "whois-info found" in data + + +def make_transform_call(test_app=None, run_endpoint=""): + with open('test_request.xml') as requestMsg: + response = test_app.post(run_endpoint, data=requestMsg.read()) + return response diff --git a/tests/test_request.xml b/tests/test_request.xml new file mode 100644 index 0000000..77c5f40 --- /dev/null +++ b/tests/test_request.xml @@ -0,0 +1,18 @@ + + + + + + + + + paterva.com + whois-info found + + paterva.com + 0 + + + + + diff --git a/tests/transforms/TestRequestPropertyMapping.py b/tests/transforms/TestRequestPropertyMapping.py new file mode 100644 index 0000000..c82426a --- /dev/null +++ b/tests/transforms/TestRequestPropertyMapping.py @@ -0,0 +1,15 @@ +from maltego_trx.entities import Phrase + +from maltego_trx.transform import DiscoverableTransform + + +class TestRequestPropertyMapping(DiscoverableTransform): + """ + Test if the automatic mapping of v2 propertyname `whois` -> `whois-info` has been done by the library. Original input + contains only whois property name. see test_request.xml + """ + + @classmethod + def create_entities(cls, request, response): + v3_property_value = request.Properties['whois-info'] + response.addEntity(Phrase, "%s" % v3_property_value) diff --git a/tests/transforms/__init__.py b/tests/transforms/__init__.py new file mode 100644 index 0000000..e69de29 From ae0a9ffdbfe0747d2374109818ff9d4a6cd81e2b Mon Sep 17 00:00:00 2001 From: Philipp Dowling Date: Tue, 2 Mar 2021 22:20:10 +0100 Subject: [PATCH 3/4] minor code style changes, add convenience method for clearing old property names from input entity --- maltego_trx/__init__.py | 2 +- maltego_trx/entities.py | 21 +++++++++++++-------- maltego_trx/maltego.py | 24 +++++++++++++++++------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/maltego_trx/__init__.py b/maltego_trx/__init__.py index bf730e4..3ba22d0 100644 --- a/maltego_trx/__init__.py +++ b/maltego_trx/__init__.py @@ -1 +1 @@ -VERSION = "1.3.7" +VERSION = "1.3.8" diff --git a/maltego_trx/entities.py b/maltego_trx/entities.py index 4446430..0f48e06 100644 --- a/maltego_trx/entities.py +++ b/maltego_trx/entities.py @@ -37,19 +37,24 @@ WebTitle = "maltego.WebTitle" # {entityName: {version2PropertyName: version3PropertyName,...}} -entity_property_map = { +entity_property_map = { "maltego.Person": {"firstname": "person.firstnames", "lastname": "person.lastname"}, "maltego.Domain": {"whois": "whois-info"}, "maltego.IPv4Address": {"whois": "whois-info"}, "maltego.URL": {"maltego.v2.value.property": "short-title", "theurl": "url", "fulltitle": "title"}, "maltego.Document": {"maltego.v2.value.property": "title", "link": "url", "metainfo": "document.meta-data"}, "maltego.Location": {"area": "location.area", "countrysc": "url", "long": "longitude", "lat": "latitude"}, - "maltego.PhoneNumber": {"countrycode": "phonenumber.countrycode", "citycode": "phonenumber.citycode", - "areacode": "phonenumber.areacode", "lastnumbers": "phonenumber.lastnumbers"}, - "maltego.affiliation.Spock": {"network": "affiliation.network", "uid": "affiliation.uid", - "profile_url": "affiliation.profile-url", "spock_websites": "spock.websites"}, - "maltego.affiliation": {"network": "affiliation.network", "uid": "affiliation.uid", - "profile_url": "affiliation.profile-url"}, + "maltego.PhoneNumber": { + "countrycode": "phonenumber.countrycode", "citycode": "phonenumber.citycode", + "areacode": "phonenumber.areacode", "lastnumbers": "phonenumber.lastnumbers" + }, + "maltego.affiliation.Spock": { + "network": "affiliation.network", "uid": "affiliation.uid", "profile_url": "affiliation.profile-url", + "spock_websites": "spock.websites" + }, + "maltego.affiliation": { + "network": "affiliation.network", "uid": "affiliation.uid", "profile_url": "affiliation.profile-url" + }, "maltego.Service": {"banner": "banner.text", "port": "port.number"}, "maltego.Alias": {"properties.alias": "alias"}, "maltego.Device": {"properties.device": "device"}, @@ -62,6 +67,6 @@ } -def translate_legacy_properties(entity_type, v2_property): +def translate_legacy_property_name(entity_type, v2_property): """Function maps a legacy version 2 entity property name to version 3 entity property name""" return entity_property_map .get(entity_type, {}).get(v2_property) diff --git a/maltego_trx/maltego.py b/maltego_trx/maltego.py index 522e386..a2d5666 100644 --- a/maltego_trx/maltego.py +++ b/maltego_trx/maltego.py @@ -2,7 +2,7 @@ from xml.dom import minidom -from .entities import Phrase, translate_legacy_properties +from .entities import Phrase, translate_legacy_property_name, entity_property_map from .overlays import Position, OverlayType from .utils import remove_invalid_xml_chars @@ -48,6 +48,7 @@ UIM_TEMPLATE = "%(text)s" OVERLAY_TEMPLATE = "" + class MaltegoEntity(object): def __init__(self, type=None, value=None): self.entityType = type if type else Phrase @@ -242,7 +243,6 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): self.Weight = self._get_int(entity, "Weight") self.Slider = self._get_int(maltego_msg, "Limits", attr_name="SoftLimit") - self._entity_types = [] self.Genealogy = [] genealogy_tag = maltego_msg.getElementsByTagName("Genealogy") genealogy_types = genealogy_tag[0].getElementsByTagName("Type") if genealogy_tag else [] @@ -251,7 +251,6 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): entity_type_old_name = genealogy_type_tag.getAttribute("OldName") entity_type = {"Name": entity_type_name, "OldName": entity_type_old_name if entity_type_old_name else None} - self._entity_types.append(entity_type_name) self.Genealogy.append(entity_type) # Additional Fields @@ -262,10 +261,10 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): name = field.getAttribute("Name") value = self._get_text(field) self.Properties[name] = value - for entity_type in self._entity_types: - v3_property = translate_legacy_properties(entity_type, name) - if v3_property: - self.Properties[v3_property] = value + for entity_type in self.Genealogy: + v3_property_name = translate_legacy_property_name(entity_type["Name"], name) + if v3_property_name is not None: + self.Properties[v3_property_name] = value # Transform Settings self.TransformSettings = {} @@ -278,6 +277,7 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): elif LocalArgs: self.Value = LocalArgs[0] self.Type = "local.Unknown" + self.Genealogy = None self.Weight = 100 self.Slider = 100 @@ -294,6 +294,16 @@ def __init__(self, MaltegoXML="", LocalArgs=[]): self.buildProperties(text.split("#"), hash_rnd, equals_rnd, bslash_rnd) self.TransformSettings = {} + def clearLegacyProperties(self): + to_clear = set() + for entity_type in self.Genealogy or []: + for prop_name in entity_property_map.get(entity_type["Name"], []): + to_clear.add(prop_name) + + for field_name in to_clear: + if field_name in self.Properties: + del self.Properties[field_name] + def buildProperties(self, key_value_array, hash_rnd, equals_rnd, bslash_rnd): self.Properties = {} for property_section in key_value_array: From 3afb3b6ac6e65a6aa39893b02ab49830b10e7d75 Mon Sep 17 00:00:00 2001 From: Philipp Dowling Date: Fri, 5 Mar 2021 15:51:08 +0100 Subject: [PATCH 4/4] adjusted code style, README, requirements, setup.py and demos ahead of release --- README.md | 80 ++++++++++++------- demo/apache/transforms/OverlayExample.py | 28 ++++--- demo/gunicorn/Dockerfile | 3 +- demo/gunicorn/prod-ssl.yml | 7 ++ demo/gunicorn/prod.yml | 2 +- demo/gunicorn/requirements.txt | 2 +- demo/gunicorn/transforms/OverlayExample.py | 28 ++++--- maltego_trx/maltego.py | 8 +- maltego_trx/overlays.py | 11 +-- .../template_dir/transforms/OverlayExample.py | 27 ++++--- setup.py | 47 ++++++----- 11 files changed, 148 insertions(+), 95 deletions(-) create mode 100644 demo/gunicorn/prod-ssl.yml diff --git a/README.md b/README.md index 54cae8a..91d7c1d 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ maltego-trx start new_project This will create a folder new_project with the recommend project structure. +Alternatively, you can copy either the `gunicorn` or `apache` example projects from the `demo` directory. +These also include Dockerfile and corresponding docker-compose configuration files for production deployment. + **Adding a Transform:** Add a new transform by creating a new python file in the "transforms" folder of your directory. @@ -207,16 +210,13 @@ The following constants can be imported from `maltego_trx.maltego`. Overlays Enums are imported from `maltego_trx.overlays` -*Overlay Position:* +*Overlay OverlayPosition:* - `NORTH = "N"` -- `NORTH_EAST = "NE"` -- `NORTH_WEST = "NW"` -- `EAST = "E"` -- `CENTER = "C"` -- `WEST = "W"` - `SOUTH = "S"` -- `SOUTH_EAST = "SE"` +- `WEST = "W"` +- `NORTH_WEST = "NW"` - `SOUTH_WEST = "SW"` +- `CENTER = "C"` *Overlay Type* - `IMAGE = "image"` @@ -240,15 +240,23 @@ The request/maltego msg object given to the transform contains the information a **Methods:** -- `getProperty(name: str)`: get a property value of the input entity -- `getTransformSetting(name: str)`: get a transform setting value +- `getProperty(name: str)`: Get a property value of the input entity +- `getTransformSetting(name: str)`: Get a transform setting value +- `clearLegacyProperties()`: Delete (duplicate) legacy properties from the input entity. This will not result in +property information being lost, it will simply clear out some properties that the TRX library duplicates on all +incoming Transform requests. In older versions of TRX, these Entity properties would have a different internal ID when +sent the server than what the Maltego client would advertise in the Entity Manager UI. For a list of Entities with such +properties and their corresponding legacy and actual IDs, see `entity_property_map` in `maltego_trx/entities.py`. For +the majority of projects this distinction can be safely ignored. ### Response/MaltegoTransform **Methods:** -- `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object created by the method. -- `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message type constant. +- `addEntity(type: str, value: str) -> Entity`: Add an entity to the transform response. Returns an Entity object +created by the method. +- `addUIMessage(message: str, messageType='Inform')`: Return a UI message to the user. For message type, use a message +type constant. ### Entity @@ -258,27 +266,45 @@ The request/maltego msg object given to the transform contains the information a - `setValue(value: str)`: Set the entity value - `setWeight(weight: int)`: Set the entity weight - `addDisplayInformation(content: str, title: str)`: Add display information for the entity. -- `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. Matching rule can be `strict` or `loose`. -- `addOverlay(property_name: str, position:Position, overlay_type:OverlayType)`: Add an overlay to the entity. `Position` and `Type` are defined in the `maltego_tx.overlays` +- `addProperty(fieldName: str, displayName: str, matchingRule: str, value: str)`: Add a property to the entity. +Matching rule can be `strict` or `loose`. +- `addOverlay(propertyName: str, position: OverlayPosition, overlay_type: OverlayType)`: Add an overlay to the entity. +`OverlayPosition` and `OverlayType` are defined in the `maltego_tx.overlays` Overlay can be added as Text, Image or Color ```python - # references the icon name `Champion` from the Maltego Desktop Client and this is will show up as an overlay on the graph - entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) - - # add a dynamic property - entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") - - # add the text value of the property `exampleDynamicPropertyName` as an overlay, any existing property name will work - entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) - - # add a color overlay - entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) - - # add a flag overlay - DE is an icon on the Maltego Desktop Client - entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) + person_name = request.Value + entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) + + # Normally, when we create an overlay, we would reference a property name so that Maltego can then use the + # value of that property to create the overlay. Sometimes that means creating a dynamic property, but usually + # it's better to either use an existing property, or, if you created the Entity yourself, and only need the + # property for the overlay, to use a hidden property. Here's an example of using a dynamic property: + entity.addProperty( + 'dynamic_overlay_icon_name', + displayName="Name for overlay image", + value="Champion" # references an icon in the Maltego client + ) + entity.addOverlay('dynamic_overlay_icon_name', OverlayPosition.WEST, OverlayType.IMAGE) + + # DISCOURAGED: + # You *can* also directly supply the string value of the property, however this is not recommended. Why? If + # the entity already has a property of the same ID (in this case, "DE"), then you would in fact be assigning the + # value of that property, not the string "DE", which is not the intention. Nevertheless, here's an example: + entity.addOverlay( + 'DE', # name of an icon, however, could also accidentally be a property name + OverlayPosition.SOUTH_WEST, + OverlayType.IMAGE + ) + + # Overlays can also be used to display extra text on an entity: + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Overlay Testing") + entity.addOverlay('exampleDynamicPropertyName', OverlayPosition.NORTH, OverlayType.TEXT) + + # Or a small color indicator: + entity.addOverlay('#45e06f', OverlayPosition.NORTH_WEST, OverlayType.COLOUR) ``` - `setIconURL(url: str)`: Set the entity icon URL diff --git a/demo/apache/transforms/OverlayExample.py b/demo/apache/transforms/OverlayExample.py index 34577a5..7bd8ecc 100644 --- a/demo/apache/transforms/OverlayExample.py +++ b/demo/apache/transforms/OverlayExample.py @@ -1,5 +1,5 @@ from maltego_trx.entities import Phrase -from maltego_trx.overlays import Position, OverlayType +from maltego_trx.overlays import OverlayPosition, OverlayType from maltego_trx.transform import DiscoverableTransform @@ -14,17 +14,23 @@ def create_entities(cls, request, response): person_name = request.Value entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) - # references the icon name `Champion` and this is will show up as an overlay on the graph - entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + # Normally, when we create an overlay, we would reference a property name so that Maltego can then use the + # value of that property to create the overlay. Sometimes that means creating a dynamic property, but usually + # it's better to either use an existing property, or, if you created the Entity yourself, and only need the + # property for the overlay, to use a hidden property. Here's an example of using a dynamic property: + entity.addProperty('dynamic_overlay_icon_name', displayName="Name for overlay image", value="Champion") + entity.addOverlay('dynamic_overlay_icon_name', OverlayPosition.WEST, OverlayType.IMAGE) - # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): - entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + # DISCOURAGED: + # You *can* also directly supply the string value of the property, however this is not recommended. Why? If + # the entity already has a property of the same ID (in this case, "DE"), then you would in fact be assigning the + # value of that property, not the string "DE", which is not the intention. Nevertheless, here's an example: + entity.addOverlay('DE', OverlayPosition.SOUTH_WEST, OverlayType.IMAGE) - # add the text of the property `exampleDynamicPropertyName` as an overlay - entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + # Overlays can also be used to display extra text on an entity: + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Overlay Testing") + entity.addOverlay('exampleDynamicPropertyName', OverlayPosition.NORTH, OverlayType.TEXT) - # add a color overlay - entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + # Or a small color indicator: + entity.addOverlay('#45e06f', OverlayPosition.NORTH_WEST, OverlayType.COLOUR) - # add a flag overlay - entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/demo/gunicorn/Dockerfile b/demo/gunicorn/Dockerfile index 66f56f7..42edd11 100644 --- a/demo/gunicorn/Dockerfile +++ b/demo/gunicorn/Dockerfile @@ -5,7 +5,7 @@ RUN mkdir /var/www/TRX/ WORKDIR /var/www/TRX/ # System dependencies -RUN apt-get update +RUN apt-get update -y RUN apt-get install python3-pip -y COPY requirements.txt requirements.txt @@ -18,4 +18,5 @@ COPY . /var/www/TRX/ RUN chown -R www-data:www-data /var/www/TRX/ +# for running a production server, use docker-compose with prod.yml or prod-ssl.yml CMD ["python3", "project.py", "runserver"] diff --git a/demo/gunicorn/prod-ssl.yml b/demo/gunicorn/prod-ssl.yml new file mode 100644 index 0000000..554c9a7 --- /dev/null +++ b/demo/gunicorn/prod-ssl.yml @@ -0,0 +1,7 @@ +version: '3' +services: + python: + build: . + command: "gunicorn --certfile=server.crt --keyfile=server.key --bind=0.0.0.0:8443 --threads=25 --workers=2 project:app" + ports: + - "8443:8443" diff --git a/demo/gunicorn/prod.yml b/demo/gunicorn/prod.yml index c243cc3..0a4c6dc 100644 --- a/demo/gunicorn/prod.yml +++ b/demo/gunicorn/prod.yml @@ -1,7 +1,7 @@ version: '3' services: python: - build: .. + build: . command: "gunicorn --bind=0.0.0.0:8080 --threads=25 --workers=2 project:app" ports: - "8080:8080" diff --git a/demo/gunicorn/requirements.txt b/demo/gunicorn/requirements.txt index c58373f..e9f1c7c 100644 --- a/demo/gunicorn/requirements.txt +++ b/demo/gunicorn/requirements.txt @@ -1 +1 @@ -maltego-trx \ No newline at end of file +maltego-trx>=1.3.8 diff --git a/demo/gunicorn/transforms/OverlayExample.py b/demo/gunicorn/transforms/OverlayExample.py index 34577a5..7bd8ecc 100644 --- a/demo/gunicorn/transforms/OverlayExample.py +++ b/demo/gunicorn/transforms/OverlayExample.py @@ -1,5 +1,5 @@ from maltego_trx.entities import Phrase -from maltego_trx.overlays import Position, OverlayType +from maltego_trx.overlays import OverlayPosition, OverlayType from maltego_trx.transform import DiscoverableTransform @@ -14,17 +14,23 @@ def create_entities(cls, request, response): person_name = request.Value entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) - # references the icon name `Champion` and this is will show up as an overlay on the graph - entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + # Normally, when we create an overlay, we would reference a property name so that Maltego can then use the + # value of that property to create the overlay. Sometimes that means creating a dynamic property, but usually + # it's better to either use an existing property, or, if you created the Entity yourself, and only need the + # property for the overlay, to use a hidden property. Here's an example of using a dynamic property: + entity.addProperty('dynamic_overlay_icon_name', displayName="Name for overlay image", value="Champion") + entity.addOverlay('dynamic_overlay_icon_name', OverlayPosition.WEST, OverlayType.IMAGE) - # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): - entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + # DISCOURAGED: + # You *can* also directly supply the string value of the property, however this is not recommended. Why? If + # the entity already has a property of the same ID (in this case, "DE"), then you would in fact be assigning the + # value of that property, not the string "DE", which is not the intention. Nevertheless, here's an example: + entity.addOverlay('DE', OverlayPosition.SOUTH_WEST, OverlayType.IMAGE) - # add the text of the property `exampleDynamicPropertyName` as an overlay - entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + # Overlays can also be used to display extra text on an entity: + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Overlay Testing") + entity.addOverlay('exampleDynamicPropertyName', OverlayPosition.NORTH, OverlayType.TEXT) - # add a color overlay - entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + # Or a small color indicator: + entity.addOverlay('#45e06f', OverlayPosition.NORTH_WEST, OverlayType.COLOUR) - # add a flag overlay - entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/maltego_trx/maltego.py b/maltego_trx/maltego.py index a2d5666..c5dc7b0 100644 --- a/maltego_trx/maltego.py +++ b/maltego_trx/maltego.py @@ -3,7 +3,7 @@ from xml.dom import minidom from .entities import Phrase, translate_legacy_property_name, entity_property_map -from .overlays import Position, OverlayType +from .overlays import OverlayPosition, OverlayType from .utils import remove_invalid_xml_chars BOOKMARK_COLOR_NONE = "-1" @@ -107,8 +107,10 @@ def setBookmark(self, bookmark): def setNote(self, note): self.addProperty('notes#', 'Notes', '', note) - def addOverlay(self, property_name=None, position=Position, overlay_type=OverlayType): - self.overlays.append([property_name, position.value, overlay_type.value]) + def addOverlay( + self, propertyName, position: OverlayPosition, overlayType: OverlayType + ): + self.overlays.append([propertyName, position.value, overlayType.value]) def add_field_to_xml(self, additional_field): name, display, matching, value = additional_field diff --git a/maltego_trx/overlays.py b/maltego_trx/overlays.py index 9cc2747..1c897b0 100644 --- a/maltego_trx/overlays.py +++ b/maltego_trx/overlays.py @@ -1,16 +1,13 @@ from enum import Enum -class Position(Enum): +class OverlayPosition(Enum): NORTH = "N" - NORTH_EAST = "NE" - NORTH_WEST = "NW" - EAST = "E" - CENTER = "C" - WEST = "W" SOUTH = "S" - SOUTH_EAST = "SE" + WEST = "W" + NORTH_WEST = "NW" SOUTH_WEST = "SW" + CENTER = "C" class OverlayType(Enum): diff --git a/maltego_trx/template_dir/transforms/OverlayExample.py b/maltego_trx/template_dir/transforms/OverlayExample.py index 34577a5..e4478d2 100644 --- a/maltego_trx/template_dir/transforms/OverlayExample.py +++ b/maltego_trx/template_dir/transforms/OverlayExample.py @@ -1,5 +1,5 @@ from maltego_trx.entities import Phrase -from maltego_trx.overlays import Position, OverlayType +from maltego_trx.overlays import OverlayPosition, OverlayType from maltego_trx.transform import DiscoverableTransform @@ -14,17 +14,22 @@ def create_entities(cls, request, response): person_name = request.Value entity = response.addEntity(Phrase, "Hi %s, nice to meet you!" % person_name) - # references the icon name `Champion` and this is will show up as an overlay on the graph - entity.addOverlay('Champion', Position.EAST, OverlayType.IMAGE) + # Normally, when we create an overlay, we would reference a property name so that Maltego can then use the + # value of that property to create the overlay. Sometimes that means creating a dynamic property, but usually + # it's better to either use an existing property, or, if you created the Entity yourself, and only need the + # property for the overlay, to use a hidden property. Here's an example of using a dynamic property: + entity.addProperty('dynamic_overlay_icon_name', displayName="Name for overlay image", value="Champion") + entity.addOverlay('dynamic_overlay_icon_name', OverlayPosition.WEST, OverlayType.IMAGE) - # addProperty(self, fieldName=None, displayName=None, matchingRule='loose', value=None): - entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Champion") + # You *can* also directly supply the string value of the property, however this is not recommended. Why? If + # the entity already has a property of the same ID (in this case, "DE"), then you would in fact be assigning the + # value of that property, not the string "DE", which is not the intention. Nevertheless, here's an example: + entity.addOverlay('DE', OverlayPosition.SOUTH_WEST, OverlayType.IMAGE) - # add the text of the property `exampleDynamicPropertyName` as an overlay - entity.addOverlay('exampleDynamicPropertyName', Position.NORTH, OverlayType.TEXT) + # Overlays can also be an additional field of text displayed on the entity: + entity.addProperty("exampleDynamicPropertyName", "Example Dynamic Property", "loose", "Maltego Overlay Testing") + entity.addOverlay('exampleDynamicPropertyName', OverlayPosition.NORTH, OverlayType.TEXT) - # add a color overlay - entity.addOverlay('#45e06f', Position.NORTH_WEST, OverlayType.COLOUR) + # Or a small color indicator + entity.addOverlay('#45e06f', OverlayPosition.NORTH_WEST, OverlayType.COLOUR) - # add a flag overlay - entity.addOverlay('DE', Position.SOUTH_WEST, OverlayType.IMAGE) diff --git a/setup.py b/setup.py index edc23fc..be1ee8b 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,28 @@ from setuptools import setup from maltego_trx import VERSION -setup(name='maltego-trx', - version=VERSION, - description='Python library used to develop Maltego transforms', - url='https://github.com/paterva/maltego-trx/', - author='Maltego Staff', - author_email='support@maltego.com', - license='MIT', - install_requires=[ - 'flask>=1', - 'six>=1', - 'cryptography>=3.3.1' - ], - packages=[ - 'maltego_trx', - 'maltego_trx/template_dir', - 'maltego_trx/template_dir/transforms' - ], - entry_points={'console_scripts': [ - 'maltego-trx = maltego_trx.commands:execute_from_command_line', - ]}, - zip_safe=False - ) +setup( + name='maltego-trx', + version=VERSION, + description='Python library used to develop Maltego transforms', + url='https://github.com/paterva/maltego-trx/', + author='Maltego Staff', + author_email='support@maltego.com', + license='MIT', + install_requires=[ + 'flask>=1', + 'six>=1', + 'cryptography==3.3.2' # pinned for now as newer versions require setuptools_rust + ], + packages=[ + 'maltego_trx', + 'maltego_trx/template_dir', + 'maltego_trx/template_dir/transforms' + ], + entry_points={ + 'console_scripts': [ + 'maltego-trx = maltego_trx.commands:execute_from_command_line', + ] + }, + zip_safe=False +)