From 41abc8a54858385c81fe2c43a91e6187c34a1697 Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Fri, 24 May 2024 11:25:37 +0200 Subject: [PATCH 1/3] Compatibility with cstruct v4 --- dissect/eventlog/bxml.py | 14 ++++---- dissect/eventlog/evt.py | 9 +++-- dissect/eventlog/evtx.py | 17 +++++----- dissect/eventlog/wevt.py | 7 ++-- dissect/eventlog/wevt_object.py | 59 ++++++++++++++++----------------- pyproject.toml | 4 +-- tests/_utils.py | 4 +-- tests/test_wevt.py | 5 +-- tox.ini | 7 ++++ 9 files changed, 64 insertions(+), 62 deletions(-) diff --git a/dissect/eventlog/bxml.py b/dissect/eventlog/bxml.py index 17d070e..e79aad0 100644 --- a/dissect/eventlog/bxml.py +++ b/dissect/eventlog/bxml.py @@ -1,5 +1,4 @@ -""" Binary XML classes """ - +"""Binary XML classes""" import binascii import uuid @@ -8,7 +7,7 @@ from io import BytesIO from typing import Any, BinaryIO, Dict, List, Tuple -from dissect.cstruct.cstruct import cstruct +from dissect.cstruct import cstruct from dissect.util.ts import wintimestamp from dissect.eventlog.exceptions import BxmlException @@ -146,8 +145,7 @@ def __str__(self) -> str: WORD wMilliseconds; }; """ -bxml_struct = cstruct() -bxml_struct.load(bxml_def) +bxml_struct = cstruct().load(bxml_def) def read_systemtime(stream): @@ -198,9 +196,9 @@ def read_sid(stream) -> str: BxmlType.BINARY: lambda stream: binascii.hexlify(stream.read()), BxmlType.GUID: read_guid, BxmlType.SIZET: ( - lambda stream: f"0x{bxml_struct.uint32(stream):x}" - if len(stream.getvalue()) == 4 - else f"0x{bxml_struct.uint64(stream):x}" + lambda stream: ( + f"0x{bxml_struct.uint32(stream):x}" if len(stream.getvalue()) == 4 else f"0x{bxml_struct.uint64(stream):x}" + ) ), BxmlType.FILETIME: lambda stream: wintimestamp(bxml_struct.uint64(stream)), BxmlType.SYSTEMTIME: lambda stream: read_systemtime(stream), diff --git a/dissect/eventlog/evt.py b/dissect/eventlog/evt.py index 8ffb64b..7b04bba 100644 --- a/dissect/eventlog/evt.py +++ b/dissect/eventlog/evt.py @@ -5,13 +5,11 @@ from collections import namedtuple from datetime import datetime, timezone -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.eventlog.exceptions import Error -c_evt = cstruct.cstruct() -c_evt.load( - """ +evt_def = """ #define ELF_LOGFILE_HEADER_DIRTY 0x0001 #define ELF_LOGFILE_HEADER_WRAP 0x0002 #define ELF_LOGFILE_LOGFULL_WRITTEN 0x0004 @@ -64,7 +62,8 @@ ULONG RecordSizeEnd; } EVENTLOGEOF; """ -) + +c_evt = cstruct().load(evt_def) EVENTLOGRECORD_SIZE = len(c_evt.EVENTLOGRECORD) diff --git a/dissect/eventlog/evtx.py b/dissect/eventlog/evtx.py index 41125d3..daaa098 100644 --- a/dissect/eventlog/evtx.py +++ b/dissect/eventlog/evtx.py @@ -5,7 +5,7 @@ import logging import os -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.eventlog.bxml import Bxml, BxmlSub, EvtxNameReader, parse_bxml from dissect.eventlog.exceptions import MalformedElfChnkException @@ -13,9 +13,7 @@ log = logging.getLogger(__name__) log.setLevel(os.getenv("DISSECT_LOG_EVTX", "CRITICAL")) -evtx = cstruct.cstruct() -evtx.load( - """ +evtx_def = """ struct EVTX_HEADER { char magic[8]; uint64 first_chunk; @@ -57,14 +55,15 @@ uint32 size_copy; }; """ -) + +c_evtx = cstruct().load(evtx_def) class ElfChnk: def __init__(self, d, path=None): self.path = path self.stream = io.BytesIO(d) - self.header = evtx.EVTX_CHUNK(self.stream) + self.header = c_evtx.EVTX_CHUNK(self.stream) if self.header.magic != b"ElfChnk\x00": if self.header.magic != b"\x00\x00\x00\x00\x00\x00\x00\x00": @@ -83,7 +82,7 @@ def read(self, records=True): while True: offset = self.stream.tell() try: - r = evtx.EVTX_RECORD(self.stream) + r = c_evtx.EVTX_RECORD(self.stream) except EOFError: break @@ -129,13 +128,13 @@ class Evtx: def __init__(self, fh, path=None): self.path = path self.fh = fh - self.header = evtx.EVTX_HEADER(self.fh) + self.header = c_evtx.EVTX_HEADER(self.fh) self.count = 0 def __iter__(self): chunk_offset = self.header.header_block_size - skip = self.header.header_block_size - len(evtx.EVTX_HEADER) + skip = self.header.header_block_size - len(c_evtx.EVTX_HEADER) if skip > 0: self.fh.read(skip) diff --git a/dissect/eventlog/wevt.py b/dissect/eventlog/wevt.py index 9781b27..81dc86e 100644 --- a/dissect/eventlog/wevt.py +++ b/dissect/eventlog/wevt.py @@ -6,7 +6,7 @@ import dissect.eventlog.wevt_object as wevt_objects from dissect.eventlog.exceptions import UnknownSignatureException -header_dev = """ +header_def = """ struct Event_Descriptor { char ProviderId[16]; uint32 offset; @@ -40,8 +40,7 @@ }; """ -c_wevt_headers = cstruct() -c_wevt_headers.load(header_dev) +c_wevt_headers = cstruct().load(header_def) def validate_signature(signature, expected_signature): @@ -120,7 +119,7 @@ def size(self): return self.header.size def __repr__(self): - return f"" + return f"" class WEVT_TYPE: diff --git a/dissect/eventlog/wevt_object.py b/dissect/eventlog/wevt_object.py index ed102ca..7482b52 100644 --- a/dissect/eventlog/wevt_object.py +++ b/dissect/eventlog/wevt_object.py @@ -20,47 +20,47 @@ }; struct TEMP { - char signature[4]; - uint32 size; - uint32 nr_of_items; - uint32 nr_of_names; - uint32 data_offset; - uint32 binxml_fragments; - char identifier[16]; + char signature[4]; + uint32 size; + uint32 nr_of_items; + uint32 nr_of_names; + uint32 data_offset; + uint32 binxml_fragments; + char identifier[16]; }; struct TEMP_DESCRIPTOR { - uint32 unknown0; - uint8 input_type; - uint8 output_type; - uint16 unknown1; - uint32 unknown2; - uint32 unknown3; - uint32 data_offset; + uint32 unknown0; + uint8 input_type; + uint8 output_type; + uint16 unknown1; + uint32 unknown2; + uint32 unknown3; + uint32 data_offset; } struct PRVA { - uint32 unknown; - uint32 data_offset; + uint32 unknown; + uint32 data_offset; }; struct TASK { - uint32 id; - uint32 message_table_id; - char mui_id[16]; - uint32 data_offset; + uint32 id; + uint32 message_table_id; + char mui_id[16]; + uint32 data_offset; }; struct KEYW { - uint64 bitmask; - uint32 message_table_id; - uint32 data_offset; + uint64 bitmask; + uint32 message_table_id; + uint32 data_offset; }; struct LEVL { - uint32 id; - uint32 message_table_id; - uint32 data_offset; + uint32 id; + uint32 message_table_id; + uint32 data_offset; }; struct EVNT { @@ -101,8 +101,7 @@ }; """ -wevt_objects = cstruct() -wevt_objects.load(wevt_object_def) +c_wevt_objects = cstruct().load(wevt_object_def) class WevtObject: @@ -110,7 +109,7 @@ class WevtObject: def __init__(self, offset, data): self.offset = offset - self.header = getattr(wevt_objects, self.__class__.__name__)(data) + self.header = getattr(c_wevt_objects, self.__class__.__name__)(data) self.data = data[len(self.header) :] self.data_start = self.offset + len(self.header) self.data_offset = self.header.data_offset - self.data_start @@ -118,7 +117,7 @@ def __init__(self, offset, data): def extract_name(self, data_offset): """data_offset is a relative offset that usually points to the data_item. This point is used to read the name for this specific""" - return wevt_objects.DATA_ITEM(self.data[data_offset:]).name.rstrip("\x00") + return c_wevt_objects.DATA_ITEM(self.data[data_offset:]).name.rstrip("\x00") def __getattribute__(self, name: str): try: diff --git a/pyproject.toml b/pyproject.toml index 02612cb..76a634a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,8 @@ classifiers = [ "Topic :: Utilities", ] dependencies = [ - "dissect.cstruct>=3.0.dev,<4.0.dev", - "dissect.util>=3.0.dev,<4.0.dev", + "dissect.cstruct>3,<5", + "dissect.util>2,<4", ] dynamic = ["version"] diff --git a/tests/_utils.py b/tests/_utils.py index 9d2ec78..ad3dc2f 100644 --- a/tests/_utils.py +++ b/tests/_utils.py @@ -1,10 +1,10 @@ from dissect.cstruct import cstruct -from dissect.eventlog.wevt import header_dev +from dissect.eventlog.wevt import header_def from dissect.eventlog.wevt_object import wevt_object_def definitions = cstruct() -definitions.load(header_dev + wevt_object_def) +definitions.load(header_def + wevt_object_def) def create_header(type, **kwargs): diff --git a/tests/test_wevt.py b/tests/test_wevt.py index 798b4fe..21d1b4b 100644 --- a/tests/test_wevt.py +++ b/tests/test_wevt.py @@ -70,8 +70,9 @@ def test_wevt_test_offset(value, expected_result, mocked_fh): def test_wevt_test_iterator(_, mocked_fh): with patch.object(WEVT, WEVT._next_type_offset.__name__): wevt = create_wevt(mocked_fh) - for index, _ in enumerate(wevt): - wevt._next_type_offset.assert_called_with(wevt.payload_types[index].offset) + with patch("dissect.cstruct.types.char.Char._read_array"): + for index, _ in enumerate(wevt): + wevt._next_type_offset.assert_called_with(wevt.payload_types[index].offset) @pytest.mark.parametrize( diff --git a/tox.ini b/tox.ini index 67e8e8a..badc32c 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,13 @@ deps = pytest pytest-cov coverage +# Unfortunately, tox does not allow separate installation flags for the project +# dependencies and the test dependencies. When running tox, we want to install the +# project dependencies with the --pre flag, so that we get the latest version of all +# dependencies. We do the installation step ourselves for this reason. +skip_install = true +commands_pre = + pip install --pre -e . commands = pytest --basetemp="{envtmpdir}" {posargs:--color=yes --cov=dissect --cov-report=term-missing -v tests} coverage report From 9c1543cef900afcc76271abe84ce7b6201dd834f Mon Sep 17 00:00:00 2001 From: pyrco <105293448+pyrco@users.noreply.github.com> Date: Thu, 30 May 2024 13:35:22 +0200 Subject: [PATCH 2/3] Use a dev extra instead of installing with --pre --- pyproject.toml | 10 ++++++++-- tox.ini | 8 +------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76a634a..28fc969 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,8 @@ classifiers = [ "Topic :: Utilities", ] dependencies = [ - "dissect.cstruct>3,<5", - "dissect.util>2,<4", + "dissect.cstruct>=4.dev,<5", + "dissect.util>=3,<4", ] dynamic = ["version"] @@ -35,6 +35,12 @@ homepage = "https://dissect.tools" documentation = "https://docs.dissect.tools/en/latest/projects/dissect.eventlog" repository = "https://github.com/fox-it/dissect.eventlog" +[project.optional-dependencies] +dev = [ + "dissect.cstruct>=4.0.dev,<5.0.dev", + "dissect.util>=3.0.dev,<4.0.dev", +] + [tool.black] line-length = 120 diff --git a/tox.ini b/tox.ini index badc32c..7bd2890 100644 --- a/tox.ini +++ b/tox.ini @@ -11,17 +11,11 @@ minversion = 4.4.3 requires = virtualenv>=20.16.6 [testenv] +extras = dev deps = pytest pytest-cov coverage -# Unfortunately, tox does not allow separate installation flags for the project -# dependencies and the test dependencies. When running tox, we want to install the -# project dependencies with the --pre flag, so that we get the latest version of all -# dependencies. We do the installation step ourselves for this reason. -skip_install = true -commands_pre = - pip install --pre -e . commands = pytest --basetemp="{envtmpdir}" {posargs:--color=yes --cov=dissect --cov-report=term-missing -v tests} coverage report From f4b9a8313e2361243236570b43e571bdbb106246 Mon Sep 17 00:00:00 2001 From: pyrco <105293448+pyrco@users.noreply.github.com> Date: Fri, 31 May 2024 11:34:39 +0200 Subject: [PATCH 3/3] Use cleaner cstruct load syntax --- tests/_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/_utils.py b/tests/_utils.py index ad3dc2f..2798411 100644 --- a/tests/_utils.py +++ b/tests/_utils.py @@ -3,8 +3,7 @@ from dissect.eventlog.wevt import header_def from dissect.eventlog.wevt_object import wevt_object_def -definitions = cstruct() -definitions.load(header_def + wevt_object_def) +definitions = cstruct().load(header_def + wevt_object_def) def create_header(type, **kwargs):