Adds the capability to generate core dumps when specified exceptions occur in a .NET process.

.gitignore

@@ -35,3 +35,4 @@
 bin/
 release/
 pkgbuild/
+obj/

INSTALL.md

@@ -1,6 +1,6 @@
 # Install ProcDump
 
-## Ubuntu 16.04, 18.04, 20.04 & 22.04
+## Ubuntu 16.04, 18.04, 20.04 & 22.04 & 22.10
 #### 1. Register Microsoft key and feed
 ```sh
 wget -q https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb -O packages-microsoft-prod.deb

Makefile

@@ -16,6 +16,12 @@ TESTOBJS=$(patsubst $(TESTDIR)/%.c, $(OBJDIR)/%.o, $(TESTSRC))
 OUT=$(BINDIR)/procdump
 TESTOUT=$(BINDIR)/ProcDumpTestApplication
 
+# Profiler
+PROFSRCDIR=profiler/src
+PROFINCDIR=profiler/inc
+PROFCXXFLAGS ?= -DELPP_NO_DEFAULT_LOG_FILE -DELPP_THREAD_SAFE -g -pthread -shared --no-undefined -Wno-invalid-noreturn -Wno-pragma-pack -Wno-writable-strings -Wno-format-security -fPIC -fms-extensions -DHOST_64BIT -DBIT64 -DPAL_STDCPP_COMPAT -DPLATFORM_UNIX -std=c++11
+PROFCLANG=clang++
+
 # Revision value from build pipeline
 REVISION:=$(if $(REVISION),$(REVISION),'99999')
 
@@ -39,22 +45,26 @@ PKG_VERSION:=$(if $(VERSION),$(VERSION),0.0.0)
 
 all: clean build
 
-build: $(OBJDIR) $(BINDIR) $(OUT) $(TESTOUT)
+build: $(OBJDIR)/ProcDumpProfiler.so $(OBJDIR) $(BINDIR) $(OUT) $(TESTOUT)
 
 install:
 	mkdir -p $(DESTDIR)$(INSTALLDIR)
 	cp $(BINDIR)/procdump $(DESTDIR)$(INSTALLDIR)
 	mkdir -p $(DESTDIR)$(MANDIR)
 	cp procdump.1 $(DESTDIR)$(MANDIR)
 
+$(OBJDIR)/ProcDumpProfiler.so: $(PROFSRCDIR)/ClassFactory.cpp $(PROFSRCDIR)/ProcDumpProfiler.cpp $(PROFSRCDIR)/dllmain.cpp $(PROFSRCDIR)/corprof_i.cpp $(PROFSRCDIR)/easylogging++.cc | $(OBJDIR)
+	$(PROFCLANG) -o $@ $(PROFCXXFLAGS) -I $(PROFINCDIR) $^
+	ld -r -b binary -o $(OBJDIR)/ProcDumpProfiler.o $(OBJDIR)/ProcDumpProfiler.so
+
 $(OBJDIR)/%.o: $(SRCDIR)/%.c | $(OBJDIR)
 	$(CC) -c -g -o $@ $< $(CCFLAGS)
 
 $(OBJDIR)/%.o: $(TESTDIR)/%.c | $(OBJDIR)
 	$(CC) -c -g -o $@ $< $(CCFLAGS)
 
 $(OUT): $(OBJS) | $(BINDIR)
-	$(CC) -o $@ $^ $(CCFLAGS)
+	$(CC) -o $@ $^ $(OBJDIR)/ProcDumpProfiler.o $(CCFLAGS)
 
 $(TESTOUT): $(TESTOBJS) | $(BINDIR)
 	$(CC) -o $@ $^ $(CCFLAGS)

README.md

@@ -12,6 +12,7 @@ ProcDump is a Linux reimagining of the classic ProcDump tool from the Sysinterna
   * Ubuntu 16.04 LTS
 * `gdb` >= 7.6.1
 * `zlib` (build-time only)
+* `clang`
 
 ## Install ProcDump
 Checkout our [install instructions](INSTALL.md) for distribution specific steps to install Procdump.
@@ -41,18 +42,20 @@ make && make rpm
 **BREAKING CHANGE** With the release of ProcDump 1.3 the switches are now aligned with the Windows ProcDump version.
 ```
 procdump [-n Count]
-        [-s Seconds]
-        [-c|-cl CPU_Usage]
-        [-m|-ml Commit_Usage]
-        [-tc Thread_Threshold]
-        [-fc FileDescriptor_Threshold]
-        [-sig Signal_Number]
-        [-pf Polling_Frequency]
-        [-o]
-        [-log]
-        {
+         [-s Seconds]
+         [-c|-cl CPU_Usage]
+         [-m|-ml Commit_Usage]
+         [-tc Thread_Threshold]
+         [-fc FileDescriptor_Threshold]
+         [-sig Signal_Number]
+         [-e]
+         [-f Include_Filter,...]
+         [-pf Polling_Frequency]
+         [-o]
+         [-log]
+         {
           {{[-w] Process_Name | [-pgid] PID} [Dump_File | Dump_Folder]}
-        }
+         }
 
 Options:
    -n      Number of dumps to write before exiting.
@@ -64,6 +67,8 @@ Options:
    -tc     Thread count threshold above which to create a dump of the process.
    -fc     File descriptor count threshold above which to create a dump of the process.
    -sig    Signal number to intercept to create a dump of the process.
+   -e      [.NET] Create dump when the process encounters an exception.
+   -f      [.NET] Filter (include) on the (comma seperated) exception name(s).
    -pf     Polling frequency.
    -o      Overwrite existing dump file.
    -log    Writes extended ProcDump tracing to syslog.
@@ -113,6 +118,10 @@ The following will create a core dump when a SIGSEGV occurs.
 ```
 sudo procdump -sig 11 1234
 ```
+The following will create a core dump when the target .NET application throws a System.InvalidOperationException
+```
+sudo procdump -e -f System.InvalidOperationException 1234
+```
 > All options can also be used with `-w`, to wait for any process with the given name.
 
 The following waits for a process named `my_application` and creates a core dump immediately when it is found.

dist/procdump.spec.in

@@ -54,6 +54,9 @@ make CFLAGS="%{optflags}"
 
 
 %changelog
+* Mon Dec 12 2022 Mario Hewardt <marioh@microsoft.com> - 1.4
+- added the capability to dump on .NET 1st chance exceptions (-e and -f)
+
 * Mon Sep 26 2022 Javid Habibi <jahabibi@microsoft.com> - 1.3
 - added process group trigger
 - BREAKING CHANGE: rework CLI interface to match that of Procdump for Windows

docs/coreclrintegration.md

@@ -1,6 +1,7 @@
-# Procdump and .NET Core 3 Integration
+# Procdump and .NET Integration
 
-Procdump is a powerful production diagnostics tool that allows you to monitor processes for specific thresholds and generate core dumps based on a specified critera. For example, imagine that you were encountering sporadic CPU spikes in your web app and you would like to generate a core dump for offline analysis. With Procdump you can use the -C switch to specify the CPU threshold of interest (say, 90%) and Procdump will monitor the process and generate a core dump when the CPU goes above 90%.
+## Core Dump Generation
+Procdump is a powerful production diagnostics tool that allows you to monitor processes for specific thresholds and generate core dumps based on a specified critera. For example, imagine that you were encountering sporadic CPU spikes in your web app and you would like to generate a core dump for offline analysis. With Procdump you can use the -c switch to specify the CPU threshold of interest (say, 90%) and Procdump will monitor the process and generate a core dump when the CPU goes above 90%.
 
 In order to understand how a core dump can help resolve production issues, we first have to understand what a core dump is. Essentially, a core dump is nothing more than a static snapshot of the contents of an applications memory. This content is written to a file that can later be loaded into a debugger and other tools to analyze the contents of memory and see if root cause can be determined. What makes core dumps great production debugging tools is that we don't have to worry about the application stopping while debugging is taking place (as is the case with live debugging where a debugger is attached to the application). How big are these dump files? Well, that depends on how much memory your application is consuming (remember, it roughly writes the contents of the application memory usage to the file). As you can imagine, if you are running a massive database application you could result in a core dump file that is many GB in size. Furthermore, on Linux, core dumps tend to be larger than on Windows. This presents a new challenge of how to effectively manage large core dump files. If you intend to copy the core dump file between production and development machines, it can be pretty time consuming.
 
@@ -27,7 +28,7 @@ Using the same web app, let's use Procdump **1.1** to generate the core dump. Th
 
 _Please note that by default the core dump will be placed into the same directory that the target application is running in._
 
-This time the core dump file size is only about 273MB. Much better and much more managable. To convince ourselves that the core dump still contains all the neccessary data to debug .NET Core 3.0 applications, we can try it out with dotnet-dump analyze (which is a REPL for SOS debugging):
+This time the core dump file size is only about 273MB. Much better and much more managable. To convince ourselves that the core dump still contains all the neccessary data to debug .NET Core applications, we can try it out with dotnet-dump analyze (which is a REPL for SOS debugging):
 
 ```console
 dotnet-dump analyze TestWebApp_time_2019-12-04_11:44:03.3066
@@ -46,7 +47,19 @@ Statistics:
 Total 32581 objects
 ```
 
-How does Procdump achieve this magic? Turns out that .NET Core 3.0 introduced the notion of a diagnostics server which at a high level enables external (out of process) tools to send diagnostics commands to the target process. In our case, we used the dump commands that are available but you can also use the diagnostics server to issue trace commands. For more information, please see the following documentation:
+How does Procdump achieve this magic? Turns out that .NET introduced the notion of a diagnostics server which at a high level enables external (out of process) tools to send diagnostics commands to the target process. In our case, we used the dump commands that are available but you can also use the diagnostics server to issue trace commands. For more information, please see the following documentation:
 
 [.NET Core Diagnostics](https://github.com/dotnet/diagnostics)
 
+## Monitoring for exceptions
+Often times, it's super useful to be able to get a core dump when a .NET application throws an exception. Starting in ProcDump **1.4** it now has the capability to do so by using the
+-e and -f switches. To better understand how ProcDump accomplishes this it's important to note that it requires ProcDump to inject a profiler into the target .NET process. This should
+be minimal overhead (unless your .NET application throws thousands of exceptions).
+
+When ProcDump monitors for exceptions, in addition to injecting the profiler into the target process, it also sets up a couple of IPC channels that allows the profiler to talk to
+ProcDump (and vice versa) to communicate status:
+
+ProcDump acts as a server that listens for status messages from the profiler (success, failure etc).
+Profiler acts a server that listens for cancellation requests from ProcDump (in case of SIGINT)
+
+In all cases, the profiler will be unloaded from the target .NET process once ProcDump has completed monitoring.

include/CoreDumpWriter.h

@@ -19,9 +19,7 @@
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <stdint.h>
-
-#include "Handle.h"
-#include "ProcDumpConfiguration.h"
+#include <linux/limits.h>
 
 #define DATE_LENGTH 26
 #define MAX_LINES 15
@@ -37,6 +35,16 @@ struct MagicVersion
     uint8_t Magic[14];
 };
 
+
+// CLSID struct
+struct CLSID
+{
+    uint32_t Data1;
+    unsigned short Data2;
+    unsigned short Data3;
+    unsigned char Data4[8];
+};
+
 // The header to be associated with every command and response
 // to/from the diagnostics server
 struct IpcHeader
@@ -61,6 +69,7 @@ enum ECoreDumpType {
     FILEDESC,               // trigger on file descriptor count
     SIGNAL,                 // trigger on signal
     TIME,                   // trigger on time interval
+    EXCEPTION,              // trigger on exception
     MANUAL                  // manual trigger
 };
 
@@ -70,7 +79,8 @@ struct CoreDumpWriter {
 };
 
 struct CoreDumpWriter *NewCoreDumpWriter(enum ECoreDumpType type, struct ProcDumpConfiguration *config);
-
+int WriteCoreDumpInternal(struct CoreDumpWriter *self, char* socketName);
 int WriteCoreDump(struct CoreDumpWriter *self);
+char* GetCoreDumpName(pid_t pid, char* procName, char* dumpPath, char* dumpName, enum ECoreDumpType type);
 
 #endif // CORE_DUMP_WRITER_H

include/DotnetHelpers.h

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License
+
+//--------------------------------------------------------------------
+//
+// .NET helpers
+//
+//--------------------------------------------------------------------
+
+#ifndef DOTNETHELPERS_H
+#define DOTNETHELPERS_H
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+bool IsCoreClrProcess(pid_t pid, char** socketName);
+bool GenerateCoreClrDump(char* socketName, char* dumpFileName);
+
+#endif // DOTNETHELPERS_H

include/GenHelpers.h

@@ -0,0 +1,103 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License
+
+//--------------------------------------------------------------------
+//
+// General purpose helpers
+//
+//--------------------------------------------------------------------
+
+#ifndef GENHELPERS_H
+#define GENHELPERS_H
+
+#include <linux/version.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <sys/utsname.h>
+#include <errno.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <dirent.h>
+
+#define MIN_KERNEL_VERSION 3
+#define MIN_KERNEL_PATCH 5
+
+//-------------------------------------------------------------------------------------
+// Auto clean up of memory using free (void)
+//-------------------------------------------------------------------------------------
+static inline void cleanup_void(void* val)
+{
+    void **ppVal = (void**)val;
+    free(*ppVal);
+}
+
+//-------------------------------------------------------------------------------------
+// Auto clean up of file descriptors using close
+//-------------------------------------------------------------------------------------
+static inline void cleanup_fd(int* val)
+{
+    if (*val)
+    {
+        close(*val);
+    }
+}
+
+//-------------------------------------------------------------------------------------
+// Auto clean up of dir using closedir
+//-------------------------------------------------------------------------------------
+static inline void cleanup_dir(DIR** val)
+{
+    if(*val)
+    {
+        closedir(*val);
+    }
+}
+
+//-------------------------------------------------------------------------------------
+// Auto clean up of FILE using fclose
+//-------------------------------------------------------------------------------------
+static inline void cleanup_file(FILE** val)
+{
+    if(*val)
+    {
+        fclose(*val);
+    }
+}
+
+//-------------------------------------------------------------------------------------
+// Auto cancel pthread
+//-------------------------------------------------------------------------------------
+static inline void cancel_pthread(unsigned long* val)
+{
+    if(*val!=-1)
+    {
+        pthread_cancel(*val);
+    }
+}
+
+#define auto_free __attribute__ ((__cleanup__(cleanup_void)))
+#define auto_free_fd __attribute__ ((__cleanup__(cleanup_fd)))
+#define auto_free_dir __attribute__ ((__cleanup__(cleanup_dir)))
+#define auto_free_file __attribute__ ((__cleanup__(cleanup_file)))
+#define auto_cancel_thread __attribute__ ((__cleanup__(cancel_pthread)))
+
+bool ConvertToInt(const char* src, int* conv);
+bool IsValidNumberArg(const char *arg);
+bool CheckKernelVersion();
+uint16_t* GetUint16(char* buffer);
+char* GetPath(char* lineBuf);
+FILE *popen2(const char *command, const char *type, pid_t *pid);
+char *sanitize(char *processName);
+int StringToGuid(char* szGuid, struct CLSID* pGuid);
+int GetHex(char* szStr, int size, void* pResult);
+bool createDir(const char *dir, mode_t perms);
+char* GetSocketPath(char* prefix, pid_t pid, pid_t targetPid);
+int send_all(int socket, void *buffer, size_t length);
+int recv_all(int socket, void* buffer, size_t length);
+
+#endif // GENHELPERS_H
+

include/Includes.h

@@ -0,0 +1,11 @@
+#include "CoreDumpWriter.h"
+#include "Events.h"
+#include "GenHelpers.h"
+#include "Handle.h"
+#include "Logging.h"
+#include "Monitor.h"
+#include "Procdump.h"
+#include "ProcDumpConfiguration.h"
+#include "Process.h"
+#include "DotnetHelpers.h"
+#include "ProfilerHelpers.h"

include/Monitor.h

@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License
+
+//--------------------------------------------------------------------
+//
+// Monitor functions
+//
+//--------------------------------------------------------------------
+
+#ifndef MONITOR_H
+#define MONITOR_H
+
+#include <signal.h>
+#include <sys/ptrace.h>
+#include <stdlib.h>
+
+#include "ProcDumpConfiguration.h"
+
+#define MAX_PROFILER_CONNECTIONS    50
+
+// Monitor functions
+void MonitorProcesses(struct ProcDumpConfiguration*self);
+int CreateMonitorThreads(struct ProcDumpConfiguration *self);
+int StartMonitor(struct ProcDumpConfiguration* monitorConfig);
+int WaitForQuit(struct ProcDumpConfiguration *self, int milliseconds);
+int WaitForQuitOrEvent(struct ProcDumpConfiguration *self, struct Handle *handle, int milliseconds);
+int WaitForAllMonitorsToTerminate(struct ProcDumpConfiguration *self);
+int WaitForSignalThreadToTerminate(struct ProcDumpConfiguration *self);
+bool IsQuit(struct ProcDumpConfiguration *self);
+int SetQuit(struct ProcDumpConfiguration *self, int quit);
+bool ContinueMonitoring(struct ProcDumpConfiguration *self);
+bool BeginMonitoring(struct ProcDumpConfiguration *self);
+
+// Monitor worker threads
+void *CommitMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
+void *CpuMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
+void *ThreadCountMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
+void *FileDescriptorCountMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
+void *SignalMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
+void *TimerThread(void *thread_args /* struct ProcDumpConfiguration* */);
+void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
+void *ProcessMonitor(void *thread_args /* struct ProcDumpConfiguration* */);
+void *WaitForProfilerCompletion(void *thread_args /* struct ProcDumpConfiguration* */);
+
+#endif // MONITOR_H