Fix stack scanning for the Julia GC when GAP is used as a library.

src/gap.h

@@ -127,6 +127,12 @@ enum {
 };
 
 
+/****************************************************************************
+**
+*F  IsUsingLibGap()  . . . . . . . . 1 if GAP is being used a library, else 0
+*/
+int IsUsingLibGap(void);
+
 /****************************************************************************
 **
 *F  InitializeGap( <argc>, <argv>, <handleSignals> )  . . . . . . .  init GAP

src/julia_gc.c

@@ -25,6 +25,7 @@
 #include "sysmem.h"
 #include "system.h"
 #include "vars.h"
+#include "gap.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -660,7 +661,18 @@ static void GapRootScanner(int full)
     // current_task != root_task.
     char * stackend = (char *)jl_task_stack_buffer(task, &size, &tid);
     stackend += size;
-    if (JuliaTLS->tid == 0 && JuliaTLS->root_task == task) {
+    // The following test overrides the stackend if the following two
+    // conditions hold:
+    //
+    // 1. GAP is not being used as a library, but is the main program
+    //    and in charge of the main() function.
+    // 2. The stack of the current task is that of the main task of the
+    //    main thread.
+    //
+    // The reason is that if Julia is being initialized from GAP, it
+    // cannot always reliably find the top of the stack for that task,
+    // so we have to fall back to GAP for that.
+    if (!IsUsingLibGap() && JuliaTLS->tid == 0 && JuliaTLS->root_task == task) {
         stackend = (char *)GapStackBottom;
     }
 

src/libgap-api.c

@@ -31,6 +31,13 @@
 #include "streams.h"
 #include "stringobj.h"
 
+static int UsingLibGap = 0;
+
+int IsUsingLibGap(void)
+{
+    return UsingLibGap;
+}
+
 
 //
 // Setup and initialisation
@@ -41,6 +48,8 @@ void GAP_Initialize(int              argc,
                     GAP_CallbackFunc errorCallback,
                     int              handleSignals)
 {
+    UsingLibGap = 1;
+
     InitializeGap(&argc, argv, handleSignals);
     SetExtraMarkFuncBags(markBagsCallback);
     STATE(JumpToCatchCallback) = errorCallback;