Add _TSK_BIG_TABLES typedef.

Closes tskit-dev#1528

.circleci/config.yml

@@ -50,6 +50,12 @@ jobs:
             CC=clang CXX=clang++ meson build-clang c
             ninja -C build-clang
 
+      - run:
+          name: Compile C with gcc in 64 bit mode
+          command: |
+            meson build-gcc-64 c -D CFLAGS=-D_TSK_BIG_TABLES -D CPPFLAGS=-D_TSK_BIG_TABLES
+            ninja -C build-gcc-64
+
       - run:
           name: Run C tests
           command: |
@@ -76,6 +82,19 @@ jobs:
             valgrind --leak-check=full --error-exitcode=1 ./build-gcc/test_file_format
             valgrind --leak-check=full --error-exitcode=1 ./build-gcc/test_minimal_cpp
 
+      - run:
+          name: Valgrind for 64 bit C tests.
+          command: |
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_core
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_tables
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_trees
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_genotypes
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_convert
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_stats
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_haplotype_matching
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_file_format
+            valgrind --leak-check=full --error-exitcode=1 ./build-gcc-64/test_minimal_cpp
+
       - run:
           name: Run clang-compiled C tests
           command: |

c/tests/test_file_format.c

@@ -361,15 +361,17 @@ verify_bad_offset_columns(tsk_treeseq_t *ts, const char *offset_col)
     tsk_table_collection_t tables;
     tsk_size_t *offset_array, *offset_copy;
     size_t offset_len;
+    int type;
     tsk_size_t data_len;
 
     ret = tsk_treeseq_dump(ts, _tmp_file_name, 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
     ret = kastore_open(&store, _tmp_file_name, "r", 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
 
-    ret = kastore_gets_uint32(&store, offset_col, &offset_array, &offset_len);
+    ret = kastore_gets(&store, offset_col, (void **) &offset_array, &offset_len, &type);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
+    CU_ASSERT_EQUAL_FATAL(type, TSK_SIZE_STORAGE_TYPE);
     offset_copy = malloc(offset_len * sizeof(*offset_array));
     CU_ASSERT_FATAL(offset_copy != NULL);
     memcpy(offset_copy, offset_array, offset_len * sizeof(*offset_array));

c/tskit/tables.c

@@ -61,8 +61,8 @@ typedef struct {
 static bool
 check_table_overflow(tsk_size_t current_size, tsk_size_t additional_rows)
 {
-    uint64_t new_size = (uint64_t) current_size + (uint64_t) additional_rows;
-    return new_size > ((uint64_t) TSK_MAX_ID) + 1;
+    tsk_size_t max_val = TSK_MAX_ID + (tsk_size_t) 1;
+    return current_size > (max_val - additional_rows);
 }
 
 /* Returns true if adding the specified number of elements would result in overflow
@@ -71,8 +71,7 @@ check_table_overflow(tsk_size_t current_size, tsk_size_t additional_rows)
 static bool
 check_offset_overflow(tsk_size_t current_size, tsk_size_t additional_elements)
 {
-    uint64_t new_size = (uint64_t) current_size + (uint64_t) additional_elements;
-    return new_size > TSK_MAX_SIZE;
+    return current_size > (TSK_MAX_SIZE - additional_elements);
 }
 
 static int

c/tskit/tables.h

@@ -52,9 +52,26 @@ when manipulating these ID values. The reserved value ``TSK_NULL`` (-1) defines
 missing data.
 @endrst
 */
+#ifdef _TSK_BIG_TABLES
+/* Allow tables to have more than 2^31 rows. This is an EXPERIMENTAL feature
+ * and is not supported in any way. This typedef is only included for
+ * future-proofing purposes, so that we can be sure that we don't make any
+ * design decisions that are incompatible with big tables by building the
+ * library in 64 bit mode in CI. See the discussion here for more background:
+
+ * https://github.com/tskit-dev/tskit/issues/343
+ *
+ * If you need big tables, please open an issue on GitHub to discuss, or comment
+ * on the thread above.
+ */
+typedef int64_t tsk_id_t;
+#define TSK_MAX_ID INT64_MAX
+#define TSK_ID_STORAGE_TYPE KAS_INT64
+#else
 typedef int32_t tsk_id_t;
 #define TSK_MAX_ID INT32_MAX
 #define TSK_ID_STORAGE_TYPE KAS_INT32
+#endif
 
 /**
 @brief Tskit sizes.
@@ -63,9 +80,16 @@ typedef int32_t tsk_id_t;
 Sizes in tskit are defined by the ``tsk_size_t`` type.
 @endrst
 */
+#ifdef _TSK_BIG_TABLES
+/* TODO get rid of this typdef once we move to 64 bit sizes */
+typedef uint64_t tsk_size_t;
+#define TSK_MAX_SIZE UINT64_MAX
+#define TSK_SIZE_STORAGE_TYPE KAS_UINT64
+#else
 typedef uint32_t tsk_size_t;
 #define TSK_MAX_SIZE UINT32_MAX
 #define TSK_SIZE_STORAGE_TYPE KAS_UINT32
+#endif
 
 /**
 @brief Container for bitwise flags.