Add support for ractors (fix #50)

README.md

@@ -332,11 +332,18 @@ p articles.to_a
 
 ## Concurrency
 
-Extralite releases the GVL while making calls to the sqlite3 library that might
-block, such as when backing up a database, or when preparing a query. Extralite
-also releases the GVL periodically when iterating over records. By default, the
-GVL is released every 1000 records iterated. The GVL release threshold can be
-set separately for each database:
+### The Ruby GVL
+
+Extralite releases the [Ruby
+GVL](https://www.speedshop.co/2020/05/11/the-ruby-gvl-and-scaling.html) while
+making calls to the sqlite3 library that might block, such as when backing up a
+database, or when preparing a query. This allows other threads to run while the
+underlying sqlite3 library is busy preparing queries, fetching records and
+backing up databases.
+
+Extralite also releases the GVL periodically when iterating over records. By
+default, the GVL is released every 1000 records iterated. The GVL release
+threshold can be set separately for each database:
 
 ```ruby
 db.gvl_release_threshold = 10 # release GVL every 10 records
@@ -345,8 +352,8 @@ db.gvl_release_threshold = nil # use default value (currently 1000)
 ```
 
 For most applications, there's no need to tune the GVL threshold value, as it
-provides [excellent](#performance) performance characteristics for both single-threaded and
-multi-threaded applications.
+provides [excellent](#performance) performance characteristics for both
+single-threaded and multi-threaded applications.
 
 In a heavily multi-threaded application, releasing the GVL more often (lower
 threshold value) will lead to less latency (for threads not running a query),
@@ -360,6 +367,21 @@ latency and throughput:
 less latency & throughput <<< GVL release threshold >>> more latency & throughput
 ```
 
+### Thread Safety
+
+A single database instance can be safely used in multiple threads simultaneously
+as long as the following conditions are met:
+
+- No explicit transactions are used.
+- Each thread issues queries by calling `Database#query_xxx`, or uses a separate
+  `Query` instance.
+- The GVL release threshold is not `0` (i.e. the GVL is released periodically
+  while running queries.)
+
+### Use with Ractors
+
+Extralite databases can be used inside ractors 
+
 ## Performance
 
 A benchmark script is included, creating a table of various row counts, then

ext/extralite/database.c

@@ -341,11 +341,15 @@ VALUE Database_execute(int argc, VALUE *argv, VALUE self) {
   return Database_perform_query(argc, argv, self, safe_query_changes);
 }
 
-/* call-seq:
- *   db.execute_multi(sql, params_array) -> changes
+/* call-seq: db.execute_multi(sql, params_array) -> changes
+ *   db.execute_multi(sql) { ... } -> changes
+ *
+ * Executes the given query for each list of parameters in params_array. If a
+ * block is given, the block is called for each iteration, and its return value
+ * is used as parameters for the query. To stop iteration, the block should
+ * return nil.
  *
- * Executes the given query for each list of parameters in params_array. Returns
- * the number of changes effected. This method is designed for inserting
+ * Returns the number of changes effected. This method is designed for inserting
  * multiple records.
  *
  *     records = [
@@ -354,6 +358,17 @@ VALUE Database_execute(int argc, VALUE *argv, VALUE self) {
  *     ]
  *     db.execute_multi('insert into foo values (?, ?, ?)', records)
  *
+ *     records = [
+ *       [1, 2, 3],
+ *       [4, 5, 6]
+ *     ]
+ *     db.execute_multi('insert into foo values (?, ?, ?)') do
+ *       x = queue.pop
+ *       y = queue.pop
+ *       z = queue.pop
+ *       [x, y, z]
+ *     end
+ * 
  */
 VALUE Database_execute_multi(VALUE self, VALUE sql, VALUE params_array) {
   Database_t *db = self_to_open_database(self);

ext/extralite/extralite_ext.c

@@ -1,8 +1,12 @@
+#include "ruby.h"
+
 void Init_ExtraliteDatabase();
 void Init_ExtraliteQuery();
 void Init_ExtraliteIterator();
 
 void Init_extralite_ext(void) {
+  rb_ext_ractor_safe(true);
+
   Init_ExtraliteDatabase();
   Init_ExtraliteQuery();
   Init_ExtraliteIterator();

test/test_database.rb

@@ -61,7 +61,7 @@ def test_query_single_column
 
     r = @db.query_single_column('select y from t where x = 2')
     assert_equal [], r
-end
+  end
 
   def test_query_single_value
     r = @db.query_single_value('select z from t order by Z desc limit 1')
@@ -616,6 +616,32 @@ def test_concurrent_transactions
     assert_equal [1, 4, 7], result
   end
 
+  def test_concurrent_queries
+    @db.query('delete from t')
+    @db.gvl_release_threshold = 1
+    q1 = @db.prepare('insert into t values (?, ?, ?)')
+    q2 = @db.prepare('insert into t values (?, ?, ?)')
+
+    t1 = Thread.new do
+      data = (1..50).each_slice(10).map { |a| a.map { |i| [i, i + 1, i + 2] } }
+      data.each do |params|
+        q1.execute_multi(params)
+      end
+    end
+
+    t2 = Thread.new do
+      data = (51..100).each_slice(10).map { |a| a.map { |i| [i, i + 1, i + 2] } }
+      data.each do |params|
+        q2.execute_multi(params)
+      end
+    end
+
+    t1.join
+    t2.join
+
+    assert_equal (1..100).to_a, @db.query_single_column('select x from t order by x')
+  end
+
   def test_database_trace
     sqls = []
     @db.trace { |sql| sqls << sql }
@@ -773,4 +799,81 @@ def test_gvl_mode_get_set
     db.gvl_release_threshold = nil
     assert_equal 1000, db.gvl_release_threshold
   end
-end
+end
+
+class RactorTest < Minitest::Test
+  def test_ractor_simple
+    fn = Tempfile.new('extralite_test_database_in_ractor').path
+
+    r = Ractor.new do
+      path = receive
+      db = Extralite::Database.new(path)
+      i = receive
+      db.execute 'insert into foo values (?)', i
+    end
+
+    r << fn
+    db = Extralite::Database.new(fn)
+    db.execute 'create table foo (x)'
+    r << 42
+    r.take # wait for ractor to terminate
+
+    assert_equal 42, db.query_single_value('select x from foo')
+  end
+
+  # Adapted from here: https://github.com/sparklemotion/sqlite3-ruby/pull/365/files
+  def test_ractor_share_database
+    db_receiver = Ractor.new do
+      db = Ractor.receive
+      Ractor.yield db.object_id
+      begin
+        db.execute("create table foo (b)")
+        raise "Should have raised an exception in db.execute()"
+      rescue => e
+        Ractor.yield e
+      end
+    end
+    db_creator = Ractor.new(db_receiver) do |db_receiver|
+      db = Extralite::Database.new(":memory:")
+      Ractor.yield db.object_id
+      db_receiver.send(db)
+      sleep 0.1
+      db.execute("create table foo (a)")
+    end
+    first_oid = db_creator.take
+    second_oid = db_receiver.take
+    refute_equal first_oid, second_oid
+    ex = db_receiver.take
+    assert_kind_of Extralite::Error, ex
+    assert_equal "Database is closed", ex.message
+  end
+
+  STRESS_DB_NAME = Tempfile.new('extralite_test_ractor_stress').path
+
+  # Adapted from here: https://github.com/sparklemotion/sqlite3-ruby/pull/365/files
+  def test_ractor_stress
+    Ractor.make_shareable(STRESS_DB_NAME)
+
+    db = Extralite::Database.new(STRESS_DB_NAME)
+    db.execute("PRAGMA journal_mode=WAL") # A little slow without this
+    db.execute("create table stress_test (a integer primary_key, b text)")
+    random = Random.new.freeze
+    ractors = (0..9).map do |ractor_number|
+      Ractor.new(random, ractor_number) do |random, ractor_number|
+        db_in_ractor = Extralite::Database.new(STRESS_DB_NAME)
+        db_in_ractor.busy_timeout = 3
+        10.times do |i|
+          db_in_ractor.execute("insert into stress_test(a, b) values (#{ractor_number * 100 + i}, '#{random.rand}')")
+        end
+      end
+    end
+    ractors.each { |r| r.take }
+    final_check = Ractor.new do
+      db_in_ractor = Extralite::Database.new(STRESS_DB_NAME)
+      count = db_in_ractor.query_single_value("select count(*) from stress_test")
+      Ractor.yield count
+    end
+    count = final_check.take
+    assert_equal 100, count
+  end
+end