From d99b2dbc7fe61a3ee5254d1b7a90fcc119771839 Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Mon, 29 Jan 2024 07:47:15 +0200 Subject: [PATCH 01/11] Add tick and mode to on_progress. --- examples/pubsub_store_polyphony.rb | 8 +- examples/pubsub_store_threads.rb | 23 +++-- ext/extralite/common.c | 3 + ext/extralite/database.c | 136 ++++++++++++++++++++++------- ext/extralite/extralite.h | 33 +++++-- ext/extralite/query.c | 9 +- test/test_database.rb | 54 ++++++++++++ 7 files changed, 213 insertions(+), 53 deletions(-) diff --git a/examples/pubsub_store_polyphony.rb b/examples/pubsub_store_polyphony.rb index 4306c68..91d97ff 100644 --- a/examples/pubsub_store_polyphony.rb +++ b/examples/pubsub_store_polyphony.rb @@ -103,9 +103,9 @@ def prune_subscribers db3 = Extralite::Database.new(fn) db3.pragma(journal_mode: :wal, synchronous: 1) -db1.on_progress(50) { |b| b ? sleep(0.0001) : snooze } -db2.on_progress(50) { |b| b ? sleep(0.0001) : snooze } -db3.on_progress(50) { |b| b ? sleep(0.0001) : snooze } +db1.on_progress(1000) { |b| b ? sleep(0.0001) : snooze } +db2.on_progress(1000) { |b| b ? sleep(0.0001) : snooze } +db3.on_progress(1000) { |b| b ? sleep(0.0001) : snooze } producer = PubSub.new(db1) producer.setup @@ -163,7 +163,7 @@ def prune_subscribers db4 = Extralite::Database.new(fn) db4.pragma(journal_mode: :wal, synchronous: 1) -db4.on_progress(10) { sleep 0.05 } +db4.on_progress(1000) { |busy| busy ? sleep(0.05) : snooze } last_t = Time.now last_publish_count = 0 diff --git a/examples/pubsub_store_threads.rb b/examples/pubsub_store_threads.rb index 8a4fdf7..bfe9970 100644 --- a/examples/pubsub_store_threads.rb +++ b/examples/pubsub_store_threads.rb @@ -34,14 +34,21 @@ def unsubscribe(*topics) end def get_messages(&block) - messages = @db.query_ary('delete from messages where subscriber_id = ? returning topic, message', @id) - if block - messages.each(&block) - nil - else - messages - end - # end + @db.transaction(:deferred) do + results = @db.query_ary('select topic, message from messages where subscriber_id = ?', @id) + return [] if results.empty? + + @db.execute('delete from messages where subscriber_id = ?', @id) + results + end + + # messages = @db.query_ary('delete from messages where subscriber_id = ? returning topic, message', @id) + # if block + # messages.each(&block) + # nil + # else + # messages + # end rescue Extralite::BusyError p busy: :get_message block_given? ? nil : [] diff --git a/ext/extralite/common.c b/ext/extralite/common.c index 664b7f0..4b8c84d 100644 --- a/ext/extralite/common.c +++ b/ext/extralite/common.c @@ -587,6 +587,7 @@ static inline VALUE batch_run_array(query_ctx *ctx, enum batch_mode batch_mode) for (int i = 0; i < count; i++) { sqlite3_reset(ctx->stmt); sqlite3_clear_bindings(ctx->stmt); + Database_issue_query(ctx->db, ctx->sql); bind_all_parameters_from_object(ctx->stmt, RARRAY_AREF(ctx->params, i)); batch_iterate(ctx, batch_mode, &rows); @@ -623,6 +624,7 @@ static VALUE batch_run_each_iter(RB_BLOCK_CALL_FUNC_ARGLIST(yield_value, vctx)) sqlite3_reset(each_ctx->ctx->stmt); sqlite3_clear_bindings(each_ctx->ctx->stmt); + Database_issue_query(each_ctx->ctx->db, each_ctx->ctx->sql); bind_all_parameters_from_object(each_ctx->ctx->stmt, yield_value); batch_iterate(each_ctx->ctx, each_ctx->batch_mode, &rows); @@ -668,6 +670,7 @@ static inline VALUE batch_run_proc(query_ctx *ctx, enum batch_mode batch_mode) { sqlite3_reset(ctx->stmt); sqlite3_clear_bindings(ctx->stmt); + Database_issue_query(ctx->db, ctx->sql); bind_all_parameters_from_object(ctx->stmt, params); batch_iterate(ctx, batch_mode, &rows); diff --git a/ext/extralite/database.c b/ext/extralite/database.c index 51c1531..4a3dd2c 100644 --- a/ext/extralite/database.c +++ b/ext/extralite/database.c @@ -35,13 +35,13 @@ static size_t Database_size(const void *ptr) { static void Database_mark(void *ptr) { Database_t *db = ptr; rb_gc_mark_movable(db->trace_proc); - rb_gc_mark_movable(db->progress_handler_proc); + rb_gc_mark_movable(db->progress_handler.proc); } static void Database_compact(void *ptr) { Database_t *db = ptr; - db->trace_proc = rb_gc_location(db->trace_proc); - db->progress_handler_proc = rb_gc_location(db->progress_handler_proc); + db->trace_proc = rb_gc_location(db->trace_proc); + db->progress_handler.proc = rb_gc_location(db->progress_handler.proc); } static void Database_free(void *ptr) { @@ -99,8 +99,6 @@ static inline int db_open_flags_from_opts(VALUE opts) { return SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; } -VALUE Database_execute(int argc, VALUE *argv, VALUE self); - void Database_apply_opts(VALUE self, Database_t *db, VALUE opts) { VALUE value = Qnil; @@ -173,9 +171,11 @@ VALUE Database_initialize(int argc, VALUE *argv, VALUE self) { #endif db->trace_proc = Qnil; - db->progress_handler_proc = Qnil; db->gvl_release_threshold = DEFAULT_GVL_RELEASE_THRESHOLD; + db->progress_handler.mode = PROGRESS_NONE; + db->progress_handler.proc = Qnil; + if (!NIL_P(opts)) Database_apply_opts(self, db, opts); return Qnil; @@ -245,16 +245,19 @@ static inline VALUE Database_perform_query(int argc, VALUE *argv, VALUE self, VA sql = rb_funcall(argv[0], ID_strip, 0); if (RSTRING_LEN(sql) == 0) return Qnil; - TRACE_SQL(db, sql); + Database_issue_query(db, sql); prepare_multi_stmt(DB_GVL_MODE(db), db->sqlite3_db, &stmt, sql); RB_GC_GUARD(sql); bind_all_parameters(stmt, argc - 1, argv + 1); query_ctx ctx = QUERY_CTX( - self, db, stmt, Qnil, transform, query_mode, ROW_YIELD_OR_MODE(ROW_MULTI), ALL_ROWS + self, sql, db, stmt, Qnil, transform, + query_mode, ROW_YIELD_OR_MODE(ROW_MULTI), ALL_ROWS ); - - return rb_ensure(SAFE(call), (VALUE)&ctx, SAFE(cleanup_stmt), (VALUE)&ctx); + + VALUE result = rb_ensure(SAFE(call), (VALUE)&ctx, SAFE(cleanup_stmt), (VALUE)&ctx); + RB_GC_GUARD(result); + return result; } /* call-seq: @@ -477,7 +480,10 @@ VALUE Database_batch_execute(VALUE self, VALUE sql, VALUE parameters) { if (RSTRING_LEN(sql) == 0) return Qnil; prepare_single_stmt(DB_GVL_MODE(db), db->sqlite3_db, &stmt, sql); - query_ctx ctx = QUERY_CTX(self, db, stmt, parameters, Qnil, QUERY_HASH, ROW_MULTI, ALL_ROWS); + query_ctx ctx = QUERY_CTX( + self, sql, db, stmt, parameters, + Qnil, QUERY_HASH, ROW_MULTI, ALL_ROWS + ); return rb_ensure(SAFE(safe_batch_execute), (VALUE)&ctx, SAFE(cleanup_stmt), (VALUE)&ctx); } @@ -508,7 +514,10 @@ VALUE Database_batch_query(VALUE self, VALUE sql, VALUE parameters) { sqlite3_stmt *stmt; prepare_single_stmt(DB_GVL_MODE(db), db->sqlite3_db, &stmt, sql); - query_ctx ctx = QUERY_CTX(self, db, stmt, parameters, Qnil, QUERY_HASH, ROW_MULTI, ALL_ROWS); + query_ctx ctx = QUERY_CTX( + self, sql, db, stmt, parameters, + Qnil, QUERY_HASH, ROW_MULTI, ALL_ROWS + ); return rb_ensure(SAFE(safe_batch_query), (VALUE)&ctx, SAFE(cleanup_stmt), (VALUE)&ctx); } @@ -539,7 +548,10 @@ VALUE Database_batch_query_ary(VALUE self, VALUE sql, VALUE parameters) { sqlite3_stmt *stmt; prepare_single_stmt(DB_GVL_MODE(db), db->sqlite3_db, &stmt, sql); - query_ctx ctx = QUERY_CTX(self, db, stmt, parameters, Qnil, QUERY_ARY, ROW_MULTI, ALL_ROWS); + query_ctx ctx = QUERY_CTX( + self, sql, db, stmt, parameters, + Qnil, QUERY_ARY, ROW_MULTI, ALL_ROWS + ); return rb_ensure(SAFE(safe_batch_query_ary), (VALUE)&ctx, SAFE(cleanup_stmt), (VALUE)&ctx); } @@ -570,7 +582,10 @@ VALUE Database_batch_query_argv(VALUE self, VALUE sql, VALUE parameters) { sqlite3_stmt *stmt; prepare_single_stmt(DB_GVL_MODE(db), db->sqlite3_db, &stmt, sql); - query_ctx ctx = QUERY_CTX(self, db, stmt, parameters, Qnil, QUERY_ARGV, ROW_MULTI, ALL_ROWS); + query_ctx ctx = QUERY_CTX( + self, sql, db, stmt, parameters, + Qnil, QUERY_ARGV, ROW_MULTI, ALL_ROWS + ); return rb_ensure(SAFE(safe_batch_query_argv), (VALUE)&ctx, SAFE(cleanup_stmt), (VALUE)&ctx); } @@ -1004,22 +1019,52 @@ VALUE Database_track_changes(int argc, VALUE *argv, VALUE self) { int Database_progress_handler(void *ptr) { Database_t *db = (Database_t *)ptr; - rb_funcall(db->progress_handler_proc, ID_call, 0); + db->progress_handler.tick_count += db->progress_handler.tick; + if (db->progress_handler.tick_count < db->progress_handler.period) + goto done; + + db->progress_handler.tick_count -= db->progress_handler.period; + db->progress_handler.call_count += 1; + rb_funcall(db->progress_handler.proc, ID_call, 0); +done: return 0; } int Database_busy_handler(void *ptr, int v) { Database_t *db = (Database_t *)ptr; - rb_funcall(db->progress_handler_proc, ID_call, 1, Qtrue); + rb_funcall(db->progress_handler.proc, ID_call, 1, Qtrue); return 1; } void Database_reset_progress_handler(VALUE self, Database_t *db) { - RB_OBJ_WRITE(self, &db->progress_handler_proc, Qnil); + db->progress_handler.mode = PROGRESS_NONE; + RB_OBJ_WRITE(self, &db->progress_handler.proc, Qnil); sqlite3_progress_handler(db->sqlite3_db, 0, NULL, NULL); sqlite3_busy_handler(db->sqlite3_db, NULL, NULL); } +static inline enum progress_handler_mode max_calls_to_progress_mode(VALUE max_calls) { + switch (NUM2INT(max_calls)) { + case 1: + return PROGRESS_ONCE; + case -1: + return PROGRESS_AT_LEAST_ONCE; + default: + return PROGRESS_NORMAL; + } +} + +inline void Database_issue_query(Database_t *db, VALUE sql) { + if (db->trace_proc != Qnil) rb_funcall(db->trace_proc, ID_call, 1, sql); + switch (db->progress_handler.mode) { + case PROGRESS_AT_LEAST_ONCE: + case PROGRESS_ONCE: + rb_funcall(db->progress_handler.proc, ID_call, 0); + default: + // do nothing + } +} + /* Installs or removes a progress handler that will be executed periodically * while a query is running. This method can be used to support switching * between fibers and threads or implementing timeouts for running queries. @@ -1068,24 +1113,53 @@ void Database_reset_progress_handler(VALUE self, Database_t *db) { * @param period [Integer] progress handler period * @returns [Extralite::Database] database */ -VALUE Database_on_progress(VALUE self, VALUE period) { +VALUE Database_on_progress(int argc, VALUE *argv, VALUE self) { Database_t *db = self_to_open_database(self); - int period_int = NUM2INT(period); + VALUE period; + VALUE opt; + static ID kw_ids[2]; + VALUE kw_args[2]; - if (period_int > 0 && rb_block_given_p()) { - RB_OBJ_WRITE(self, &db->progress_handler_proc, rb_block_proc()); - db->gvl_release_threshold = -1; + rb_scan_args(argc, argv, "10:", &period, &opt); + int period_int = NUM2INT(period); - sqlite3_progress_handler(db->sqlite3_db, period_int, &Database_progress_handler, db); - sqlite3_busy_handler(db->sqlite3_db, &Database_busy_handler, db); - } - else { - RB_OBJ_WRITE(self, &db->progress_handler_proc, Qnil); + if (!rb_block_given_p() || period_int == 0) { + Database_reset_progress_handler(self, db); db->gvl_release_threshold = DEFAULT_GVL_RELEASE_THRESHOLD; - sqlite3_progress_handler(db->sqlite3_db, 0, NULL, NULL); - sqlite3_busy_handler(db->sqlite3_db, NULL, NULL); + return self; } + int tick_int = 10; + enum progress_handler_mode mode = PROGRESS_NORMAL; + + if (!NIL_P(opt)) { + if (!kw_ids[0]) { + CONST_ID(kw_ids[0], "tick"); + CONST_ID(kw_ids[1], "max_calls"); + } + + rb_get_kwargs(opt, kw_ids, 0, 2, kw_args); + if (kw_args[0] != Qundef) { tick_int = NUM2INT(kw_args[0]); } + if (kw_args[1] != Qundef) { mode = max_calls_to_progress_mode(kw_args[1]); } + } + if (tick_int > period_int) tick_int = period_int; + + db->gvl_release_threshold = -1; + db->progress_handler.mode = mode; + RB_OBJ_WRITE(self, &db->progress_handler.proc, rb_block_proc()); + db->progress_handler.period = period_int; + db->progress_handler.tick = tick_int; + db->progress_handler.tick_count = 0; + db->progress_handler.call_count = 0; + + // The PROGRESS_ONCE mode works by invoking the progress handler proc exactly + // once, before iterating over the result set, so in that mode we don't + // actually need to set the progress handler at the sqlite level. + if (mode != PROGRESS_ONCE) + sqlite3_progress_handler(db->sqlite3_db, tick_int, &Database_progress_handler, db); + + sqlite3_busy_handler(db->sqlite3_db, &Database_busy_handler, db); + return self; } @@ -1178,7 +1252,7 @@ VALUE Database_gvl_release_threshold_set(VALUE self, VALUE value) { if (value_int < -1) rb_raise(eArgumentError, "Invalid GVL release threshold value (expect integer >= -1)"); - if (value_int > -1 && !NIL_P(db->progress_handler_proc)) + if (value_int > -1 && db->progress_handler.mode != PROGRESS_NONE) Database_reset_progress_handler(self, db); db->gvl_release_threshold = value_int; break; @@ -1235,7 +1309,7 @@ void Init_ExtraliteDatabase(void) { DEF("load_extension", Database_load_extension, 1); #endif - DEF("on_progress", Database_on_progress, 1); + DEF("on_progress", Database_on_progress, -1); DEF("prepare", Database_prepare_hash, -1); DEF("prepare_argv", Database_prepare_argv, -1); DEF("prepare_ary", Database_prepare_ary, -1); diff --git a/ext/extralite/extralite.h b/ext/extralite/extralite.h index 3a866c2..98346f1 100644 --- a/ext/extralite/extralite.h +++ b/ext/extralite/extralite.h @@ -44,11 +44,27 @@ extern VALUE SYM_argv; extern VALUE SYM_ary; extern VALUE SYM_hash; +enum progress_handler_mode { + PROGRESS_NONE, + PROGRESS_NORMAL, + PROGRESS_ONCE, + PROGRESS_AT_LEAST_ONCE, +}; + +struct progress_handler { + enum progress_handler_mode mode; + VALUE proc; + int period; + int tick; + int tick_count; + int call_count; +}; + typedef struct { - sqlite3 *sqlite3_db; - VALUE trace_proc; - VALUE progress_handler_proc; - int gvl_release_threshold; + sqlite3 *sqlite3_db; + VALUE trace_proc; + int gvl_release_threshold; + struct progress_handler progress_handler; } Database_t; enum query_mode { @@ -95,9 +111,11 @@ enum row_mode { typedef struct { VALUE self; + VALUE sql; VALUE params; VALUE transform_proc; + Database_t *db; sqlite3 *sqlite3_db; sqlite3_stmt *stmt; @@ -119,10 +137,12 @@ enum gvl_mode { #define SINGLE_ROW -2 #define ROW_YIELD_OR_MODE(default) (rb_block_given_p() ? ROW_YIELD : default) #define ROW_MULTI_P(mode) (mode == ROW_MULTI) -#define QUERY_CTX(self, db, stmt, params, transform_proc, query_mode, row_mode, max_rows) { \ +#define QUERY_CTX(self, sql, db, stmt, params, transform_proc, query_mode, row_mode, max_rows) { \ self, \ + sql, \ params, \ transform_proc, \ + db, \ db->sqlite3_db, \ stmt, \ db->gvl_release_threshold, \ @@ -132,8 +152,6 @@ enum gvl_mode { 0, \ 0 \ } -#define TRACE_SQL(db, sql) \ - if (db->trace_proc != Qnil) rb_funcall(db->trace_proc, ID_call, 1, sql); #define DEFAULT_GVL_RELEASE_THRESHOLD 1000 @@ -165,6 +183,7 @@ void bind_all_parameters_from_object(sqlite3_stmt *stmt, VALUE obj); int stmt_iterate(query_ctx *ctx); VALUE cleanup_stmt(query_ctx *ctx); +void Database_issue_query(Database_t *db, VALUE sql); sqlite3 *Database_sqlite3_db(VALUE self); enum gvl_mode Database_prepare_gvl_mode(Database_t *db); Database_t *self_to_database(VALUE self); diff --git a/ext/extralite/query.c b/ext/extralite/query.c index c6584d4..652ff0d 100644 --- a/ext/extralite/query.c +++ b/ext/extralite/query.c @@ -123,7 +123,7 @@ VALUE Query_initialize(VALUE self, VALUE db, VALUE sql, VALUE mode) { static inline void query_reset(Query_t *query) { if (!query->stmt) prepare_single_stmt(DB_GVL_MODE(query), query->sqlite3_db, &query->stmt, query->sql); - TRACE_SQL(query->db_struct, query->sql); + Database_issue_query(query->db_struct, query->sql); sqlite3_reset(query->stmt); query->eof = 0; } @@ -132,7 +132,7 @@ static inline void query_reset_and_bind(Query_t *query, int argc, VALUE * argv) if (!query->stmt) prepare_single_stmt(DB_GVL_MODE(query), query->sqlite3_db, &query->stmt, query->sql); - TRACE_SQL(query->db_struct, query->sql); + Database_issue_query(query->db_struct, query->sql); sqlite3_reset(query->stmt); query->eof = 0; @@ -160,7 +160,7 @@ VALUE Query_reset(VALUE self) { if (query->closed) rb_raise(cError, "Query is closed"); query_reset(query); - TRACE_SQL(query->db_struct, query->sql); + Database_issue_query(query->db_struct, query->sql); return self; } @@ -216,6 +216,7 @@ static inline VALUE Query_perform_next(VALUE self, int max_rows, safe_query_impl query_ctx ctx = QUERY_CTX( self, + query->sql, query->db_struct, query->stmt, Qnil, @@ -388,6 +389,7 @@ VALUE Query_batch_execute(VALUE self, VALUE parameters) { query_ctx ctx = QUERY_CTX( self, + query->sql, query->db_struct, query->stmt, parameters, @@ -436,6 +438,7 @@ VALUE Query_batch_query(VALUE self, VALUE parameters) { query_ctx ctx = QUERY_CTX( self, + query->sql, query->db_struct, query->stmt, parameters, diff --git a/test/test_database.rb b/test/test_database.rb index 01f5da6..9950884 100644 --- a/test/test_database.rb +++ b/test/test_database.rb @@ -1297,6 +1297,60 @@ def test_progress_handler_simple assert_in_range 2..4, buf.size end + def test_progress_handler_normal_mode + db = Extralite::Database.new(':memory:') + + count = 0 + db.on_progress(1, tick: 1) { count += 1 } + 10.times { db.query('select 1 as a') } + assert_equal 50, count + + count = 0 + db.on_progress(10, tick: 1) { count += 1 } + 10.times { db.query('select 1 as a') } + assert_equal 5, count + end + + def test_progress_handler_at_least_once_mode + db = Extralite::Database.new(':memory:') + + count = 0 + db.on_progress(1, tick: 1, max_calls: -1) { count += 1 } + 10.times { db.query('select 1 as a') } + assert_equal 50 + 10, count + + count = 0 + db.on_progress(10, tick: 1, max_calls: -1) { count += 1 } + 10.times { db.query('select 1 as a') } + assert_equal 5 + 10, count + end + + def test_progress_handler_once_mode + db = Extralite::Database.new(':memory:') + + count = 0 + db.on_progress(1, tick: 1, max_calls: 1) { count += 1 } + 10.times { db.query('select 1 as a') } + assert_equal 10, count + + count = 0 + db.on_progress(10, tick: 1, max_calls: 1) { count += 1 } + 10.times { db.query('select 1 as a') } + assert_equal 10, count + end + + def test_progress_handler_once_mode_batch_query + db = Extralite::Database.new(':memory:') + + count = 0 + db.on_progress(1, tick: 1, max_calls: 1) { count += 1 } + db.batch_query('select ?', 1..10) + assert_equal 10, count + + db.batch_query('select ?', 1..3) + assert_equal 13, count + end + LONG_QUERY = <<~SQL WITH RECURSIVE fibo (curr, next) From 2ce6ed57b16153f9aa5596fd519d216ea435bc9f Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Mon, 5 Feb 2024 11:56:17 +0200 Subject: [PATCH 02/11] Improve on_progress implementation - Add tick parameter. - Add mode: normal, once, at least once The tick parameter is used to control the granularity of the progress handler. This is the value passed to sqlite3_progress_handler. The period parameter is used to control how often the progress handler proc is called. So, a period of 1000 and a tick of 10 will cause the internal progress handler callback to be called every 10 SQLite VM instructions, but the proc will be called only every 1000 (cumulative) VM instructions. The different modes: - normal: the progress handler proc is called only on SQLite progress. - once: the progress handler proc is called only when preparing to run a query. - at least once: the progress handler proc is when preparing to run a query, and on SQLite progress. --- ext/extralite/database.c | 29 ++++++++------- ext/extralite/extralite.h | 2 ++ ext/extralite/query.c | 4 --- lib/extralite.rb | 2 +- test/test_database.rb | 75 ++++++++++++++++++++++++++++----------- 5 files changed, 74 insertions(+), 38 deletions(-) diff --git a/ext/extralite/database.c b/ext/extralite/database.c index 4a3dd2c..e6566c2 100644 --- a/ext/extralite/database.c +++ b/ext/extralite/database.c @@ -21,7 +21,10 @@ ID ID_strip; ID ID_to_s; ID ID_track; +VALUE SYM_at_least_once; VALUE SYM_gvl_release_threshold; +VALUE SYM_once; +VALUE SYM_normal; VALUE SYM_pragma; VALUE SYM_read_only; VALUE SYM_wal; @@ -59,6 +62,9 @@ static const rb_data_type_t Database_type = { static VALUE Database_allocate(VALUE klass) { Database_t *db = ALLOC(Database_t); db->sqlite3_db = NULL; + db->trace_proc = Qnil; + db->progress_handler.proc = Qnil; + db->progress_handler.mode = PROGRESS_NONE; return TypedData_Wrap_Struct(klass, &Database_type, db); } @@ -1043,15 +1049,11 @@ void Database_reset_progress_handler(VALUE self, Database_t *db) { sqlite3_busy_handler(db->sqlite3_db, NULL, NULL); } -static inline enum progress_handler_mode max_calls_to_progress_mode(VALUE max_calls) { - switch (NUM2INT(max_calls)) { - case 1: - return PROGRESS_ONCE; - case -1: - return PROGRESS_AT_LEAST_ONCE; - default: - return PROGRESS_NORMAL; - } +static inline enum progress_handler_mode symbol_to_progress_mode(VALUE mode) { + if (mode == SYM_at_least_once) return PROGRESS_AT_LEAST_ONCE; + if (mode == SYM_once) return PROGRESS_ONCE; + if (mode == SYM_normal) return PROGRESS_NORMAL; + rb_raise(eArgumentError, "Invalid progress handler mode"); } inline void Database_issue_query(Database_t *db, VALUE sql) { @@ -1135,12 +1137,12 @@ VALUE Database_on_progress(int argc, VALUE *argv, VALUE self) { if (!NIL_P(opt)) { if (!kw_ids[0]) { CONST_ID(kw_ids[0], "tick"); - CONST_ID(kw_ids[1], "max_calls"); + CONST_ID(kw_ids[1], "mode"); } rb_get_kwargs(opt, kw_ids, 0, 2, kw_args); if (kw_args[0] != Qundef) { tick_int = NUM2INT(kw_args[0]); } - if (kw_args[1] != Qundef) { mode = max_calls_to_progress_mode(kw_args[1]); } + if (kw_args[1] != Qundef) { mode = symbol_to_progress_mode(kw_args[1]); } } if (tick_int > period_int) tick_int = period_int; @@ -1351,9 +1353,12 @@ void Init_ExtraliteDatabase(void) { ID_to_s = rb_intern("to_s"); ID_track = rb_intern("track"); + SYM_at_least_once = ID2SYM(rb_intern("at_least_once")); SYM_gvl_release_threshold = ID2SYM(rb_intern("gvl_release_threshold")); - SYM_read_only = ID2SYM(rb_intern("read_only")); + SYM_once = ID2SYM(rb_intern("once")); + SYM_normal = ID2SYM(rb_intern("normal")); SYM_pragma = ID2SYM(rb_intern("pragma")); + SYM_read_only = ID2SYM(rb_intern("read_only")); SYM_wal = ID2SYM(rb_intern("wal")); rb_gc_register_mark_object(SYM_gvl_release_threshold); diff --git a/ext/extralite/extralite.h b/ext/extralite/extralite.h index 98346f1..a4c6438 100644 --- a/ext/extralite/extralite.h +++ b/ext/extralite/extralite.h @@ -17,6 +17,8 @@ VALUE s = rb_funcall(obj, rb_intern("inspect"), 0); \ printf(": %s\n", StringValueCStr(s)); \ } +#define CALLER() rb_funcall(rb_mKernel, rb_intern("caller"), 0) +#define TRACE_CALLER() INSPECT("caller: ", CALLER()) #define SAFE(f) (VALUE (*)(VALUE))(f) diff --git a/ext/extralite/query.c b/ext/extralite/query.c index 652ff0d..35c032c 100644 --- a/ext/extralite/query.c +++ b/ext/extralite/query.c @@ -131,9 +131,7 @@ static inline void query_reset(Query_t *query) { static inline void query_reset_and_bind(Query_t *query, int argc, VALUE * argv) { if (!query->stmt) prepare_single_stmt(DB_GVL_MODE(query), query->sqlite3_db, &query->stmt, query->sql); - Database_issue_query(query->db_struct, query->sql); - sqlite3_reset(query->stmt); query->eof = 0; if (argc > 0) { @@ -160,8 +158,6 @@ VALUE Query_reset(VALUE self) { if (query->closed) rb_raise(cError, "Query is closed"); query_reset(query); - Database_issue_query(query->db_struct, query->sql); - return self; } diff --git a/lib/extralite.rb b/lib/extralite.rb index 5bff47a..cb2db3e 100644 --- a/lib/extralite.rb +++ b/lib/extralite.rb @@ -32,7 +32,7 @@ class ParameterError < Error # This class encapsulates an SQLite database connection. class Database # @!visibility private - TABLES_SQL = <<~SQL + TABLES_SQL = (<<~SQL).freeze SELECT name FROM %s.sqlite_master WHERE type ='table' AND name NOT LIKE 'sqlite_%%'; diff --git a/test/test_database.rb b/test/test_database.rb index 9950884..e1a9957 100644 --- a/test/test_database.rb +++ b/test/test_database.rb @@ -1315,12 +1315,12 @@ def test_progress_handler_at_least_once_mode db = Extralite::Database.new(':memory:') count = 0 - db.on_progress(1, tick: 1, max_calls: -1) { count += 1 } + db.on_progress(1, tick: 1, mode: :at_least_once) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal 50 + 10, count count = 0 - db.on_progress(10, tick: 1, max_calls: -1) { count += 1 } + db.on_progress(10, tick: 1, mode: :at_least_once) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal 5 + 10, count end @@ -1329,21 +1329,21 @@ def test_progress_handler_once_mode db = Extralite::Database.new(':memory:') count = 0 - db.on_progress(1, tick: 1, max_calls: 1) { count += 1 } + db.on_progress(1, tick: 1, mode: :once) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal 10, count count = 0 - db.on_progress(10, tick: 1, max_calls: 1) { count += 1 } + db.on_progress(10, tick: 1, mode: :once) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal 10, count end - def test_progress_handler_once_mode_batch_query + def test_progress_handler_once_mode_with_batch_query db = Extralite::Database.new(':memory:') count = 0 - db.on_progress(1, tick: 1, max_calls: 1) { count += 1 } + db.on_progress(1, tick: 1, mode: :once) { count += 1 } db.batch_query('select ?', 1..10) assert_equal 10, count @@ -1351,6 +1351,35 @@ def test_progress_handler_once_mode_batch_query assert_equal 13, count end + def test_progress_handler_once_mode_with_prepared_query + db = Extralite::Database.new(':memory:') + db.execute 'create table foo (x)' + db.batch_query('insert into foo values (?)', 1..10) + q = db.prepare('select x from foo') + + count = 0 + db.on_progress(1, tick: 1, mode: :once) { count += 1 } + + q.to_a + assert_equal 1, count + + q.reset + record_count = 0 + assert_equal 2, count + while q.next + record_count += 1 + end + assert_equal 10, record_count + assert_equal 2, count + + q.reset + assert_equal 3, count + while q.next + record_count += 1 + end + assert_equal 3, count + end + LONG_QUERY = <<~SQL WITH RECURSIVE fibo (curr, next) @@ -1539,31 +1568,35 @@ def test_ractor_share_database assert_equal 'Database is closed', ex.message end - STRESS_DB_NAME = Tempfile.new('extralite_test_ractor_stress').path - # Adapted from here: https://github.com/sparklemotion/sqlite3-ruby/pull/365/files def test_ractor_stress skip if SKIP_RACTOR_TESTS - - Ractor.make_shareable(STRESS_DB_NAME) - db = Extralite::Database.new(STRESS_DB_NAME) - db.execute('PRAGMA journal_mode=WAL') # A little slow without this - db.execute('create table stress_test (a integer primary_key, b text)') + fn = Tempfile.new('extralite_test_ractor_stress').path random = Random.new.freeze + ractors = (0..9).map do |ractor_number| - Ractor.new(random, ractor_number) do |r, n| - db_in_ractor = Extralite::Database.new(STRESS_DB_NAME) - db_in_ractor.busy_timeout = 3 + sleep 0.05 + Ractor.new(fn, random, ractor_number) do |rfn, r, n| + rdb = Extralite::Database.new(rfn) + rdb.busy_timeout = 3 + rdb.pragma(journal_mode: 'wal', synchronous: 1) + rdb.execute('create table if not exists stress_test (a integer primary_key, b text)') + changes = 0 10.times do |i| - db_in_ractor.execute('insert into stress_test(a, b) values (?, ?)', n * 100 + i, r.rand) + changes += rdb.execute('insert into stress_test(a, b) values (?, ?)', n * 100 + i, r.rand) end + Ractor.yield changes end end - ractors.each { |r| r.take } - final_check = Ractor.new do - db_in_ractor = Extralite::Database.new(STRESS_DB_NAME) - count = db_in_ractor.query_single_argv('select count(*) from stress_test') + + buf = [] + ractors.each { |r| buf << r.take } + assert_equal [10] * 10, buf + + final_check = Ractor.new(fn) do |rfn| + rdb = Extralite::Database.new(rfn, wal: true) + count = rdb.query_single_argv('select count(*) from stress_test') Ractor.yield count end count = final_check.take From 0ce09107bd187fcc874a019d8ad9ee718f57b0e1 Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Mon, 5 Feb 2024 12:18:36 +0200 Subject: [PATCH 03/11] Update docs for on_progress --- ext/extralite/database.c | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/ext/extralite/database.c b/ext/extralite/database.c index e6566c2..8037a63 100644 --- a/ext/extralite/database.c +++ b/ext/extralite/database.c @@ -1075,6 +1075,24 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { * machine instructions that are evaluated between successive invocations of the * progress handler. A period of less than 1 removes the progress handler. * + * The optional `tick` parameter specifies the granularity of how often the + * progress handler is called. By default the tick value is 10, which means that + * Extralite's underlying progress callback will be called every 10 SQLite VM + * instructions. The given progress proc, however, will be only called every + * `period` (cumulative) VM instructions. This allows the progress handler to + * work correctly also when running simple queries that don't include many + * VM instructions. + * + * The optional `mode` parameter controls the progress handler mode, which is + * one of the following: + * + * - `:normal` (default): the progress handler proc is invoked on query + * progress. + * - `:once`: the progress handler proc is invoked only once, when preparing the + * query. + * - `:at_least_once`: the progress handler proc is invoked when prearing the + * query, and on query progress. + * * The progress handler is called also when the database is busy. This lets the * application perform work while waiting for the database to become unlocked, * or implement a timeout. Note that setting the database's busy_timeout _after_ @@ -1085,34 +1103,37 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { * -1, which means that the GVL will not be released at all when preparing or * running queries. It is the application's responsibility to let other threads * or fibers run by calling e.g. Thread.pass: - * + * * db.on_progress(1000) do * do_something_interesting * Thread.pass # let other threads run * end - * - * Note that the progress handler is set globally for the database and that + * + * Note that the progress handler is set globally for the database and that * Extralite does provide any hooks for telling which queries are currently - * running or at what time they were started. This means that you'll need - * to wrap the stock #query_xxx and #execute methods with your own code that + * running or at what time they were started. This means that you'll need to + * wrap the stock #query_xxx and #execute methods with your own code that * calculates timeouts, for example: - * + * * def setup_progress_handler * @db.on_progress(1000) do * raise TimeoutError if Time.now - @t0 >= @timeout * Thread.pass * end * end - * + * * def query(sql, *) * @t0 = Time.now * @db.query(sql, *) * end - * + * * If the gvl release threshold is set to a value equal to or larger than 0 * after setting the progress handler, the progress handler will be reset. * * @param period [Integer] progress handler period + * @param [Hash] opts progress options + * @option opts [Integer] :tick tick value (`10` by default) + * @option opts [Symbol] :mode progress handler mode (`:normal` by default) * @returns [Extralite::Database] database */ VALUE Database_on_progress(int argc, VALUE *argv, VALUE self) { From b5a286a8b6709a789b82370b95310c11a53a2fd7 Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Mon, 5 Feb 2024 12:18:48 +0200 Subject: [PATCH 04/11] Update deps, fix tests --- gemspec.rb | 8 ++++---- test/test_changeset.rb | 2 +- test/test_database.rb | 8 ++++---- test/test_extralite.rb | 2 +- test/test_iterator.rb | 2 +- test/test_query.rb | 4 ++-- test/test_sequel.rb | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/gemspec.rb b/gemspec.rb index 2cb8c6c..b51c74f 100644 --- a/gemspec.rb +++ b/gemspec.rb @@ -17,9 +17,9 @@ def common_spec(s) s.require_paths = ['lib'] s.required_ruby_version = '>= 3.0' - s.add_development_dependency 'rake-compiler', '1.1.6' - s.add_development_dependency 'minitest', '5.15.0' + s.add_development_dependency 'rake-compiler', '1.2.7' + s.add_development_dependency 'minitest', '5.21.2' s.add_development_dependency 'simplecov', '0.17.1' - s.add_development_dependency 'yard', '0.9.27' - s.add_development_dependency 'sequel', '5.51.0' + s.add_development_dependency 'yard', '0.9.34' + s.add_development_dependency 'sequel', '5.77.0' end diff --git a/test/test_changeset.rb b/test/test_changeset.rb index f82acd6..9483154 100644 --- a/test/test_changeset.rb +++ b/test/test_changeset.rb @@ -5,7 +5,7 @@ require 'date' require 'tempfile' -class ChangesetTest < MiniTest::Test +class ChangesetTest < Minitest::Test def setup @db = Extralite::Database.new(':memory:') skip if !@db.respond_to?(:track_changes) diff --git a/test/test_database.rb b/test/test_database.rb index e1a9957..14b6e31 100644 --- a/test/test_database.rb +++ b/test/test_database.rb @@ -6,7 +6,7 @@ require 'tempfile' require 'json' -class DatabaseTest < MiniTest::Test +class DatabaseTest < Minitest::Test def setup @db = Extralite::Database.new(':memory:') @db.query('create table if not exists t (x,y,z)') @@ -1017,7 +1017,7 @@ def test_prepare_ary_with_transform end end -class ScenarioTest < MiniTest::Test +class ScenarioTest < Minitest::Test def setup @fn = Tempfile.new('extralite_scenario_test').path @db = Extralite::Database.new(@fn) @@ -1141,7 +1141,7 @@ def test_database_trace end end -class BackupTest < MiniTest::Test +class BackupTest < Minitest::Test def setup @src = Extralite::Database.new(':memory:') @dst = Extralite::Database.new(':memory:') @@ -1604,7 +1604,7 @@ def test_ractor_stress end end -class DatabaseTransformTest < MiniTest::Test +class DatabaseTransformTest < Minitest::Test def setup @db = Extralite::Database.new(':memory:') @db.query('create table t (a, b, c)') diff --git a/test/test_extralite.rb b/test/test_extralite.rb index 5dad8b3..4efd673 100644 --- a/test/test_extralite.rb +++ b/test/test_extralite.rb @@ -2,7 +2,7 @@ require_relative 'helper' -class ExtraliteTest < MiniTest::Test +class ExtraliteTest < Minitest::Test def test_sqlite3_version assert_match(/^3\.\d+\.\d+$/, Extralite.sqlite3_version) end diff --git a/test/test_iterator.rb b/test/test_iterator.rb index abb1b83..7841587 100644 --- a/test/test_iterator.rb +++ b/test/test_iterator.rb @@ -2,7 +2,7 @@ require_relative 'helper' -class IteratorTest < MiniTest::Test +class IteratorTest < Minitest::Test def setup @db = Extralite::Database.new(':memory:') @db.query('create table if not exists t (x,y,z)') diff --git a/test/test_query.rb b/test/test_query.rb index 7e21f9f..82a2a81 100644 --- a/test/test_query.rb +++ b/test/test_query.rb @@ -4,7 +4,7 @@ require 'date' require 'json' -class QueryTest < MiniTest::Test +class QueryTest < Minitest::Test def setup @db = Extralite::Database.new(':memory:') @db.query('create table if not exists t (x,y,z)') @@ -1007,7 +1007,7 @@ def test_query_dup_with_transform end end -class QueryTransformTest < MiniTest::Test +class QueryTransformTest < Minitest::Test def setup @db = Extralite::Database.new(':memory:') @db.query('create table t (a, b, c)') diff --git a/test/test_sequel.rb b/test/test_sequel.rb index 50cb8a0..9634598 100644 --- a/test/test_sequel.rb +++ b/test/test_sequel.rb @@ -3,7 +3,7 @@ require_relative 'helper' require 'sequel' -class SequelExtraliteTest < MiniTest::Test +class SequelExtraliteTest < Minitest::Test def setup @db = Sequel.connect('extralite::memory:') @db.create_table :items do From 254c7254c10c21d28aa0c02d93bb110cd094c0f0 Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Mon, 5 Feb 2024 17:38:00 +0200 Subject: [PATCH 05/11] Fix instruction count in on_progress tests --- test/test_database.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/test_database.rb b/test/test_database.rb index 14b6e31..e6773f6 100644 --- a/test/test_database.rb +++ b/test/test_database.rb @@ -1300,10 +1300,16 @@ def test_progress_handler_simple def test_progress_handler_normal_mode db = Extralite::Database.new(':memory:') + count = 0 + db.on_progress(1, tick: 1) { count += 1 } + db.query('select 1 as a') + assert count > 0 + base_count = count + count = 0 db.on_progress(1, tick: 1) { count += 1 } 10.times { db.query('select 1 as a') } - assert_equal 50, count + assert_equal base_count * 10, count count = 0 db.on_progress(10, tick: 1) { count += 1 } @@ -1314,10 +1320,16 @@ def test_progress_handler_normal_mode def test_progress_handler_at_least_once_mode db = Extralite::Database.new(':memory:') + count = 0 + db.on_progress(1, tick: 1) { count += 1 } + db.query('select 1 as a') + assert count > 0 + base_count = count + count = 0 db.on_progress(1, tick: 1, mode: :at_least_once) { count += 1 } 10.times { db.query('select 1 as a') } - assert_equal 50 + 10, count + assert_equal base_count * 10 + 10, count count = 0 db.on_progress(10, tick: 1, mode: :at_least_once) { count += 1 } From 215ecb0e7b274bfaae4f6fef99e287dacafb3f8c Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Mon, 5 Feb 2024 17:40:06 +0200 Subject: [PATCH 06/11] Fix on_progress tests (again) --- test/test_database.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_database.rb b/test/test_database.rb index e6773f6..8d82af9 100644 --- a/test/test_database.rb +++ b/test/test_database.rb @@ -1314,7 +1314,7 @@ def test_progress_handler_normal_mode count = 0 db.on_progress(10, tick: 1) { count += 1 } 10.times { db.query('select 1 as a') } - assert_equal 5, count + assert_equal base_count, count end def test_progress_handler_at_least_once_mode @@ -1334,7 +1334,7 @@ def test_progress_handler_at_least_once_mode count = 0 db.on_progress(10, tick: 1, mode: :at_least_once) { count += 1 } 10.times { db.query('select 1 as a') } - assert_equal 5 + 10, count + assert_equal base_count + 10, count end def test_progress_handler_once_mode From 632ea80bd3872e19be684dc84ff7eb3823a921a0 Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Mon, 5 Feb 2024 17:46:41 +0200 Subject: [PATCH 07/11] Fix compiler warnings and errors --- ext/extralite/changeset.c | 6 +++--- ext/extralite/database.c | 3 ++- ext/extralite/extralite.h | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ext/extralite/changeset.c b/ext/extralite/changeset.c index ac02888..2413ea5 100644 --- a/ext/extralite/changeset.c +++ b/ext/extralite/changeset.c @@ -186,13 +186,13 @@ static inline VALUE convert_value(sqlite3_value *value) { case SQLITE_BLOB: { int len = sqlite3_value_bytes(value); - void *blob = sqlite3_value_blob(value); + const void *blob = sqlite3_value_blob(value); return rb_str_new(blob, len); } case SQLITE_TEXT: { int len = sqlite3_value_bytes(value); - void *text = sqlite3_value_text(value); + const void *text = sqlite3_value_text(value); return rb_enc_str_new(text, len, UTF8_ENCODING); } default: @@ -339,7 +339,7 @@ VALUE Changeset_to_a(VALUE self) { // copied from: https://sqlite.org/sessionintro.html static int xConflict(void *pCtx, int eConflict, sqlite3_changeset_iter *pIter){ - int ret = (long)pCtx; + int ret = (int)pCtx; return ret; } diff --git a/ext/extralite/database.c b/ext/extralite/database.c index 8037a63..e3fe1ea 100644 --- a/ext/extralite/database.c +++ b/ext/extralite/database.c @@ -1063,7 +1063,8 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { case PROGRESS_ONCE: rb_funcall(db->progress_handler.proc, ID_call, 0); default: - // do nothing + ; // do nothing + } } diff --git a/ext/extralite/extralite.h b/ext/extralite/extralite.h index a4c6438..83f19d9 100644 --- a/ext/extralite/extralite.h +++ b/ext/extralite/extralite.h @@ -100,7 +100,7 @@ typedef struct { #ifdef EXTRALITE_ENABLE_CHANGESET typedef struct { - int changeset_len; + long changeset_len; void *changeset_ptr; } Changeset_t; #endif From d747dd0f184b9939ff4c37dd041fa4dba8efffdd Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Tue, 6 Feb 2024 08:00:45 +0200 Subject: [PATCH 08/11] Make period an optional parameter, add more tests --- ext/extralite/database.c | 68 ++++++++++++++++++++-------------- test/test_database.rb | 80 ++++++++++++++++++++++++++++++++-------- 2 files changed, 105 insertions(+), 43 deletions(-) diff --git a/ext/extralite/database.c b/ext/extralite/database.c index e3fe1ea..3ce04d4 100644 --- a/ext/extralite/database.c +++ b/ext/extralite/database.c @@ -24,6 +24,7 @@ ID ID_track; VALUE SYM_at_least_once; VALUE SYM_gvl_release_threshold; VALUE SYM_once; +VALUE SYM_none; VALUE SYM_normal; VALUE SYM_pragma; VALUE SYM_read_only; @@ -1053,6 +1054,7 @@ static inline enum progress_handler_mode symbol_to_progress_mode(VALUE mode) { if (mode == SYM_at_least_once) return PROGRESS_AT_LEAST_ONCE; if (mode == SYM_once) return PROGRESS_ONCE; if (mode == SYM_normal) return PROGRESS_NORMAL; + if (mode == SYM_none) return PROGRESS_NONE; rb_raise(eArgumentError, "Invalid progress handler mode"); } @@ -1072,20 +1074,22 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { * while a query is running. This method can be used to support switching * between fibers and threads or implementing timeouts for running queries. * - * The given period parameter specifies the approximate number of SQLite virtual - * machine instructions that are evaluated between successive invocations of the - * progress handler. A period of less than 1 removes the progress handler. + * The `period` parameter specifies the approximate number of SQLite + * virtual machine instructions that are evaluated between successive + * invocations of the progress handler. A period of less than 1 removes the + * progress handler. The default period value is 1000. * * The optional `tick` parameter specifies the granularity of how often the - * progress handler is called. By default the tick value is 10, which means that + * progress handler is called. The default tick value is 10, which means that * Extralite's underlying progress callback will be called every 10 SQLite VM * instructions. The given progress proc, however, will be only called every * `period` (cumulative) VM instructions. This allows the progress handler to * work correctly also when running simple queries that don't include many - * VM instructions. + * VM instructions. If the `tick` value is greater than the period value it is + * automatically capped to the period value. * - * The optional `mode` parameter controls the progress handler mode, which is - * one of the following: + * The `mode` parameter controls the progress handler mode, which is one of the + * following: * * - `:normal` (default): the progress handler proc is invoked on query * progress. @@ -1098,14 +1102,15 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { * application perform work while waiting for the database to become unlocked, * or implement a timeout. Note that setting the database's busy_timeout _after_ * setting a progress handler may lead to undefined behaviour in a concurrent - * application. + * application. When busy, the progress handler proc is passed `true` as the + * first argument. * * When the progress handler is set, the gvl release threshold value is set to * -1, which means that the GVL will not be released at all when preparing or * running queries. It is the application's responsibility to let other threads * or fibers run by calling e.g. Thread.pass: * - * db.on_progress(1000) do + * db.on_progress do * do_something_interesting * Thread.pass # let other threads run * end @@ -1117,7 +1122,7 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { * calculates timeouts, for example: * * def setup_progress_handler - * @db.on_progress(1000) do + * @db.on_progress do * raise TimeoutError if Time.now - @t0 >= @timeout * Thread.pass * end @@ -1133,40 +1138,42 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { * * @param period [Integer] progress handler period * @param [Hash] opts progress options + * @option opts [Integer] :period period value (`1000` by default) * @option opts [Integer] :tick tick value (`10` by default) * @option opts [Symbol] :mode progress handler mode (`:normal` by default) * @returns [Extralite::Database] database */ VALUE Database_on_progress(int argc, VALUE *argv, VALUE self) { Database_t *db = self_to_open_database(self); - VALUE period; VALUE opt; - static ID kw_ids[2]; - VALUE kw_args[2]; + static ID kw_ids[3]; + VALUE kw_args[3]; - rb_scan_args(argc, argv, "10:", &period, &opt); - int period_int = NUM2INT(period); - - if (!rb_block_given_p() || period_int == 0) { - Database_reset_progress_handler(self, db); - db->gvl_release_threshold = DEFAULT_GVL_RELEASE_THRESHOLD; - return self; - } + rb_scan_args(argc, argv, "00:", &opt); + int period_int = 1000; int tick_int = 10; enum progress_handler_mode mode = PROGRESS_NORMAL; if (!NIL_P(opt)) { if (!kw_ids[0]) { - CONST_ID(kw_ids[0], "tick"); - CONST_ID(kw_ids[1], "mode"); + CONST_ID(kw_ids[0], "period"); + CONST_ID(kw_ids[1], "tick"); + CONST_ID(kw_ids[2], "mode"); } - rb_get_kwargs(opt, kw_ids, 0, 2, kw_args); - if (kw_args[0] != Qundef) { tick_int = NUM2INT(kw_args[0]); } - if (kw_args[1] != Qundef) { mode = symbol_to_progress_mode(kw_args[1]); } + rb_get_kwargs(opt, kw_ids, 0, 3, kw_args); + if (kw_args[0] != Qundef) { period_int = NUM2INT(kw_args[0]); } + if (kw_args[1] != Qundef) { tick_int = NUM2INT(kw_args[1]); } + if (kw_args[2] != Qundef) { mode = symbol_to_progress_mode(kw_args[2]); } + if (tick_int > period_int) tick_int = period_int; + } + + if (!rb_block_given_p() || mode == PROGRESS_NONE || period_int == 0) { + Database_reset_progress_handler(self, db); + db->gvl_release_threshold = DEFAULT_GVL_RELEASE_THRESHOLD; + return self; } - if (tick_int > period_int) tick_int = period_int; db->gvl_release_threshold = -1; db->progress_handler.mode = mode; @@ -1378,14 +1385,19 @@ void Init_ExtraliteDatabase(void) { SYM_at_least_once = ID2SYM(rb_intern("at_least_once")); SYM_gvl_release_threshold = ID2SYM(rb_intern("gvl_release_threshold")); SYM_once = ID2SYM(rb_intern("once")); + SYM_none = ID2SYM(rb_intern("none")); SYM_normal = ID2SYM(rb_intern("normal")); SYM_pragma = ID2SYM(rb_intern("pragma")); SYM_read_only = ID2SYM(rb_intern("read_only")); SYM_wal = ID2SYM(rb_intern("wal")); + rb_gc_register_mark_object(SYM_at_least_once); rb_gc_register_mark_object(SYM_gvl_release_threshold); - rb_gc_register_mark_object(SYM_read_only); + rb_gc_register_mark_object(SYM_once); + rb_gc_register_mark_object(SYM_none); + rb_gc_register_mark_object(SYM_normal); rb_gc_register_mark_object(SYM_pragma); + rb_gc_register_mark_object(SYM_read_only); rb_gc_register_mark_object(SYM_wal); UTF8_ENCODING = rb_utf8_encoding(); diff --git a/test/test_database.rb b/test/test_database.rb index 8d82af9..0d09bfe 100644 --- a/test/test_database.rb +++ b/test/test_database.rb @@ -1283,14 +1283,14 @@ def test_progress_handler_simple db = Extralite::Database.new(':memory:') buf = [] - db.on_progress(1) { buf << :progress } + db.on_progress(period: 1) { buf << :progress } result = db.query_single('select 1 as a, 2 as b, 3 as c') assert_equal({ a: 1, b: 2, c: 3 }, result) assert_in_range 5..7, buf.size buf = [] - db.on_progress(2) { buf << :progress } + db.on_progress(period: 2) { buf << :progress } result = db.query_single('select 1 as a, 2 as b, 3 as c') assert_equal({ a: 1, b: 2, c: 3 }, result) @@ -1301,18 +1301,18 @@ def test_progress_handler_normal_mode db = Extralite::Database.new(':memory:') count = 0 - db.on_progress(1, tick: 1) { count += 1 } + db.on_progress(period: 1) { count += 1 } db.query('select 1 as a') assert count > 0 base_count = count count = 0 - db.on_progress(1, tick: 1) { count += 1 } + db.on_progress(period: 1) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal base_count * 10, count count = 0 - db.on_progress(10, tick: 1) { count += 1 } + db.on_progress(period: 10, tick: 1) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal base_count, count end @@ -1321,18 +1321,18 @@ def test_progress_handler_at_least_once_mode db = Extralite::Database.new(':memory:') count = 0 - db.on_progress(1, tick: 1) { count += 1 } + db.on_progress(period: 1) { count += 1 } db.query('select 1 as a') assert count > 0 base_count = count count = 0 - db.on_progress(1, tick: 1, mode: :at_least_once) { count += 1 } + db.on_progress(period: 1, mode: :at_least_once) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal base_count * 10 + 10, count count = 0 - db.on_progress(10, tick: 1, mode: :at_least_once) { count += 1 } + db.on_progress(period: 10, tick: 1, mode: :at_least_once) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal base_count + 10, count end @@ -1341,12 +1341,17 @@ def test_progress_handler_once_mode db = Extralite::Database.new(':memory:') count = 0 - db.on_progress(1, tick: 1, mode: :once) { count += 1 } + db.on_progress(mode: :once) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal 10, count count = 0 - db.on_progress(10, tick: 1, mode: :once) { count += 1 } + db.on_progress(period: 1, mode: :once) { count += 1 } + 10.times { db.query('select 1 as a') } + assert_equal 10, count + + count = 0 + db.on_progress(period: 10, tick: 1, mode: :once) { count += 1 } 10.times { db.query('select 1 as a') } assert_equal 10, count end @@ -1355,7 +1360,7 @@ def test_progress_handler_once_mode_with_batch_query db = Extralite::Database.new(':memory:') count = 0 - db.on_progress(1, tick: 1, mode: :once) { count += 1 } + db.on_progress(period: 1, mode: :once) { count += 1 } db.batch_query('select ?', 1..10) assert_equal 10, count @@ -1363,6 +1368,51 @@ def test_progress_handler_once_mode_with_batch_query assert_equal 13, count end + def test_progress_handler_reset + db = Extralite::Database.new(':memory:') + + count = 0 + set_progress = -> { + count = 0 + db.on_progress(mode: :once) { count += 1 } + } + + set_progress.() + 10.times { db.query('select 1 as a') } + assert_equal 10, count + + count = 0 + db.on_progress(mode: :none) + 10.times { db.query('select 1 as a') } + assert_equal 0, count + + set_progress.() + 10.times { db.query('select 1 as a') } + assert_equal 10, count + + count = 0 + db.on_progress(period: 0) { foo } + 10.times { db.query('select 1 as a') } + assert_equal 0, count + + set_progress.() + 10.times { db.query('select 1 as a') } + assert_equal 10, count + + count = 0 + db.on_progress + 10.times { db.query('select 1 as a') } + assert_equal 0, count + end + + def test_progress_handler_invalid_arg + db = Extralite::Database.new(':memory:') + + assert_raises(TypeError) { db.on_progress(period: :foo) } + assert_raises(TypeError) { db.on_progress(tick: :foo) } + assert_raises(ArgumentError) { db.on_progress(mode: :foo) } + end + def test_progress_handler_once_mode_with_prepared_query db = Extralite::Database.new(':memory:') db.execute 'create table foo (x)' @@ -1370,7 +1420,7 @@ def test_progress_handler_once_mode_with_prepared_query q = db.prepare('select x from foo') count = 0 - db.on_progress(1, tick: 1, mode: :once) { count += 1 } + db.on_progress(period: 1, mode: :once) { count += 1 } q.to_a assert_equal 1, count @@ -1406,7 +1456,7 @@ def test_progress_handler_once_mode_with_prepared_query def test_progress_handler_timeout_interrupt db = Extralite::Database.new(':memory:') t0 = Time.now - db.on_progress(1000) do + db.on_progress do Thread.pass db.interrupt if Time.now - t0 >= 0.2 end @@ -1448,7 +1498,7 @@ class CustomTimeoutError < RuntimeError def test_progress_handler_timeout_raise db = Extralite::Database.new(':memory:') t0 = Time.now - db.on_progress(1000) do + db.on_progress do Thread.pass raise CustomTimeoutError if Time.now - t0 >= 0.2 end @@ -1493,7 +1543,7 @@ def test_progress_handler_busy_timeout assert_raises(Extralite::BusyError) { db2.query('begin exclusive') } t0 = Time.now - db2.on_progress(1000) do + db2.on_progress do Thread.pass raise CustomTimeoutError if Time.now - t0 >= 0.2 end From c486b63dfe4cfaf7e87679b8839822bc0e1a9645 Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Wed, 7 Feb 2024 10:10:36 +0200 Subject: [PATCH 09/11] Add support for global progress handler settings --- ext/extralite/database.c | 196 ++++++++++++++++++++++++++++---------- ext/extralite/extralite.h | 4 +- test/test_database.rb | 23 +++++ 3 files changed, 170 insertions(+), 53 deletions(-) diff --git a/ext/extralite/database.c b/ext/extralite/database.c index 3ce04d4..07f30b6 100644 --- a/ext/extralite/database.c +++ b/ext/extralite/database.c @@ -30,6 +30,15 @@ VALUE SYM_pragma; VALUE SYM_read_only; VALUE SYM_wal; +struct progress_handler global_progress_handler = { + .mode = PROGRESS_NONE, + .proc = Qnil, + .period = DEFAULT_PROGRESS_HANDLER_PERIOD, + .tick = DEFAULT_PROGRESS_HANDLER_TICK, + .tick_count = 0, + .call_count = 0 +}; + #define DB_GVL_MODE(db) Database_prepare_gvl_mode(db) static size_t Database_size(const void *ptr) { @@ -129,6 +138,25 @@ void Database_apply_opts(VALUE self, Database_t *db, VALUE opts) { } } +int Database_progress_handler(void *ptr) { + Database_t *db = (Database_t *)ptr; + db->progress_handler.tick_count += db->progress_handler.tick; + if (db->progress_handler.tick_count < db->progress_handler.period) + goto done; + + db->progress_handler.tick_count -= db->progress_handler.period; + db->progress_handler.call_count += 1; + rb_funcall(db->progress_handler.proc, ID_call, 0); +done: + return 0; +} + +int Database_busy_handler(void *ptr, int v) { + Database_t *db = (Database_t *)ptr; + rb_funcall(db->progress_handler.proc, ID_call, 1, Qtrue); + return 1; +} + /* Initializes a new SQLite database with the given path and options: * * - `:gvl_release_threshold` (`Integer`): sets the GVL release threshold (see @@ -180,11 +208,17 @@ VALUE Database_initialize(int argc, VALUE *argv, VALUE self) { db->trace_proc = Qnil; db->gvl_release_threshold = DEFAULT_GVL_RELEASE_THRESHOLD; - db->progress_handler.mode = PROGRESS_NONE; - db->progress_handler.proc = Qnil; + db->progress_handler = global_progress_handler; + db->progress_handler.tick_count = 0; + db->progress_handler.call_count = 0; + if (db->progress_handler.mode != PROGRESS_NONE) { + db->gvl_release_threshold = -1; + if (db->progress_handler.mode != PROGRESS_ONCE) + sqlite3_progress_handler(db->sqlite3_db, db->progress_handler.tick, &Database_progress_handler, db); + sqlite3_busy_handler(db->sqlite3_db, &Database_busy_handler, db); + } if (!NIL_P(opts)) Database_apply_opts(self, db, opts); - return Qnil; } @@ -1024,25 +1058,6 @@ VALUE Database_track_changes(int argc, VALUE *argv, VALUE self) { } #endif -int Database_progress_handler(void *ptr) { - Database_t *db = (Database_t *)ptr; - db->progress_handler.tick_count += db->progress_handler.tick; - if (db->progress_handler.tick_count < db->progress_handler.period) - goto done; - - db->progress_handler.tick_count -= db->progress_handler.period; - db->progress_handler.call_count += 1; - rb_funcall(db->progress_handler.proc, ID_call, 0); -done: - return 0; -} - -int Database_busy_handler(void *ptr, int v) { - Database_t *db = (Database_t *)ptr; - rb_funcall(db->progress_handler.proc, ID_call, 1, Qtrue); - return 1; -} - void Database_reset_progress_handler(VALUE self, Database_t *db) { db->progress_handler.mode = PROGRESS_NONE; RB_OBJ_WRITE(self, &db->progress_handler.proc, Qnil); @@ -1070,6 +1085,35 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { } } +struct progress_handler parse_progress_handler_opts(VALUE opts) { + static ID kw_ids[3]; + VALUE kw_args[3]; + struct progress_handler prog = { + .mode = rb_block_given_p() ? PROGRESS_NORMAL : PROGRESS_NONE, + .proc = rb_block_given_p() ? rb_block_proc() : Qnil, + .period = DEFAULT_PROGRESS_HANDLER_PERIOD, + .tick = DEFAULT_PROGRESS_HANDLER_TICK + }; + + if (!NIL_P(opts)) { + if (!kw_ids[0]) { + CONST_ID(kw_ids[0], "period"); + CONST_ID(kw_ids[1], "tick"); + CONST_ID(kw_ids[2], "mode"); + } + + rb_get_kwargs(opts, kw_ids, 0, 3, kw_args); + if (kw_args[0] != Qundef) { prog.period = NUM2INT(kw_args[0]); } + if (kw_args[1] != Qundef) { prog.tick = NUM2INT(kw_args[1]); } + if (kw_args[2] != Qundef) { prog.mode = symbol_to_progress_mode(kw_args[2]); } + if (prog.tick > prog.period) prog.tick = prog.period; + } + if (NIL_P(prog.proc) || (prog.period <= 0)) prog.mode = PROGRESS_NONE; + if (prog.mode == PROGRESS_NONE) prog.proc = Qnil; + + return prog; +} + /* Installs or removes a progress handler that will be executed periodically * while a query is running. This method can be used to support switching * between fibers and threads or implementing timeouts for running queries. @@ -1145,52 +1189,99 @@ inline void Database_issue_query(Database_t *db, VALUE sql) { */ VALUE Database_on_progress(int argc, VALUE *argv, VALUE self) { Database_t *db = self_to_open_database(self); - VALUE opt; - static ID kw_ids[3]; - VALUE kw_args[3]; + VALUE opts; + struct progress_handler prog; - rb_scan_args(argc, argv, "00:", &opt); + rb_scan_args(argc, argv, "00:", &opts); + prog = parse_progress_handler_opts(opts); - int period_int = 1000; - int tick_int = 10; - enum progress_handler_mode mode = PROGRESS_NORMAL; - - if (!NIL_P(opt)) { - if (!kw_ids[0]) { - CONST_ID(kw_ids[0], "period"); - CONST_ID(kw_ids[1], "tick"); - CONST_ID(kw_ids[2], "mode"); - } - - rb_get_kwargs(opt, kw_ids, 0, 3, kw_args); - if (kw_args[0] != Qundef) { period_int = NUM2INT(kw_args[0]); } - if (kw_args[1] != Qundef) { tick_int = NUM2INT(kw_args[1]); } - if (kw_args[2] != Qundef) { mode = symbol_to_progress_mode(kw_args[2]); } - if (tick_int > period_int) tick_int = period_int; - } - - if (!rb_block_given_p() || mode == PROGRESS_NONE || period_int == 0) { + if (prog.mode == PROGRESS_NONE) { Database_reset_progress_handler(self, db); db->gvl_release_threshold = DEFAULT_GVL_RELEASE_THRESHOLD; return self; } db->gvl_release_threshold = -1; - db->progress_handler.mode = mode; - RB_OBJ_WRITE(self, &db->progress_handler.proc, rb_block_proc()); - db->progress_handler.period = period_int; - db->progress_handler.tick = tick_int; + db->progress_handler.mode = prog.mode; + RB_OBJ_WRITE(self, &db->progress_handler.proc, prog.proc); + db->progress_handler.period = prog.period; + db->progress_handler.tick = prog.tick; db->progress_handler.tick_count = 0; db->progress_handler.call_count = 0; // The PROGRESS_ONCE mode works by invoking the progress handler proc exactly // once, before iterating over the result set, so in that mode we don't // actually need to set the progress handler at the sqlite level. - if (mode != PROGRESS_ONCE) - sqlite3_progress_handler(db->sqlite3_db, tick_int, &Database_progress_handler, db); + if (prog.mode != PROGRESS_ONCE) + sqlite3_progress_handler(db->sqlite3_db, prog.tick, &Database_progress_handler, db); + if (prog.mode != PROGRESS_NONE) + sqlite3_busy_handler(db->sqlite3_db, &Database_busy_handler, db); + + return self; +} - sqlite3_busy_handler(db->sqlite3_db, &Database_busy_handler, db); +/* Installs or removes a global progress handler that will be executed + * periodically while a query is running. This method can be used to support + * switching between fibers and threads or implementing timeouts for running + * queries. + * + * This method sets the progress handler settings and behaviour for all + * subsequently created `Database` instances. Calling this method will have no + * effect on already existing `Database` instances + * + * The `period` parameter specifies the approximate number of SQLite + * virtual machine instructions that are evaluated between successive + * invocations of the progress handler. A period of less than 1 removes the + * progress handler. The default period value is 1000. + * + * The optional `tick` parameter specifies the granularity of how often the + * progress handler is called. The default tick value is 10, which means that + * Extralite's underlying progress callback will be called every 10 SQLite VM + * instructions. The given progress proc, however, will be only called every + * `period` (cumulative) VM instructions. This allows the progress handler to + * work correctly also when running simple queries that don't include many + * VM instructions. If the `tick` value is greater than the period value it is + * automatically capped to the period value. + * + * The `mode` parameter controls the progress handler mode, which is one of the + * following: + * + * - `:normal` (default): the progress handler proc is invoked on query + * progress. + * - `:once`: the progress handler proc is invoked only once, when preparing the + * query. + * - `:at_least_once`: the progress handler proc is invoked when prearing the + * query, and on query progress. + * + * The progress handler is called also when the database is busy. This lets the + * application perform work while waiting for the database to become unlocked, + * or implement a timeout. Note that setting the database's busy_timeout _after_ + * setting a progress handler may lead to undefined behaviour in a concurrent + * application. When busy, the progress handler proc is passed `true` as the + * first argument. + * + * When the progress handler is set, the gvl release threshold value is set to + * -1, which means that the GVL will not be released at all when preparing or + * running queries. It is the application's responsibility to let other threads + * or fibers run by calling e.g. Thread.pass: + * + * Extralite.on_progress do + * do_something_interesting + * Thread.pass # let other threads run + * end + * + * @param period [Integer] progress handler period + * @param [Hash] opts progress options + * @option opts [Integer] :period period value (`1000` by default) + * @option opts [Integer] :tick tick value (`10` by default) + * @option opts [Symbol] :mode progress handler mode (`:normal` by default) + * @returns [Extralite::Database] database + */ +VALUE Extralite_on_progress(int argc, VALUE *argv, VALUE self) { + VALUE opts; + rb_scan_args(argc, argv, "00:", &opts); + global_progress_handler = parse_progress_handler_opts(opts); return self; } @@ -1302,6 +1393,7 @@ void Init_ExtraliteDatabase(void) { VALUE mExtralite = rb_define_module("Extralite"); rb_define_singleton_method(mExtralite, "runtime_status", Extralite_runtime_status, -1); rb_define_singleton_method(mExtralite, "sqlite3_version", Extralite_sqlite3_version, 0); + rb_define_singleton_method(mExtralite, "on_progress", Extralite_on_progress, -1); cDatabase = rb_define_class_under(mExtralite, "Database", rb_cObject); rb_define_alloc_func(cDatabase, Database_allocate); diff --git a/ext/extralite/extralite.h b/ext/extralite/extralite.h index 83f19d9..8da8188 100644 --- a/ext/extralite/extralite.h +++ b/ext/extralite/extralite.h @@ -100,7 +100,7 @@ typedef struct { #ifdef EXTRALITE_ENABLE_CHANGESET typedef struct { - long changeset_len; + int changeset_len; void *changeset_ptr; } Changeset_t; #endif @@ -156,6 +156,8 @@ enum gvl_mode { } #define DEFAULT_GVL_RELEASE_THRESHOLD 1000 +#define DEFAULT_PROGRESS_HANDLER_PERIOD 1000 +#define DEFAULT_PROGRESS_HANDLER_TICK 10 extern rb_encoding *UTF8_ENCODING; diff --git a/test/test_database.rb b/test/test_database.rb index 0d09bfe..00ae892 100644 --- a/test/test_database.rb +++ b/test/test_database.rb @@ -1576,6 +1576,29 @@ def test_progress_handler_busy_timeout assert_equal 1, ((t1 - t0) * 5).round.to_i assert_kind_of CustomTimeoutError, err end + + def test_global_progress_handler + count = 0 + Extralite.on_progress(tick: 1, period: 1) { count += 1 } + + db = Extralite::Database.new(':memory:') + 10.times { db.query('select 1') } + refute_equal 0, count + + old_count = count + Extralite.on_progress # remove global progress handler + + # already opened db should preserve progress handler behaviour + 10.times { db.query('select 1') } + refute_equal old_count, count + + old_count = count + db2 = Extralite::Database.new(':memory:') + 10.times { db2.query('select 1') } + assert_equal old_count, count + ensure + Extralite.on_progress(mode: :none) + end end class RactorTest < Minitest::Test From dcd156ff30b92fa1f4d3057f5ea911ece01720e1 Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Wed, 7 Feb 2024 10:13:38 +0200 Subject: [PATCH 10/11] Skip ractor tests, add caveat in README --- README.md | 9 ++++++++- test/helper.rb | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af42d36..3748324 100644 --- a/README.md +++ b/README.md @@ -1016,7 +1016,14 @@ as long as the following conditions are met: ### Use with Ractors -Extralite databases can safely be used inside ractors. Note that ractors are still an experimental feature of Ruby. A ractor has the benefit of using a separate GVL from the maine one, which allows true parallelism for Ruby apps. So when you use Extralite to access SQLite databases from within a ractor, you can do so without any considerations for what's happening outside the ractor when it runs queries. +Extralite databases can safely be used inside ractors. A ractor has the benefit +of using a separate GVL from the maine one, which allows true parallelism for +Ruby apps. So when you use Extralite to access SQLite databases from within a +ractor, you can do so without any considerations for what's happening outside +the ractor when it runs queries. + +**Note**: Ractors are considered an experimental feature of Ruby. You may +encounter errors or inconsistent behaviour when using ractors. ## Advanced Usage diff --git a/test/helper.rb b/test/helper.rb index 7bf5a7f..678aed0 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -7,7 +7,8 @@ puts "sqlite3 version: #{Extralite.sqlite3_version}" IS_LINUX = RUBY_PLATFORM =~ /linux/ -SKIP_RACTOR_TESTS = !IS_LINUX || (RUBY_VERSION =~ /^3\.[01]/) +# Ractors are kinda flaky, there's no point in testing this +SKIP_RACTOR_TESTS = true #!IS_LINUX || (RUBY_VERSION =~ /^3\.[01]/) module Minitest::Assertions def assert_in_range exp_range, act From 2556692ec63e3f11d00c685453e81a3e16820986 Mon Sep 17 00:00:00 2001 From: Sharon Rosner Date: Wed, 7 Feb 2024 10:22:11 +0200 Subject: [PATCH 11/11] Update README --- README.md | 51 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3748324..6a540bb 100644 --- a/README.md +++ b/README.md @@ -877,8 +877,7 @@ that specifies the approximate number of SQLite VM instructions between successive calls to the progress handler: ```ruby -# Run progress handler every 100 SQLite VM instructions -db.on_progress(100) do +db.on_progress do check_for_timeout # Allow other threads to run Thread.pass @@ -891,7 +890,7 @@ above, calling `#interrupt` causes the query to raise a `Extralite::InterruptError` exception: ```ruby -db.on_progress(100) { db.interrupt } +db.on_progress { db.interrupt } db.query('select 1') #=> Extralite::InterruptError! ``` @@ -900,7 +899,7 @@ You can also interrupt queries in progress by raising an exception. The query will be stopped, and the exception will propagate to the call site: ```ruby -db.on_progress(100) do +db.on_progress do raise 'BOOM!' end @@ -912,7 +911,7 @@ Here's how a timeout might be implemented using the progress handler: ```ruby def setup_progress_handler - @db.on_progress(100) do + @db.on_progress do raise TimeoutError if Time.now - @t0 >= @timeout Thread.pass end @@ -956,15 +955,41 @@ This allows you to implement separate logic to deal with busy states, for example sleeping for a small period of time, or implementing a different timeout period. -### Tuning the Progress Handler Period +### Advanced Progress Handler Settings + +You can further tune the behaviour of the progress handler with the following +options passed to `#on_progress`: + +- `:mode`: the following modes are supported: + - `:none` : the progress handler is disabled. + - `:normal`: the progress handler is called on query progress (this is the + default mode). + - `:once`: the progress handler is called once before running the query. + - `:at_least_once`: the progress handler is called once before running the + query, and on query progress. +- `:period`: controls the approximate number of SQLite VM instructions executed + between consecutive calls to the progress handler. Default value: 1000. +- `:tick`: controls the granularity of the progress handler. This is the value + passed internally to the SQLite library. Default value: 10. + +```ruby +db.on_progress(mode: :at_least_once, period: 640, tick: 5) { snooze } +``` + +### Global Progress Handler Settings + +You can set the global progress handler behaviour by calling +`Extralite.on_progress`. You can use this API to set the global progress +settings, without needing to set a progress handler individually for each +`Database` instance. This method takes the same options as +`Database#on_progress`: -The progress period passed to `#on_progress` determines how often the progress -handler will be called. For very simple queries and a big enough period, the -progress handler will not be called at all. On the other hand, setting the -period too low might hurt the performance of your queries, since there's some -overhead to invoking the progress handler, even if it does nothing. Therefore -you might want to experiment with different period values to see what offers the -best performance for your specific situation. +```ruby +Extralite.on_progress(mode: :at_least_once, period: 640, tick: 5) { snooze } + +# the new database instance uses the global progress handler settings +db = Database.new(':memory:') +``` ### Extralite and Fibers