Add GC levels to the test suite, and a valgrind docker image

CONTRIBUTING.md

@@ -27,6 +27,71 @@ explicitly closed, it will be closed when it is GCed.
 will be closed/finalized when it is GCed.
 
 
+## Debugging memory issues
+
+Please install `valgrind` and `gdb` before you use the tools in this section.
+
+Run the test suite under valgrind and [`ruby_memcheck`](https://github.com/Shopify/ruby_memcheck) to
+look for memory leaks and other memory errors:
+
+``` sh
+bundle exec rake compile test:valgrind
+```
+
+If you can't install valgrind on your system, use the `sqlite3-dev` docker image, which contains
+valgrind:
+
+``` sh
+# build the image
+bundle exec rake docker:dev:build
+
+# run the test suite in a container
+bundle exec rake docker:dev:test
+
+# run the test suite under valgrind in a container
+bundle exec rake docker:dev:test:valgrind
+```
+
+Each `docker:dev:test` task builds the image first, then mounts your working copy at `/sqlite3` in
+the container. Note that the container compiles into the mounted working copy, so please re-run
+`rake compile` on your machine afterwards.
+
+Run the test suite in the debugger:
+
+``` sh
+bundle exec rake compile test:gdb
+```
+
+You can also run the test suite with a variety of GC behaviors, which is useful to localize some
+classes of memory bugs. Set the `SQLITE3_TEST_GC_LEVEL` environment variable (see `test/helper.rb`
+for more info). A more stressful level finds more bugs, but makes the suite slower:
+
+``` sh
+# ordinary GC behavior (the default)
+SQLITE3_TEST_GC_LEVEL=normal bundle exec rake compile test
+
+# minor GC after each test
+SQLITE3_TEST_GC_LEVEL=minor bundle exec rake compile test
+
+# major GC after each test
+SQLITE3_TEST_GC_LEVEL=major bundle exec rake compile test
+
+# major GC after each test, and GC compaction after every 20 tests
+SQLITE3_TEST_GC_LEVEL=compact bundle exec rake compile test
+
+# verify references after compaction, after every 20 tests
+# (see https://alanwu.space/post/check-compaction/)
+SQLITE3_TEST_GC_LEVEL=verify bundle exec rake compile test
+
+# run each test with GC "stress mode" on
+SQLITE3_TEST_GC_LEVEL=stress bundle exec rake compile test
+```
+
+The `compact` and `verify` levels fall back to `normal` on a platform that does not support GC
+compaction. The `stress` level makes the suite about 150 times slower, and it makes
+timing-sensitive tests unreliable.
+
+
 ## Building gems
 
 As a prerequisite please make sure you have `docker` correctly installed, so that you're able to cross-compile the native gems.

misc/Dockerfile.dev

@@ -0,0 +1,15 @@
+FROM ruby:4.0
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y apt-utils
+
+RUN apt-get install -y valgrind
+
+COPY Gemfile system/
+COPY Gemfile.lock system/
+COPY sqlite3.gemspec system/
+
+RUN gem install bundler -v "$(grep -A 1 "BUNDLED WITH" system/Gemfile.lock | tail -n 1)"
+RUN cd system && bundle install

rakelib/check-manifest.rake

@@ -15,6 +15,7 @@ task :check_manifest do
     doc
     gems
     issues
+    misc
     patches
     pkg
     ports
@@ -26,11 +27,13 @@ task :check_manifest do
   }
   ignore_files = %w[
     .editorconfig
+    .git
     .gitignore
     .rdoc_options
     .rubocop.yml
     Gemfile*
     Rakefile
+    SECURITY.md
     [a-z]*.{log,out}
     [0-9]*
     appveyor.yml

rakelib/docker-dev.rake

@@ -0,0 +1,39 @@
+module DockerDevHelper
+  extend Rake::DSL
+
+  IMAGE = "sqlite3-dev"
+  PROJECT_DIR = File.expand_path("..", __dir__)
+  DOCKERFILE = File.join(PROJECT_DIR, "misc", "Dockerfile.dev")
+  MOUNT_DIR = "/sqlite3"
+
+  class << self
+    def build
+      sh "docker build -t #{IMAGE} -f #{DOCKERFILE} #{PROJECT_DIR}"
+    end
+
+    def run(command)
+      sh "docker run --rm -v #{PROJECT_DIR}:#{MOUNT_DIR} -w #{MOUNT_DIR} #{IMAGE} #{command}"
+    end
+  end
+end
+
+namespace "docker" do
+  namespace "dev" do
+    desc "Build a 'sqlite3-dev' docker image for development and testing"
+    task "build" do
+      DockerDevHelper.build
+    end
+
+    desc "Run the test suite in a 'sqlite3-dev' container"
+    task "test" => "docker:dev:build" do
+      DockerDevHelper.run("bundle exec rake compile test")
+    end
+
+    namespace "test" do
+      desc "Run the test suite under valgrind in a 'sqlite3-dev' container"
+      task "valgrind" => "docker:dev:build" do
+        DockerDevHelper.run("bundle exec rake compile test:valgrind")
+      end
+    end
+  end
+end

test/helper.rb

@@ -1,3 +1,15 @@
+#
+# Some environment variables that are used to configure the test suite:
+#
+# - SQLITE3_TEST_GC_LEVEL: (roughly in order of stress)
+#   - "normal" - normal GC behavior (default)
+#   - "minor" - force a minor GC cycle after each test
+#   - "major" - force a major GC cycle after each test
+#   - "compact" - force a major GC after each test, and GC compaction after every 20 tests
+#   - "verify" - force a major GC after each test, and verify references-after-compaction after
+#     every 20 tests
+#   - "stress" - run each test with GC.stress set to true
+#
 require "sqlite3"
 require "minitest/autorun"
 require "yaml"
@@ -6,10 +18,98 @@
 
 module SQLite3
   class TestCase < Minitest::Test
+    COMPACT_EVERY = 20
+
+    class << self
+      attr_accessor :test_count
+
+      def gc_level
+        @gc_level ||= detect_gc_level
+      end
+
+      private
+
+      def detect_gc_level
+        case ENV["SQLITE3_TEST_GC_LEVEL"]&.to_sym
+        when :stress then :stress
+        when :minor then :minor
+        when :major then :major
+        when :compact then gc_compaction_supported? ? :compact : :normal
+        when :verify then gc_compaction_verifiable? ? :verify : :normal
+        else :normal
+        end
+      end
+
+      def gc_compaction_supported?
+        # the only way to detect an unsupported platform is to try GC compaction
+        GC.compact
+        true
+      rescue NotImplementedError
+        warn("#{__FILE__}:#{__LINE__}: GC compaction is not supported by this platform")
+        false
+      end
+
+      def gc_compaction_verifiable?
+        gc_compaction_supported? && GC.respond_to?(:verify_compaction_references)
+      end
+    end
+
+    self.test_count = 0
+
     alias_method :assert_not_equal, :refute_equal
     alias_method :assert_not_nil, :refute_nil
     alias_method :assert_raise, :assert_raises
 
+    def before_setup
+      TestCase.test_count += 1
+      GC.stress = true if gc_level == :stress
+
+      super
+    end
+
+    def after_teardown
+      case gc_level
+      when :minor
+        GC.start(full_mark: false)
+      when :major
+        GC.start(full_mark: true)
+      when :compact
+        if compaction_scheduled?
+          GC.compact
+          putc("<")
+        else
+          GC.start(full_mark: true)
+        end
+      when :verify
+        if compaction_scheduled?
+          gc_verify_compaction_references
+          putc("!")
+        end
+        GC.start(full_mark: true)
+      when :stress
+        GC.stress = false
+      end
+
+      super
+    end
+
+    def gc_level
+      TestCase.gc_level
+    end
+
+    def compaction_scheduled?
+      TestCase.test_count % COMPACT_EVERY == 0
+    end
+
+    def gc_verify_compaction_references
+      # https://alanwu.space/post/check-compaction/
+      GC.verify_compaction_references(expand_heap: true, toward: :empty)
+    end
+
+    def skip_unless_compaction_supported
+      skip("GC compaction is unsupported on this runtime") unless GC.respond_to?(:verify_compaction_references)
+    end
+
     def assert_nothing_raised
       yield
     end
@@ -19,8 +119,15 @@ def i_am_running_in_valgrind
       ENV["LD_PRELOAD"] =~ /valgrind|vgpreload/
     end
 
+    def skip_if_timing_unreliable
+      skip("valgrind is too slow to measure throughput") if i_am_running_in_valgrind
+      skip("GC stress is too slow to measure throughput") if gc_level == :stress
+    end
+
     def windows?
       ::RUBY_PLATFORM =~ /mingw|mswin/
     end
   end
 end
+
+puts "SQLITE3_TEST_GC_LEVEL: #{SQLite3::TestCase.gc_level}"

test/test_integration_pending.rb

@@ -114,6 +114,8 @@ def test_busy_timeout
   end
 
   def test_busy_handler_timeout_releases_gvl
+    skip_if_timing_unreliable
+
     @db.busy_handler_timeout = 100
 
     t1sync = ThreadSynchronizer.new