Adds multi_step/3 (#128)

Author: warmwaffles

CHANGELOG.md

@@ -1,25 +1,30 @@
 # Changelog
 
-## [Unreleased]
+
+## [0.5.2] - 2021-03-23
 
 ### Added
-- Guide for Windows users
+- Guide for Windows users.
+- `Exqlite.Sqlite3.multi_step/3` to step through results chunks at a time.
+- `default_chunk_size` configuration.
 
 ## [0.5.1] - 2021-03-19
 
 ### Changed
 - Bumped SQLite3 amalgamation to version 3.35.2
 - Replaced old references of [github.com/warmwaffles](http://github.com/warmwaffles)
 
+
 ## [0.5.0] - 2021-03-17
 
 ### Removed
-- Removed `Ecto.Adapters.Exqlite`  
+- Removed `Ecto.Adapters.Exqlite`
   Replaced with [Ecto Sqlite3][ecto_sqlite3] library.
 
 
 [ecto_sqlite3]: <https://github.com/elixir-sqlite/ecto_sqlite3>
 
 [Unreleased]: https://github.com/elixir-sqlite/exqlite/compare/v0.5.1...main
+[0.5.2]: https://github.com/elixir-sqlite/exqlite/compare/v0.5.1...v0.5.2
 [0.5.1]: https://github.com/elixir-sqlite/exqlite/compare/v0.5.0...v0.5.1
 [0.5.0]: https://github.com/elixir-sqlite/exqlite/compare/v0.4.9...v0.5.0

README.md

@@ -36,6 +36,16 @@ end
 ```
 
 
+## Configuration
+
+```elixir
+config :exqlite, default_chunk_size: 100
+```
+
+* `default_chunk_size` - The chunk size that is used when multi-stepping when
+  not specifying the chunk size explicitly.
+
+
 ## Usage
 
 The `Exqlite.Sqlite3` module usage is fairly straight forward.

c_src/sqlite3_nif.c

@@ -445,7 +445,63 @@ make_row(ErlNifEnv* env, sqlite3_stmt* statement)
 
     enif_free(columns);
 
-    return enif_make_tuple2(env, make_atom(env, "row"), row);
+    return row;
+}
+
+static ERL_NIF_TERM
+exqlite_multi_step(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    assert(env);
+
+    statement_t* statement = NULL;
+    connection_t* conn     = NULL;
+    int chunk_size;
+
+    if (argc != 3) {
+        return enif_make_badarg(env);
+    }
+
+    if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) {
+        return make_error_tuple(env, "invalid_connection");
+    }
+
+    if (!enif_get_resource(env, argv[1], statement_type, (void**)&statement)) {
+        return make_error_tuple(env, "invalid_statement");
+    }
+
+    if (!enif_get_int(env, argv[2], &chunk_size)) {
+        return make_error_tuple(env, "invalid_chunk_size");
+    }
+
+    if (chunk_size < 1) {
+        return make_error_tuple(env, "invalid_chunk_size");
+    }
+
+    ERL_NIF_TERM rows = enif_make_list_from_array(env, NULL, 0);
+    for (int i = 0; i < chunk_size; i++) {
+        ERL_NIF_TERM row;
+
+        int rc = sqlite3_step(statement->statement);
+        switch (rc) {
+            case SQLITE_BUSY:
+                sqlite3_reset(statement->statement);
+                return make_atom(env, "busy");
+
+            case SQLITE_DONE:
+                return enif_make_tuple2(env, make_atom(env, "done"), rows);
+
+            case SQLITE_ROW:
+                row = make_row(env, statement->statement);
+                rows = enif_make_list_cell(env, row, rows);
+                break;
+
+            default:
+                sqlite3_reset(statement->statement);
+                return make_sqlite3_error_tuple(env, rc, conn->db);
+        }
+    }
+
+    return enif_make_tuple2(env, make_atom(env, "rows"), rows);
 }
 
 static ERL_NIF_TERM
@@ -471,7 +527,11 @@ exqlite_step(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     int rc = sqlite3_step(statement->statement);
     switch (rc) {
         case SQLITE_ROW:
-            return make_row(env, statement->statement);
+            return enif_make_tuple2(
+                env,
+                make_atom(env, "row"),
+                make_row(env, statement->statement)
+            );
         case SQLITE_BUSY:
             return make_atom(env, "busy");
         case SQLITE_DONE:
@@ -642,6 +702,7 @@ static ErlNifFunc nif_funcs[] = {
   {"prepare", 2, exqlite_prepare, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"bind", 3, exqlite_bind, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"step", 2, exqlite_step, ERL_NIF_DIRTY_JOB_IO_BOUND},
+  {"multi_step", 3, exqlite_multi_step, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"columns", 2, exqlite_columns, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"last_insert_rowid", 1, exqlite_last_insert_rowid, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"transaction_status", 1, exqlite_transaction_status, ERL_NIF_DIRTY_JOB_IO_BOUND},

lib/exqlite/sqlite3.ex

@@ -19,6 +19,7 @@ defmodule Exqlite.Sqlite3 do
   @type db() :: reference()
   @type statement() :: reference()
   @type reason() :: atom() | String.t()
+  @type row() :: []
 
   @doc """
   Opens a new sqlite database at the Path provided.
@@ -78,6 +79,30 @@ defmodule Exqlite.Sqlite3 do
   @spec step(db(), statement()) :: :done | :busy | {:row, []}
   def step(conn, statement), do: Sqlite3NIF.step(conn, statement)
 
+  @spec multi_step(db(), statement()) :: :busy | {:rows, [row()]} | {:done, [row()]}
+  def multi_step(conn, statement) do
+    chunk_size = Application.get_env(:exqlite, :default_chunk_size, 50)
+    multi_step(conn, statement, chunk_size)
+  end
+
+  @spec multi_step(db(), statement(), integer()) ::
+          :busy | {:rows, [row()]} | {:done, [row()]}
+  def multi_step(conn, statement, chunk_size) do
+    case Sqlite3NIF.multi_step(conn, statement, chunk_size) do
+      :busy ->
+        {:error, "Database busy"}
+
+      {:error, reason} ->
+        {:error, reason}
+
+      {:rows, rows} ->
+        {:rows, Enum.reverse(rows)}
+
+      {:done, rows} ->
+        {:done, Enum.reverse(rows)}
+    end
+  end
+
   @spec last_insert_rowid(db()) :: {:ok, integer()}
   def last_insert_rowid(conn), do: Sqlite3NIF.last_insert_rowid(conn)
 
@@ -93,22 +118,29 @@ defmodule Exqlite.Sqlite3 do
     Sqlite3NIF.execute(conn, String.to_charlist("PRAGMA shrink_memory"))
   end
 
-  @spec fetch_all(db(), statement()) :: {:ok, []} | {:error, reason()}
-  def fetch_all(conn, statement) do
+  @spec fetch_all(db(), statement()) :: {:ok, [row()]} | {:error, reason()}
+  def fetch_all(conn, statement, chunk_size \\ 50) do
     # TODO: Should this be done in the NIF? It can be _much_ faster to build a
     # list there, but at the expense that it could block other dirty nifs from
     # getting work done.
     #
     # For now this just works
-    fetch_all(conn, statement, [])
+    fetch_all(conn, statement, chunk_size, [])
   end
 
-  defp fetch_all(conn, statement, result) do
-    case step(conn, statement) do
-      :busy -> {:error, "Database busy"}
-      {:error, reason} -> {:error, reason}
-      :done -> {:ok, result}
-      {:row, row} -> fetch_all(conn, statement, result ++ [row])
+  defp fetch_all(conn, statement, chunk_size, accum) do
+    case multi_step(conn, statement, chunk_size) do
+      {:done, rows} ->
+        {:ok, accum ++ rows}
+
+      {:rows, rows} ->
+        fetch_all(conn, statement, accum ++ rows)
+
+      {:error, reason} ->
+        {:error, reason}
+
+      :busy ->
+        {:error, "Database busy"}
     end
   end
 

lib/exqlite/sqlite3_nif.ex

@@ -37,6 +37,10 @@ defmodule Exqlite.Sqlite3NIF do
   @spec step(db(), statement()) :: :done | :busy | {:row, []}
   def step(_conn, _statement), do: :erlang.nif_error(:not_loaded)
 
+  @spec multi_step(db(), statement(), integer()) ::
+          :busy | {:rows, [[]]} | {:done, [[]]}
+  def multi_step(_conn, _statement, _chunk_size), do: :erlang.nif_error(:not_loaded)
+
   @spec columns(db(), statement()) :: {:ok, []} | {:error, reason()}
   def columns(_conn, _statement), do: :erlang.nif_error(:not_loaded)
 

test/exqlite/sqlite3_test.exs

@@ -165,7 +165,8 @@ defmodule Exqlite.Sqlite3Test do
       :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('Another test')")
       {:ok, 2} = Sqlite3.last_insert_rowid(conn)
 
-      {:ok, statement} = Sqlite3.prepare(conn, "select id, stuff from test")
+      {:ok, statement} =
+        Sqlite3.prepare(conn, "select id, stuff from test order by id asc")
 
       {:row, columns} = Sqlite3.step(conn, statement)
       assert [1, "This is a test"] == columns
@@ -202,6 +203,61 @@ defmodule Exqlite.Sqlite3Test do
     end
   end
 
+  describe ".multi_step/3" do
+    test "returns results" do
+      {:ok, conn} = Sqlite3.open(":memory:")
+
+      :ok =
+        Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)")
+
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('one')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('two')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('three')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('four')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('five')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('six')")
+
+      {:ok, statement} =
+        Sqlite3.prepare(conn, "select id, stuff from test order by id asc")
+
+      {:rows, rows} = Sqlite3.multi_step(conn, statement, 4)
+      assert rows == [[1, "one"], [2, "two"], [3, "three"], [4, "four"]]
+
+      {:done, rows} = Sqlite3.multi_step(conn, statement, 4)
+      assert rows == [[5, "five"], [6, "six"]]
+    end
+  end
+
+  describe ".multi_step/2" do
+    test "returns results" do
+      {:ok, conn} = Sqlite3.open(":memory:")
+
+      :ok =
+        Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)")
+
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('one')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('two')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('three')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('four')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('five')")
+      :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('six')")
+
+      {:ok, statement} =
+        Sqlite3.prepare(conn, "select id, stuff from test order by id asc")
+
+      {:done, rows} = Sqlite3.multi_step(conn, statement)
+
+      assert rows == [
+               [1, "one"],
+               [2, "two"],
+               [3, "three"],
+               [4, "four"],
+               [5, "five"],
+               [6, "six"]
+             ]
+    end
+  end
+
   describe "working with prepared statements after close" do
     test "returns proper error" do
       {:ok, conn} = Sqlite3.open(":memory:")