diff --git a/.gitignore b/.gitignore index 328eaa1..2d4e330 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ target/ *.db - +*.cdtor.c +*.res +*.cdtor.o +rustc*/ wasm/ diff --git a/Cargo.lock b/Cargo.lock index 1133c7d..f405c67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -246,9 +246,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.133" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libloading" diff --git a/Cargo.toml b/Cargo.toml index 6431ecd..0d8c42e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,15 +15,17 @@ sqlite-loadable-macros={version="0.0.3", path="./sqlite-loadable-macros"} serde = {version="1.0.147", features = ["derive"]} serde_json = "1.0.87" bitflags = "1.3.2" -libsqlite3-sys = {version="0.26.0", optional=true, features=["bundled"]} +libsqlite3-sys = {version="^0.26", optional=true, features=["bundled"]} [dev-dependencies] rusqlite = "0.29.0" -libsqlite3-sys = {version="0.26.0", default-features = false, features=["bundled"]} +libsqlite3-sys = {version="^0.26", default-features = false, features=["bundled"]} [features] static = ["libsqlite3-sys"] exec = [] +vfs_syscall = [] +vfs_loadext = [] [lib] doctest = false @@ -47,3 +49,7 @@ crate-type = ["cdylib"] [[example]] name = "load_permanent" crate-type = ["cdylib"] + +[[example]] +name = "mem_vfs" +crate-type = ["cdylib"] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6aef86d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# kernel 6.1 +FROM debian:bookworm-slim + +RUN apt-get update + +# development +RUN apt-get install -y curl valgrind build-essential clang pahole git + +# connect with vs code remote via ssh +RUN apt install -y openssh-server + +# project +RUN apt-get install -y libsqlite3-dev sqlite3 liburing-dev + +# upgrade kernel to 6.1 +RUN apt upgrade -y linux-image-arm64 + +# rust +ENV RUST_VERSION=stable +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=$RUST_VERSION + +# Install cargo-valgrind +RUN /bin/bash -c "source /root/.cargo/env && cargo install cargo-valgrind" + +# Check sqlite compile options: +RUN echo "PRAGMA compile_options;" | sqlite3 + +RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config + +EXPOSE 22 + +ENTRYPOINT service ssh start && bash + +# on visual code studio +# install "remote development" "remote - ssh" "rust-analyzer" diff --git a/README.md b/README.md index f39c746..8578d66 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,20 @@ select * from xxx; Some real-world non-Rust examples of traditional virtual tables in SQLite include the [CSV virtual table](https://www.sqlite.org/csv.html), the full-text search [fts5 extension](https://www.sqlite.org/fts5.html#fts5_table_creation_and_initialization), and the [R-Tree extension](https://www.sqlite.org/rtree.html#creating_an_r_tree_index). +### Virtual file system + +There are two examples of how to apply this library to create your own vfs. +1. [io_uring_vfs](./benchmarks/vfs/io_uring/) +2. [mem_vfs](./examples/mem_vfs.rs) + +In summary, you need to extend two traits "SqliteIoMethods" and "SqliteVfs", then attach those together +in the open function in SqliteVfs. + +You can load the custom vfs in a compiled sqlite3 binary, by doing the following: +* In Cargo.toml, make sure the feature "static" is disabled: sqlite-loadable = {path="..."} +* Load the dynamic object, e.g.: sqlite3 -cmd '.load ./target/debug/lib_myvfs' +* in SQL: ATTACH 'file:my.db?vfs=myvfs' as myvfs + ## Examples The [`examples/`](./examples/) directory has a few bare-bones examples of extensions, which you can build with: @@ -243,7 +257,7 @@ A hello world extension in C is `17KB`, while one in Rust is `469k`. It's still - [ ] Stabilize virtual table interface - [ ] Support [aggregate window functions](https://www.sqlite.org/windowfunctions.html#udfwinfunc) ([#1](https://github.com/asg017/sqlite-loadable-rs/issues/1)) - [ ] Support [collating sequences](https://www.sqlite.org/c3ref/create_collation.html) ([#2](https://github.com/asg017/sqlite-loadable-rs/issues/2)) -- [ ] Support [virtual file systems](sqlite.org/vfs.html) ([#3](https://github.com/asg017/sqlite-loadable-rs/issues/3)) +- [x] Support [virtual file systems](sqlite.org/vfs.html) ([#3](https://github.com/asg017/sqlite-loadable-rs/issues/3)) ## Supporting diff --git a/benchmarks/vfs/io_uring/.gitignore b/benchmarks/vfs/io_uring/.gitignore new file mode 100644 index 0000000..fc1f0aa --- /dev/null +++ b/benchmarks/vfs/io_uring/.gitignore @@ -0,0 +1,6 @@ +*.db +*-journal +sqlite3 +leaky.txt +db/** +md/** diff --git a/benchmarks/vfs/io_uring/Cargo.lock b/benchmarks/vfs/io_uring/Cargo.lock new file mode 100644 index 0000000..f67e550 --- /dev/null +++ b/benchmarks/vfs/io_uring/Cargo.lock @@ -0,0 +1,905 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bindgen" +version = "0.60.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "clap", + "env_logger", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "which", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clang-sys" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex", + "indexmap", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.3", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "io-uring" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460648e47a07a43110fbfa2e0b14afb2be920093c31e5dccc50e49568e099762" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rusqlite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" +dependencies = [ + "bitflags 2.4.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfeae074e687625746172d639330f1de242a178bf3189b51e35a7a21573513ac" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "sqlite-loadable" +version = "0.0.6-alpha.6" +dependencies = [ + "bitflags 1.3.2", + "libsqlite3-sys", + "serde", + "serde_json", + "sqlite-loadable-macros", + "sqlite3ext-sys", +] + +[[package]] +name = "sqlite-loadable-macros" +version = "0.0.3" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "sqlite3_vfs_io_uring" +version = "0.1.0" +dependencies = [ + "env_logger", + "io-uring", + "libc", + "libsqlite3-sys", + "log", + "rand", + "rusqlite", + "sqlite-loadable", + "sqlite3ext-sys", + "strum", + "tempfile", +] + +[[package]] +name = "sqlite3ext-sys" +version = "0.0.1" +dependencies = [ + "bindgen", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.39", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "termcolor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "zerocopy" +version = "0.7.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] diff --git a/benchmarks/vfs/io_uring/Cargo.toml b/benchmarks/vfs/io_uring/Cargo.toml new file mode 100644 index 0000000..652f5eb --- /dev/null +++ b/benchmarks/vfs/io_uring/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "sqlite3_vfs_io_uring" +version = "0.1.0" +edition = "2021" + +[dependencies] +io-uring = "0.6.1" +sqlite3ext-sys = {path="../../../sqlite3ext-sys"} +sqlite-loadable = {path="../../../", features = ["static"]} +libc = "0.2.148" +log = "0.4" +env_logger = "0.9" +strum = { version = "0.25", features = ["derive"] } + +[dev-dependencies] +rusqlite = "0.29.0" +libsqlite3-sys = {version="0.26.0", default-features = false} +tempfile = "3.8.0" +rand = "0.8.5" + +[lib] +name = "_iouringvfs" +crate-type = ["lib", "staticlib", "cdylib"] + +build = "build.rs" diff --git a/benchmarks/vfs/io_uring/README.md b/benchmarks/vfs/io_uring/README.md new file mode 100644 index 0000000..a2edc91 --- /dev/null +++ b/benchmarks/vfs/io_uring/README.md @@ -0,0 +1,373 @@ +# sqlite3_vfs_io_uring_rs +Performance test: sqlite3 vfs + IO Uring with WAL and rollback journalling + +***Warning***: IO Uring is only supported on linux, where this IO Uring has been activated. +IO Uring has been turned off on many distros due to certain security risks. + +This project was tested on Docker and VirtualBox. Your mileage will vary. +Also, all of the tests ran on [rusqlite](https://github.com/rusqlite/rusqlite). + +## Benchmark speeds with hyperfine + +[This script](./run-hyperfine.sh) was written to benchmark and compare, memory vfs as baseline, unix vfs and +the custom IO Uring based vfs, with the default [rollback journalling, and WAL](https://fly.io/blog/sqlite-internals-wal/). + +## Tests + +[Tests](./examples/) were [derived from this archived sqlite document](https://www.sqlite.org/speed.html), +to show whether adding IO Uring support to a custom vfs will impact sqlite3's performance positively. + +16 tests are run, on volatile memory, file storage and file storage via io-uring, where memory storage serves as a baseline. + +| Test | Description | +| --- | --- | +| [1](./examples/test_1.rs) | INSERTs | +| [2](./examples/test_2.rs) | INSERTs in a transaction | +| [3](./examples/test_3.rs) | INSERTs into an indexed table | +| [4](./examples/test_4.rs) | SELECTs without an index | +| [5](./examples/test_5.rs) | SELECTs on a string comparison | +| [6](./examples/test_6.rs) | Creating an index | +| [7](./examples/test_7.rs) | SELECTs with an index | +| [8](./examples/test_8.rs) | UPDATEs without an index | +| [9](./examples/test_9.rs) | UPDATEs with an index | +| [10](./examples/test_10.rs) | Text UPDATEs with an index | +| [11](./examples/test_11.rs) | INSERTs from a SELECT | +| [12](./examples/test_12.rs) | DELETE without an index | +| [13](./examples/test_13.rs) | DELETE with an index | +| [14](./examples/test_14.rs) | A big INSERT after a big DELETE | +| [15](./examples/test_15.rs) | A big DELETE followed by many small INSERTs | +| [16](./examples/test_16.rs) | DROP TABLE | + +## Run the tests +Run [this script](./run-hyperfine.sh) in a shell +```bash +sh run-hyperfine.sh +``` + +If you don't have linux running on your machine (yet), use +[the docker script provided here](../../../run-docker.sh). + +## Logging + +```bash +RUST_LOG=trace cargo test +``` + +## Results + +The numbers here were generated on a noisy machine on Docker. +Your mileage might vary. + +Lower "Relative" speed is better. + +### Apple M2, Docker, Linux 6.3.13-linuxkit + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_1` | 5.3 ± 4.0 | 4.4 | 72.4 | 1.00 | +| `test_1 db/test_1.db` | 1711.4 ± 75.7 | 1631.3 | 1847.3 | 320.26 ± 237.45 | +| `test_1 db/test_1.ring.db` | 1531.8 ± 59.3 | 1488.8 | 1657.8 | 286.65 ± 212.44 | +| `test_1 db/test_1.ring.wal.db` | 1534.6 ± 31.9 | 1498.6 | 1611.6 | 287.18 ± 212.63 | +| `test_1 db/test_1.wal.db` | 230.8 ± 5.7 | 225.2 | 242.4 | 43.19 ± 31.98 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_2` | 98.4 ± 1.9 | 95.5 | 102.6 | 1.00 | +| `test_2 db/test_2.db` | 120.6 ± 1.9 | 115.4 | 125.2 | 1.23 ± 0.03 | +| `test_2 db/test_2.ring.db` | 130.3 ± 1.7 | 127.0 | 133.2 | 1.32 ± 0.03 | +| `test_2 db/test_2.ring.wal.db` | 131.6 ± 5.7 | 128.0 | 154.8 | 1.34 ± 0.06 | +| `test_2 db/test_2.wal.db` | 192.4 ± 30.4 | 164.0 | 259.9 | 1.96 ± 0.31 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_3` | 150.5 ± 24.8 | 125.5 | 186.1 | 1.00 | +| `test_3 db/test_3.db` | 4062.4 ± 271.3 | 3654.4 | 4634.7 | 27.00 ± 4.81 | +| `test_3 db/test_3.ring.db` | 5786.9 ± 221.8 | 5373.7 | 6080.0 | 38.45 ± 6.51 | +| `test_3 db/test_3.ring.wal.db` | 5255.5 ± 465.5 | 4633.1 | 6161.7 | 34.92 ± 6.54 | +| `test_3 db/test_3.wal.db` | 3534.8 ± 236.0 | 3205.1 | 3941.8 | 23.49 ± 4.18 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_4` | 96.9 ± 0.8 | 95.6 | 99.9 | 1.00 | +| `test_4 db/test_4.db` | 149.5 ± 18.9 | 119.5 | 190.2 | 1.54 ± 0.20 | +| `test_4 db/test_4.ring.db` | 135.1 ± 10.0 | 128.4 | 166.4 | 1.39 ± 0.10 | +| `test_4 db/test_4.ring.wal.db` | 131.9 ± 3.2 | 129.0 | 142.4 | 1.36 ± 0.03 | +| `test_4 db/test_4.wal.db` | 167.4 ± 2.2 | 163.4 | 170.2 | 1.73 ± 0.03 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_5` | 100.2 ± 11.4 | 95.8 | 156.2 | 1.00 | +| `test_5 db/test_5.db` | 121.7 ± 2.3 | 115.3 | 126.7 | 1.22 ± 0.14 | +| `test_5 db/test_5.ring.db` | 132.3 ± 5.3 | 128.6 | 152.6 | 1.32 ± 0.16 | +| `test_5 db/test_5.ring.wal.db` | 131.9 ± 1.7 | 128.7 | 135.8 | 1.32 ± 0.15 | +| `test_5 db/test_5.wal.db` | 165.3 ± 4.4 | 148.8 | 169.6 | 1.65 ± 0.19 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_6` | 95.3 ± 1.9 | 94.0 | 104.3 | 1.00 | +| `test_6 db/test_6.db` | 7144.6 ± 409.3 | 6438.6 | 8050.1 | 74.97 ± 4.55 | +| `test_6 db/test_6.ring.db` | 10845.3 ± 682.9 | 10037.5 | 12086.2 | 113.81 ± 7.52 | +| `test_6 db/test_6.ring.wal.db` | 9824.5 ± 457.6 | 8931.9 | 10521.4 | 103.09 ± 5.22 | +| `test_6 db/test_6.wal.db` | 6078.7 ± 330.6 | 5591.8 | 6572.9 | 63.79 ± 3.69 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_7` | 125.5 ± 4.0 | 122.7 | 142.1 | 1.00 | +| `test_7 db/test_7.db` | 3461.0 ± 231.6 | 3128.8 | 4033.8 | 27.59 ± 2.05 | +| `test_7 db/test_7.ring.db` | 4988.5 ± 393.6 | 4464.7 | 5771.1 | 39.76 ± 3.39 | +| `test_7 db/test_7.ring.wal.db` | 4386.0 ± 287.8 | 3839.2 | 4848.0 | 34.96 ± 2.56 | +| `test_7 db/test_7.wal.db` | 2758.9 ± 205.4 | 2436.4 | 3052.6 | 21.99 ± 1.78 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_8` | 645.1 ± 11.5 | 622.3 | 661.6 | 1.00 | +| `test_8 db/test_8.db` | 24049.7 ± 2134.5 | 20911.2 | 27012.6 | 37.28 ± 3.37 | +| `test_8 db/test_8.ring.db` | 74134.3 ± 6544.0 | 64428.8 | 83722.0 | 114.93 ± 10.35 | +| `test_8 db/test_8.ring.wal.db` | 46114.0 ± 6595.8 | 36568.0 | 55722.2 | 71.49 ± 10.30 | +| `test_8 db/test_8.wal.db` | 14945.2 ± 2118.4 | 12069.9 | 18285.6 | 23.17 ± 3.31 | + +| Command | Mean [s] | Min [s] | Max [s] | Relative | +| --- | --- | --- | --- | --- | +| `test_9` | 1.412 ± 0.017 | 1.385 | 1.442 | 1.00 | +| `test_9 db/test_9.db` | 12.207 ± 4.533 | 5.460 | 18.856 | 8.64 ± 3.21 | +| `test_9 db/test_9.ring.db` | 12.158 ± 4.376 | 5.542 | 18.552 | 8.61 ± 3.10 | +| `test_9 db/test_9.ring.wal.db` | 12.206 ± 4.383 | 5.726 | 18.597 | 8.64 ± 3.11 | +| `test_9 db/test_9.wal.db` | 12.195 ± 4.370 | 5.635 | 18.386 | 8.63 ± 3.10 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_10` | 40.7 ± 1.3 | 40.0 | 48.9 | 1.00 | +| `test_10 db/test_10.db` | 810.3 ± 45.6 | 741.3 | 880.1 | 19.93 ± 1.28 | +| `test_10 db/test_10.ring.db` | 1058.8 ± 61.5 | 963.6 | 1158.3 | 26.04 ± 1.72 | +| `test_10 db/test_10.ring.wal.db` | 843.9 ± 62.7 | 751.4 | 928.9 | 20.76 ± 1.67 | +| `test_10 db/test_10.wal.db` | 626.8 ± 42.3 | 566.5 | 684.2 | 15.42 ± 1.15 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_11` | 9.6 ± 0.2 | 9.3 | 11.6 | 1.00 | +| `test_11 db/test_11.db` | 20.8 ± 0.5 | 20.0 | 22.8 | 2.16 ± 0.08 | +| `test_11 db/test_11.ring.db` | 27.8 ± 5.7 | 20.7 | 52.8 | 2.88 ± 0.59 | +| `test_11 db/test_11.ring.wal.db` | 26.4 ± 8.8 | 21.0 | 89.2 | 2.74 ± 0.92 | +| `test_11 db/test_11.wal.db` | 25.9 ± 0.7 | 22.6 | 31.0 | 2.69 ± 0.10 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_12` | 22.4 ± 0.6 | 21.8 | 25.9 | 1.00 | +| `test_12 db/test_12.db` | 70.1 ± 19.7 | 40.9 | 179.4 | 3.13 ± 0.88 | +| `test_12 db/test_12.ring.db` | 107.5 ± 31.5 | 52.1 | 161.9 | 4.80 ± 1.41 | +| `test_12 db/test_12.ring.wal.db` | 111.2 ± 34.0 | 49.8 | 173.8 | 4.97 ± 1.53 | +| `test_12 db/test_12.wal.db` | 69.3 ± 14.2 | 42.4 | 94.5 | 3.10 ± 0.64 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_13` | 26.3 ± 0.5 | 25.9 | 29.0 | 1.00 | +| `test_13 db/test_13.db` | 394.9 ± 235.7 | 49.8 | 788.3 | 15.00 ± 8.96 | +| `test_13 db/test_13.ring.db` | 375.3 ± 235.3 | 68.3 | 845.4 | 14.26 ± 8.94 | +| `test_13 db/test_13.ring.wal.db` | 375.6 ± 234.6 | 70.3 | 810.0 | 14.27 ± 8.92 | +| `test_13 db/test_13.wal.db` | 330.9 ± 195.5 | 62.2 | 675.6 | 12.57 ± 7.43 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_14` | 34.5 ± 0.6 | 33.8 | 36.7 | 1.00 | +| `test_14 db/test_14.db` | 484.5 ± 243.8 | 118.3 | 1015.3 | 14.06 ± 7.08 | +| `test_14 db/test_14.ring.db` | 494.4 ± 170.9 | 247.9 | 776.9 | 14.35 ± 4.97 | +| `test_14 db/test_14.ring.wal.db` | 526.9 ± 223.5 | 202.0 | 895.1 | 15.29 ± 6.49 | +| `test_14 db/test_14.wal.db` | 410.6 ± 130.4 | 232.0 | 609.7 | 11.92 ± 3.79 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_15` | 34.7 ± 0.8 | 34.0 | 38.8 | 1.00 | +| `test_15 db/test_15.db` | 60.8 ± 1.3 | 56.8 | 63.5 | 1.75 ± 0.05 | +| `test_15 db/test_15.ring.db` | 73.9 ± 3.2 | 69.9 | 87.3 | 2.13 ± 0.10 | +| `test_15 db/test_15.ring.wal.db` | 74.4 ± 6.0 | 68.9 | 109.6 | 2.15 ± 0.18 | +| `test_15 db/test_15.wal.db` | 62.5 ± 0.9 | 60.4 | 65.1 | 1.80 ± 0.05 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_16` | 33.4 ± 0.6 | 32.6 | 36.3 | 1.00 | +| `test_16 db/test_16.db` | 47.9 ± 3.8 | 44.7 | 71.9 | 1.43 ± 0.12 | +| `test_16 db/test_16.ring.db` | 53.2 ± 1.3 | 51.2 | 56.3 | 1.59 ± 0.05 | +| `test_16 db/test_16.ring.wal.db` | 53.7 ± 1.9 | 51.0 | 64.1 | 1.61 ± 0.07 | +| `test_16 db/test_16.wal.db` | 64.0 ± 1.2 | 61.6 | 66.5 | 1.91 ± 0.05 | + +### x86_64 intel, VirtualBox, 6.1.0-14-amd64 + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_1` | 10.4 ± 9.4 | 8.6 | 153.4 | 1.00 | +| `test_1 db/test_1.db` | 4630.9 ± 531.5 | 4197.1 | 5561.1 | 447.40 ± 410.79 | +| `test_1 db/test_1.ring.db` | 1951.1 ± 685.1 | 1213.0 | 3503.8 | 188.50 ± 184.03 | +| `test_1 db/test_1.ring.wal.db` | 2769.1 ± 1240.6 | 1346.5 | 4854.9 | 267.53 ± 271.59 | +| `test_1 db/test_1.wal.db` | 2241.6 ± 183.8 | 2015.5 | 2596.8 | 216.56 ± 198.08 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_2` | 371.4 ± 66.4 | 247.1 | 420.9 | 1.03 ± 0.29 | +| `test_2 db/test_2.db` | 435.4 ± 46.6 | 373.8 | 553.5 | 1.21 ± 0.29 | +| `test_2 db/test_2.ring.db` | 397.4 ± 89.9 | 276.8 | 487.1 | 1.11 ± 0.34 | +| `test_2 db/test_2.ring.wal.db` | 472.7 ± 36.7 | 423.8 | 548.1 | 1.31 ± 0.30 | +| `test_2 db/test_2.wal.db` | 359.4 ± 76.1 | 257.9 | 450.0 | 1.00 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_3` | 527.9 ± 57.4 | 461.1 | 658.5 | 1.00 | +| `test_3 db/test_3.db` | 1454.1 ± 180.4 | 1249.1 | 1760.7 | 2.75 ± 0.45 | +| `test_3 db/test_3.ring.db` | 4028.8 ± 1773.4 | 1361.9 | 6070.2 | 7.63 ± 3.46 | +| `test_3 db/test_3.ring.wal.db` | 3849.9 ± 2358.4 | 675.9 | 7601.8 | 7.29 ± 4.54 | +| `test_3 db/test_3.wal.db` | 1032.9 ± 310.6 | 518.2 | 1313.2 | 1.96 ± 0.63 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_4` | 382.1 ± 47.0 | 245.2 | 417.7 | 1.00 | +| `test_4 db/test_4.db` | 385.4 ± 58.8 | 258.0 | 441.8 | 1.01 ± 0.20 | +| `test_4 db/test_4.ring.db` | 430.9 ± 35.5 | 375.4 | 489.9 | 1.13 ± 0.17 | +| `test_4 db/test_4.ring.wal.db` | 427.9 ± 82.9 | 264.6 | 554.9 | 1.12 ± 0.26 | +| `test_4 db/test_4.wal.db` | 420.7 ± 24.6 | 368.7 | 446.2 | 1.10 ± 0.15 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_5` | 326.6 ± 64.7 | 226.6 | 385.4 | 1.00 | +| `test_5 db/test_5.db` | 391.5 ± 23.1 | 339.2 | 419.6 | 1.20 ± 0.25 | +| `test_5 db/test_5.ring.db` | 387.9 ± 89.0 | 274.0 | 486.3 | 1.19 ± 0.36 | +| `test_5 db/test_5.ring.wal.db` | 441.9 ± 27.2 | 406.4 | 484.0 | 1.35 ± 0.28 | +| `test_5 db/test_5.wal.db` | 344.8 ± 78.3 | 251.2 | 448.1 | 1.06 ± 0.32 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_6` | 396.6 ± 43.5 | 345.7 | 506.1 | 1.00 | +| `test_6 db/test_6.db` | 1856.5 ± 542.8 | 1206.1 | 3224.9 | 4.68 ± 1.46 | +| `test_6 db/test_6.ring.db` | 5879.5 ± 3489.1 | 2376.7 | 10758.2 | 14.82 ± 8.95 | +| `test_6 db/test_6.ring.wal.db` | 3898.6 ± 2451.7 | 788.7 | 9459.3 | 9.83 ± 6.28 | +| `test_6 db/test_6.wal.db` | 1084.6 ± 240.4 | 677.9 | 1388.0 | 2.73 ± 0.68 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_7` | 494.5 ± 68.5 | 434.7 | 660.2 | 1.00 | +| `test_7 db/test_7.db` | 1106.8 ± 152.7 | 803.0 | 1282.8 | 2.24 ± 0.44 | +| `test_7 db/test_7.ring.db` | 2731.3 ± 1946.5 | 697.8 | 5350.9 | 5.52 ± 4.01 | +| `test_7 db/test_7.ring.wal.db` | 2601.2 ± 1627.5 | 623.4 | 5079.9 | 5.26 ± 3.37 | +| `test_7 db/test_7.wal.db` | 822.0 ± 209.7 | 564.6 | 1228.1 | 1.66 ± 0.48 | + +| Command | Mean [s] | Min [s] | Max [s] | Relative | +| --- | --- | --- | --- | --- | +| `test_8` | 2.814 ± 0.281 | 2.314 | 3.102 | 1.00 | +| `test_8 db/test_8.db` | 24.036 ± 8.930 | 11.041 | 37.724 | 8.54 ± 3.29 | +| `test_8 db/test_8.ring.db` | 23.984 ± 9.013 | 11.103 | 38.410 | 8.52 ± 3.31 | +| `test_8 db/test_8.ring.wal.db` | 24.274 ± 9.171 | 11.133 | 38.712 | 8.63 ± 3.37 | +| `test_8 db/test_8.wal.db` | 24.158 ± 8.908 | 11.472 | 37.448 | 8.59 ± 3.28 | + +| Command | Mean [s] | Min [s] | Max [s] | Relative | +| --- | --- | --- | --- | --- | +| `test_9` | 6.269 ± 0.283 | 5.881 | 6.623 | 1.00 | +| `test_9 db/test_9.db` | 52.981 ± 19.343 | 24.574 | 82.915 | 8.45 ± 3.11 | +| `test_9 db/test_9.ring.db` | 53.092 ± 19.455 | 24.432 | 83.084 | 8.47 ± 3.13 | +| `test_9 db/test_9.ring.wal.db` | 53.961 ± 20.785 | 24.343 | 86.876 | 8.61 ± 3.34 | +| `test_9 db/test_9.wal.db` | 55.347 ± 18.596 | 26.589 | 79.522 | 8.83 ± 2.99 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_10` | 168.6 ± 32.5 | 115.4 | 216.5 | 1.00 | +| `test_10 db/test_10.db` | 272.7 ± 28.7 | 204.9 | 320.7 | 1.62 ± 0.36 | +| `test_10 db/test_10.ring.db` | 373.0 ± 125.0 | 188.3 | 555.4 | 2.21 ± 0.86 | +| `test_10 db/test_10.ring.wal.db` | 371.2 ± 74.0 | 264.3 | 514.3 | 2.20 ± 0.61 | +| `test_10 db/test_10.wal.db` | 234.3 ± 49.2 | 159.4 | 292.6 | 1.39 ± 0.40 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_11` | 41.6 ± 7.9 | 26.8 | 98.6 | 1.00 | +| `test_11 db/test_11.db` | 70.8 ± 6.8 | 57.4 | 87.9 | 1.70 ± 0.36 | +| `test_11 db/test_11.ring.db` | 67.7 ± 12.1 | 47.6 | 86.0 | 1.63 ± 0.43 | +| `test_11 db/test_11.ring.wal.db` | 78.5 ± 6.9 | 54.9 | 84.4 | 1.89 ± 0.40 | +| `test_11 db/test_11.wal.db` | 55.3 ± 5.3 | 34.7 | 63.7 | 1.33 ± 0.28 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_12` | 77.1 ± 35.6 | 50.2 | 267.4 | 1.00 | +| `test_12 db/test_12.db` | 161.1 ± 41.7 | 74.9 | 211.9 | 2.09 ± 1.11 | +| `test_12 db/test_12.ring.db` | 148.1 ± 38.6 | 109.8 | 246.6 | 1.92 ± 1.02 | +| `test_12 db/test_12.ring.wal.db` | 179.8 ± 25.1 | 119.6 | 212.5 | 2.33 ± 1.12 | +| `test_12 db/test_12.wal.db` | 137.6 ± 22.2 | 105.1 | 188.5 | 1.78 ± 0.87 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_13` | 101.9 ± 14.4 | 63.5 | 117.8 | 1.00 | +| `test_13 db/test_13.db` | 185.0 ± 30.1 | 128.7 | 245.4 | 1.81 ± 0.39 | +| `test_13 db/test_13.ring.db` | 250.9 ± 83.4 | 160.5 | 381.9 | 2.46 ± 0.89 | +| `test_13 db/test_13.ring.wal.db` | 332.1 ± 106.0 | 186.7 | 504.2 | 3.26 ± 1.14 | +| `test_13 db/test_13.wal.db` | 168.7 ± 19.3 | 137.1 | 199.7 | 1.65 ± 0.30 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_14` | 148.4 ± 14.5 | 108.1 | 166.5 | 1.00 | +| `test_14 db/test_14.db` | 334.2 ± 61.9 | 249.5 | 448.3 | 2.25 ± 0.47 | +| `test_14 db/test_14.ring.db` | 624.7 ± 276.3 | 329.0 | 1044.9 | 4.21 ± 1.91 | +| `test_14 db/test_14.ring.wal.db` | 606.2 ± 177.9 | 339.3 | 926.0 | 4.08 ± 1.26 | +| `test_14 db/test_14.wal.db` | 339.9 ± 86.7 | 214.9 | 453.9 | 2.29 ± 0.63 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_15` | 138.9 ± 24.7 | 93.6 | 180.5 | 1.00 | +| `test_15 db/test_15.db` | 178.4 ± 12.5 | 153.1 | 197.6 | 1.28 ± 0.25 | +| `test_15 db/test_15.ring.db` | 220.3 ± 23.9 | 171.4 | 257.9 | 1.59 ± 0.33 | +| `test_15 db/test_15.ring.wal.db` | 194.6 ± 41.9 | 150.1 | 255.0 | 1.40 ± 0.39 | +| `test_15 db/test_15.wal.db` | 170.9 ± 8.6 | 145.4 | 186.8 | 1.23 ± 0.23 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| --- | --- | --- | --- | --- | +| `test_16` | 154.5 ± 17.7 | 127.9 | 217.1 | 1.02 ± 0.25 | +| `test_16 db/test_16.db` | 163.0 ± 42.7 | 119.6 | 306.8 | 1.08 ± 0.37 | +| `test_16 db/test_16.ring.db` | 200.4 ± 20.3 | 145.3 | 225.0 | 1.32 ± 0.32 | +| `test_16 db/test_16.ring.wal.db` | 196.2 ± 17.9 | 144.0 | 218.7 | 1.29 ± 0.30 | +| `test_16 db/test_16.wal.db` | 151.6 ± 32.8 | 116.3 | 198.7 | 1.00 | + + +## Rollback Journalling +Rollback Journal + Unix VFS, is the fastest on every test. + +## WAL +WAL on IO Uring VFS has some competitive edge, less than half, but not on all tests. + +## Conclusion + +***Warning***: Do not trust these numbers, run them yourself. These tests ran on "noisy" virtualized / containerized machines. + +For speed, there is no reason to use IO Uring at all, with this specific implementation. +Maybe except for those specific cases were IO Uring + WAL seems to have a competitive edge. + +## Future research ideas +* Release build, speed difference? +* Implement on [windows IoRing](https://learn.microsoft.com/en-us/windows/win32/api/ioringapi/) +* Apply insert optimizations [mentioned here](https://voidstar.tech/sqlite_insert_speed) +* Vfs consensus via IO Uring (IO Uring) sockets + Raft, e.g. rqlite +* Turn on libc::O_DIRECT as u64 | libc::O_SYNC as u64 on storage devices that support it +* Reimplement everything with Zig + liburing + +## Determine whether your kernel supports IO Uring + +Linux command-line: +1. uname -a # expect 5 and above +2. grep io_uring_setup /proc/kallsyms # expect 2 lines +3. gcc test_io_uring.c -o test_io_uring && ./test_io_uring + +```C +// test_io_uring.c + +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (syscall(__NR_io_uring_register, 0, IORING_UNREGISTER_BUFFERS, NULL, 0) && errno == ENOSYS) { + printf("%s", "nope\n"); + return -1; + } else { + printf("%s", "yep\n"); + return 0; + } +} + +``` + diff --git a/benchmarks/vfs/io_uring/Taskfile.yml b/benchmarks/vfs/io_uring/Taskfile.yml new file mode 100644 index 0000000..9371940 --- /dev/null +++ b/benchmarks/vfs/io_uring/Taskfile.yml @@ -0,0 +1,17 @@ +# https://taskfile.dev + +version: '3' + +# vars: +# GREETING: Hello, World! + +tasks: + load_test: + cmds: + - sqlite3 -init test.sql + silent: false + valgrind: + cmds: + - VALGRINDFLAGS="--leak-check=yes --trace-children=yes" cargo valgrind test + silent: false + diff --git a/benchmarks/vfs/io_uring/build.rs b/benchmarks/vfs/io_uring/build.rs new file mode 100644 index 0000000..8614cb4 --- /dev/null +++ b/benchmarks/vfs/io_uring/build.rs @@ -0,0 +1,11 @@ +fn main() { + if cfg!(target_os = "linux") { + // Linux-specific build logic goes here + println!("Building for Linux"); + // Continue with your build logic for Linux + } else { + // Print a message and abort the build for other operating systems + eprintln!("This project only supports Linux."); + std::process::exit(1); + } +} diff --git a/benchmarks/vfs/io_uring/examples/test_1.rs b/benchmarks/vfs/io_uring/examples/test_1.rs new file mode 100644 index 0000000..19a4131 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_1.rs @@ -0,0 +1,21 @@ +use rand::thread_rng; +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let conn = create_test_database(args)?; + + let mut stmt = conn.prepare_cached("INSERT INTO t1 (a, b) VALUES (?, ?)")?; + + for _ in 0..1000 { + let value1: i32 = thread_rng().gen_range(0..1000); + let value2: String = format!("Value {}", thread_rng().gen_range(0..1000)); + + stmt.execute((value1, value2))?; + } + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_10.rs b/benchmarks/vfs/io_uring/examples/test_10.rs new file mode 100644 index 0000000..56118ef --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_10.rs @@ -0,0 +1,31 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..5000 { + let value: i32 = rng.gen(); + + tx.execute( + "INSERT INTO t10 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + } + tx.commit()?; + + let tx2 = conn.transaction()?; + for i in 0..5000 { + let r: i32 = rng.gen(); + tx2.execute("UPDATE t10 SET c=?1 WHERE a = ?2", (r, i + 1))?; + } + tx2.commit()?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_11.rs b/benchmarks/vfs/io_uring/examples/test_11.rs new file mode 100644 index 0000000..ae11c2c --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_11.rs @@ -0,0 +1,39 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..1000 { + let value: i32 = rng.gen(); + + tx.execute( + "INSERT INTO t4 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + tx.execute( + "INSERT INTO t5 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + } + tx.commit()?; + + let tx2 = conn.transaction()?; + tx2.execute("INSERT INTO t4 SELECT b,a,c FROM t5", ())?; + tx2.execute("INSERT INTO t5 SELECT b,a,c FROM t4", ())?; + tx2.commit()?; + + // prevent doubling insertions every run, >50gb file is no joke + let tx3 = conn.transaction()?; + tx3.execute("DELETE FROM t4", ())?; + tx3.execute("DELETE FROM t5", ())?; + tx3.commit()?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_12.rs b/benchmarks/vfs/io_uring/examples/test_12.rs new file mode 100644 index 0000000..f921af3 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_12.rs @@ -0,0 +1,31 @@ +use rand::thread_rng; +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + + for _ in 0..5000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + let value3: String = format!("Value {}", thread_rng().gen_range(0..25000)); + + tx.execute( + "INSERT INTO t2 (a, b, c) VALUES (?, ?, ?)", + (value1, value2, value3), + )?; + } + + tx.commit()?; + + conn.execute("DELETE FROM t2 WHERE c LIKE '%50%'", ())?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_13.rs b/benchmarks/vfs/io_uring/examples/test_13.rs new file mode 100644 index 0000000..6ddfd57 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_13.rs @@ -0,0 +1,31 @@ +use rand::thread_rng; +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + + for _ in 0..5000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + let value3: String = format!("Value {}", thread_rng().gen_range(0..25000)); + + tx.execute( + "INSERT INTO t10 (a, b, c) VALUES (?, ?, ?)", + (value1, value2, value3), + )?; + } + + tx.commit()?; + + conn.execute("DELETE FROM t10 WHERE a>10 AND a <20000", ())?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_14.rs b/benchmarks/vfs/io_uring/examples/test_14.rs new file mode 100644 index 0000000..c3b3f37 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_14.rs @@ -0,0 +1,31 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..5000 { + let value: i32 = rng.gen(); + + tx.execute( + "INSERT INTO t4 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + tx.execute( + "INSERT INTO t5 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + } + tx.commit()?; + + conn.execute("DELETE FROM t4 WHERE a % 2 = 0", ())?; + conn.execute("INSERT INTO t4 SELECT * FROM t5;", ())?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_15.rs b/benchmarks/vfs/io_uring/examples/test_15.rs new file mode 100644 index 0000000..400b541 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_15.rs @@ -0,0 +1,37 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..5000 { + let value: i32 = rng.gen(); + + tx.execute( + "INSERT INTO t4 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + } + tx.commit()?; + + let tx2 = conn.transaction()?; + tx2.execute("DELETE FROM t4", ())?; + for _ in 0..5000 { + let value: i32 = rng.gen(); + + tx2.execute( + "INSERT INTO t4 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + } + + tx2.commit()?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_16.rs b/benchmarks/vfs/io_uring/examples/test_16.rs new file mode 100644 index 0000000..2fd7ef1 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_16.rs @@ -0,0 +1,30 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..5000 { + let value: i32 = rng.gen(); + + tx.execute( + "INSERT INTO t4 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + tx.execute( + "INSERT INTO t5 (a, b, c) VALUES (?, ?, ?)", + (value, value, format!("Value {}", value).as_str()), + )?; + } + tx.commit()?; + + conn.execute("DELETE FROM t4", ())?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_2.rs b/benchmarks/vfs/io_uring/examples/test_2.rs new file mode 100644 index 0000000..2bebfdb --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_2.rs @@ -0,0 +1,28 @@ +use rand::thread_rng; +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + + for _ in 0..25000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + let value3: String = format!("Value {}", thread_rng().gen_range(0..25000)); + + tx.execute( + "INSERT INTO t2 (a, b, c) VALUES (?, ?, ?)", + (value1, value2, value3), + )?; + } + + tx.commit()?; + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_3.rs b/benchmarks/vfs/io_uring/examples/test_3.rs new file mode 100644 index 0000000..ad160a3 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_3.rs @@ -0,0 +1,28 @@ +use rand::thread_rng; +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction().expect("Failed to start tx"); + + for _ in 0..25000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + let value3: String = format!("Value {}", thread_rng().gen_range(0..25000)); + + tx.execute( + "INSERT INTO t3 (a, b, c) VALUES (?, ?, ?)", + (value1, value2, value3), + )?; + } + + tx.commit()?; + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_4.rs b/benchmarks/vfs/io_uring/examples/test_4.rs new file mode 100644 index 0000000..de91d50 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_4.rs @@ -0,0 +1,38 @@ +use rand::thread_rng; +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + + for _ in 0..25000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + let value3: String = format!("Value {}", thread_rng().gen_range(0..25000)); + + tx.execute( + "INSERT INTO t4 (a, b, c) VALUES (?, ?, ?)", + (value1, value2, value3), + )?; + } + tx.commit()?; + + let tx2 = conn.transaction()?; + for i in 0..100 { + let lower_bound = i * 100; + let upper_bound = (i + 1) * 1000; + + let _ = tx2 + .prepare("SELECT count(*), avg(b) FROM t4 WHERE b >= ?1 AND b < ?2")? + .query([lower_bound, upper_bound])?; + } + tx2.commit()?; + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_5.rs b/benchmarks/vfs/io_uring/examples/test_5.rs new file mode 100644 index 0000000..6e4aee5 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_5.rs @@ -0,0 +1,34 @@ +use rand::thread_rng; +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..25000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + let value3: String = format!("Value {}", thread_rng().gen_range(0..25000)); + + tx.execute( + "INSERT INTO t5 (a, b, c) VALUES (?, ?, ?)", + (value1, value2, value3), + )?; + } + tx.commit()?; + + let tx2 = conn.transaction()?; + for i in 0..9 { + let _ = tx2 + .prepare("SELECT count(*), avg(b) FROM t5 WHERE c LIKE ?1")? + .query([i])?; + } + tx2.commit()?; + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_6.rs b/benchmarks/vfs/io_uring/examples/test_6.rs new file mode 100644 index 0000000..b900184 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_6.rs @@ -0,0 +1,26 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..25000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + + tx.execute("INSERT INTO t6 (a, b) VALUES (?, ?)", (value1, value2))?; + } + tx.commit()?; + + // fails if file is already indexed, TODO fix + conn.execute("CREATE INDEX IF NOT EXISTS i6a ON t6(a)", ())?; + conn.execute("CREATE INDEX IF NOT EXISTS i6b ON t6(b)", ())?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_7.rs b/benchmarks/vfs/io_uring/examples/test_7.rs new file mode 100644 index 0000000..6eb54df --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_7.rs @@ -0,0 +1,30 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..25000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + + tx.execute("INSERT INTO t7 (a, b) VALUES (?, ?)", (value1, value2))?; + } + tx.commit()?; + + for i in 0..5000 { + let lower_bound = i * 100; + let upper_bound = (i + 1) + 100; + + let _ = conn + .prepare("SELECT count(*), avg(b) FROM t7 WHERE b >= ?1 AND b < ?2")? + .query([lower_bound, upper_bound])?; + } + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_8.rs b/benchmarks/vfs/io_uring/examples/test_8.rs new file mode 100644 index 0000000..643a315 --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_8.rs @@ -0,0 +1,34 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..10000 { + let value1: i32 = rng.gen(); + let value2: i32 = rng.gen(); + + tx.execute("INSERT INTO t8 (a, b) VALUES (?, ?)", (value1, value2))?; + } + tx.commit()?; + + let tx2 = conn.transaction()?; + for i in 0..1000 { + let lower_bound = i * 10; + let upper_bound = (i + 1) + 10; + + tx2.execute( + "UPDATE t8 SET b=b*2 WHERE a >= ?1 AND a < ?2", + (lower_bound, upper_bound), + )?; + } + tx2.commit()?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/examples/test_9.rs b/benchmarks/vfs/io_uring/examples/test_9.rs new file mode 100644 index 0000000..f9b541a --- /dev/null +++ b/benchmarks/vfs/io_uring/examples/test_9.rs @@ -0,0 +1,30 @@ +use rand::Rng; +use std::env; + +include!("../include/conn.in.rs"); + +fn main() -> rusqlite::Result<()> { + let args: Vec = env::args().collect(); + + let mut conn = create_test_database(args)?; + let mut rng = rand::thread_rng(); + + let tx = conn.transaction()?; + for _ in 0..5000 { + let value: i32 = rng.gen(); + + tx.execute("INSERT INTO t9 (a, b) VALUES (?, ?)", (value, value))?; + } + tx.commit()?; + + let tx2 = conn.transaction()?; + for i in 0..5000 { + let r: i32 = rng.gen(); + let upper_bound = i + 1; + + tx2.execute("UPDATE t9 SET b=?1 WHERE a = ?2", (r, upper_bound))?; + } + tx2.commit()?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/include/conn.in.rs b/benchmarks/vfs/io_uring/include/conn.in.rs new file mode 100644 index 0000000..2da0af8 --- /dev/null +++ b/benchmarks/vfs/io_uring/include/conn.in.rs @@ -0,0 +1,70 @@ +use _iouringvfs::sqlite3_iouringvfs_init; +use rusqlite::{ffi::sqlite3_auto_extension, Connection}; + +pub const IOURING_DB_ALIAS: &str = "ring"; + +fn open_io_uring_connection(db: &str) -> rusqlite::Result { + use rusqlite::OpenFlags; + use _iouringvfs::EXTENSION_NAME; + + let conn = Connection::open_with_flags_and_vfs( + db, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE, + EXTENSION_NAME, + )?; + + Ok(conn) +} + +#[allow(dead_code)] +/// Tests were derived from: https://www.sqlite.org/speed.html +fn create_test_database(args: Vec) -> rusqlite::Result { + assert!(args.len() <= 2); + + unsafe { + sqlite3_auto_extension(Some(std::mem::transmute( + sqlite3_iouringvfs_init as *const (), + ))); + } + + // Necessary to load the custom vfs + let _conn = Connection::open_in_memory()?; + _conn.close().expect("error occurred while closing"); + + let conn = if args.len() == 2 { + let file_path = args[1].as_str(); + + let conn = if file_path.contains("ring") { + open_io_uring_connection(file_path)? + }else { + Connection::open(file_path)? + }; + + if file_path.contains("wal") { + conn.execute_batch("PRAGMA journal_mode = WAL")?; + } + + conn + }else { + Connection::open_in_memory()? + }; + + conn.execute_batch( + r"CREATE TABLE IF NOT EXISTS t1(a integer, b varchar(100)); + CREATE TABLE IF NOT EXISTS t2(a integer, b integer, c varchar(100)); + CREATE TABLE IF NOT EXISTS t3(a integer, b integer, c varchar(100)); + CREATE INDEX IF NOT EXISTS i3 ON t3(c); + CREATE TABLE IF NOT EXISTS t4(a integer, b integer, c varchar(100)); + CREATE TABLE IF NOT EXISTS t5(a integer, b integer, c varchar(100)); + CREATE TABLE IF NOT EXISTS t6(a integer, b integer); + CREATE TABLE IF NOT EXISTS t7(a integer, b integer); + CREATE INDEX IF NOT EXISTS i7 ON t7(b); + CREATE TABLE IF NOT EXISTS t8(a integer, b integer); + CREATE TABLE IF NOT EXISTS t9(a integer, b integer); + CREATE TABLE IF NOT EXISTS t10(a integer, b integer, c varchar(100)); + CREATE INDEX IF NOT EXISTS i10 ON t10(a);" + )?; + + Ok(conn) +} \ No newline at end of file diff --git a/benchmarks/vfs/io_uring/iouring.sql b/benchmarks/vfs/io_uring/iouring.sql new file mode 100644 index 0000000..1ded9bb --- /dev/null +++ b/benchmarks/vfs/io_uring/iouring.sql @@ -0,0 +1,31 @@ +-- HOWTO +-- 0. Make sure the sqlite3 was built with the build tag: SQLITE3VFS_LOADABLE_EXT +-- 1. In Cargo.toml, disable the 'static' feature, from the sqlite-loadable library +-- it should look like this: sqlite-loadable = {path="../../../"} +-- 2. Cargo build +-- 3. load this script: sqlite3 --init iouring.sql + +-- This script was tested on 3.44.2, compiled with: +-- gcc -g -DSQLITE_DEBUG shell.c sqlite3.c -lpthread -ldl -o sqlite3 + +.mode box +.header on + +.load target/debug/lib_iouringvfs + +--ATTACH io_uring_vfs_from_file('iouring.db') AS "iouring0"; +--SELECT io_uring_vfs_from_file('iouring.db'); + +ATTACH 'file:iouring.db?vfs=iouring' as iouring; + +.open "iouring.db" + +.vfslist + +CREATE TABLE IF NOT EXISTS t3(x varchar(10), y integer); + +INSERT INTO t3 VALUES('a', 4), + ('b', 5), + ('c', 3), + ('d', 8), + ('e', 1); diff --git a/benchmarks/vfs/io_uring/run-hyperfine.sh b/benchmarks/vfs/io_uring/run-hyperfine.sh new file mode 100644 index 0000000..840514a --- /dev/null +++ b/benchmarks/vfs/io_uring/run-hyperfine.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +which hyperfine || cargo install --locked hyperfine +cargo build --examples + +mkdir -p db md + +# echo "journal" + +# # mem_vfs io_uring_vfs unix_vfs (latter 2 with default rollback journal) +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_1" "./target/debug/examples/test_1 db/test_1.db" "./target/debug/examples/test_1 db/test_1.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_2" "./target/debug/examples/test_2 db/test_2.db" "./target/debug/examples/test_2 db/test_2.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_3" "./target/debug/examples/test_3 db/test_3.db" "./target/debug/examples/test_3 db/test_3.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_4" "./target/debug/examples/test_4 db/test_4.db" "./target/debug/examples/test_4 db/test_4.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_5" "./target/debug/examples/test_5 db/test_5.db" "./target/debug/examples/test_5 db/test_5.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_6" "./target/debug/examples/test_6 db/test_6.db" "./target/debug/examples/test_6 db/test_6.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_7" "./target/debug/examples/test_7 db/test_7.db" "./target/debug/examples/test_7 db/test_7.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_8" "./target/debug/examples/test_8 db/test_8.db" "./target/debug/examples/test_8 db/test_8.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_9" "./target/debug/examples/test_9 db/test_9.db" "./target/debug/examples/test_9 db/test_9.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_10" "./target/debug/examples/test_10 db/test_10.db" "./target/debug/examples/test_10 db/test_10.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_11" "./target/debug/examples/test_11 db/test_11.db" "./target/debug/examples/test_11 db/test_11.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_12" "./target/debug/examples/test_12 db/test_12.db" "./target/debug/examples/test_12 db/test_12.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_13" "./target/debug/examples/test_13 db/test_13.db" "./target/debug/examples/test_13 db/test_13.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_14" "./target/debug/examples/test_14 db/test_14.db" "./target/debug/examples/test_14 db/test_14.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_15" "./target/debug/examples/test_15 db/test_15.db" "./target/debug/examples/test_15 db/test_15.ring.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_16" "./target/debug/examples/test_16 db/test_16.db" "./target/debug/examples/test_16 db/test_16.ring.db" + +# echo "wal" + +# # mem_vfs io_uring_vfs+wal unix_vfs+wal +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_1" "./target/debug/examples/test_1 db/test_1.wal.db" "./target/debug/examples/test_1 db/test_1.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_2" "./target/debug/examples/test_2 db/test_2.wal.db" "./target/debug/examples/test_2 db/test_2.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_3" "./target/debug/examples/test_3 db/test_3.wal.db" "./target/debug/examples/test_3 db/test_3.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_4" "./target/debug/examples/test_4 db/test_4.wal.db" "./target/debug/examples/test_4 db/test_4.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_5" "./target/debug/examples/test_5 db/test_5.wal.db" "./target/debug/examples/test_5 db/test_5.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_6" "./target/debug/examples/test_6 db/test_6.wal.db" "./target/debug/examples/test_6 db/test_6.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_7" "./target/debug/examples/test_7 db/test_7.wal.db" "./target/debug/examples/test_7 db/test_7.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_8" "./target/debug/examples/test_8 db/test_8.wal.db" "./target/debug/examples/test_8 db/test_8.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_9" "./target/debug/examples/test_9 db/test_9.wal.db" "./target/debug/examples/test_9 db/test_9.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_10" "./target/debug/examples/test_10 db/test_10.wal.db" "./target/debug/examples/test_10 db/test_10.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_11" "./target/debug/examples/test_11 db/test_11.wal.db" "./target/debug/examples/test_11 db/test_11.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_12" "./target/debug/examples/test_12 db/test_12.wal.db" "./target/debug/examples/test_12 db/test_12.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_13" "./target/debug/examples/test_13 db/test_13.wal.db" "./target/debug/examples/test_13 db/test_13.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_14" "./target/debug/examples/test_14 db/test_14.wal.db" "./target/debug/examples/test_14 db/test_14.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_15" "./target/debug/examples/test_15 db/test_15.wal.db" "./target/debug/examples/test_15 db/test_15.ring.wal.db" +# hyperfine --show-output --warmup 3 "./target/debug/examples/test_16" "./target/debug/examples/test_16 db/test_16.wal.db" "./target/debug/examples/test_16 db/test_16.ring.db" + +echo "everything together" + +# mem_vfs io_uring_vfs+rollback unix_vfs+rollback io_uring_vfs+wal unix_vfs+wal +hyperfine -N --export-markdown md/test_1.md --show-output --warmup 3 "./target/debug/examples/test_1" "./target/debug/examples/test_1 db/test_1.db" "./target/debug/examples/test_1 db/test_1.ring.db" "./target/debug/examples/test_1 db/test_1.ring.wal.db" "./target/debug/examples/test_1 db/test_1.wal.db" +hyperfine -N --export-markdown md/test_2.md --show-output --warmup 3 "./target/debug/examples/test_2" "./target/debug/examples/test_2 db/test_2.db" "./target/debug/examples/test_2 db/test_2.ring.db" "./target/debug/examples/test_2 db/test_2.ring.wal.db" "./target/debug/examples/test_2 db/test_2.wal.db" +hyperfine -N --export-markdown md/test_3.md --show-output --warmup 3 "./target/debug/examples/test_3" "./target/debug/examples/test_3 db/test_3.db" "./target/debug/examples/test_3 db/test_3.ring.db" "./target/debug/examples/test_3 db/test_3.ring.wal.db" "./target/debug/examples/test_3 db/test_3.wal.db" +hyperfine -N --export-markdown md/test_4.md --show-output --warmup 3 "./target/debug/examples/test_4" "./target/debug/examples/test_4 db/test_4.db" "./target/debug/examples/test_4 db/test_4.ring.db" "./target/debug/examples/test_4 db/test_4.ring.wal.db" "./target/debug/examples/test_4 db/test_4.wal.db" +hyperfine -N --export-markdown md/test_5.md --show-output --warmup 3 "./target/debug/examples/test_5" "./target/debug/examples/test_5 db/test_5.db" "./target/debug/examples/test_5 db/test_5.ring.db" "./target/debug/examples/test_5 db/test_5.ring.wal.db" "./target/debug/examples/test_5 db/test_5.wal.db" +hyperfine -N --export-markdown md/test_6.md --show-output --warmup 3 "./target/debug/examples/test_6" "./target/debug/examples/test_6 db/test_6.db" "./target/debug/examples/test_6 db/test_6.ring.db" "./target/debug/examples/test_6 db/test_6.ring.wal.db" "./target/debug/examples/test_6 db/test_6.wal.db" +hyperfine -N --export-markdown md/test_7.md --show-output --warmup 3 "./target/debug/examples/test_7" "./target/debug/examples/test_7 db/test_7.db" "./target/debug/examples/test_7 db/test_7.ring.db" "./target/debug/examples/test_7 db/test_7.ring.wal.db" "./target/debug/examples/test_7 db/test_7.wal.db" +hyperfine -N --export-markdown md/test_8.md --show-output --warmup 3 "./target/debug/examples/test_8" "./target/debug/examples/test_8 db/test_8.db" "./target/debug/examples/test_8 db/test_8.ring.db" "./target/debug/examples/test_8 db/test_8.ring.wal.db" "./target/debug/examples/test_8 db/test_8.wal.db" +hyperfine -N --export-markdown md/test_9.md --show-output --warmup 3 "./target/debug/examples/test_9" "./target/debug/examples/test_9 db/test_9.db" "./target/debug/examples/test_9 db/test_9.ring.db" "./target/debug/examples/test_9 db/test_9.ring.wal.db" "./target/debug/examples/test_9 db/test_9.wal.db" +hyperfine -N --export-markdown md/test_10.md --show-output --warmup 3 "./target/debug/examples/test_10" "./target/debug/examples/test_10 db/test_10.db" "./target/debug/examples/test_10 db/test_10.ring.db" "./target/debug/examples/test_10 db/test_10.ring.wal.db" "./target/debug/examples/test_10 db/test_10.wal.db" +hyperfine -N --export-markdown md/test_11.md --show-output --warmup 3 "./target/debug/examples/test_11" "./target/debug/examples/test_11 db/test_11.db" "./target/debug/examples/test_11 db/test_11.ring.db" "./target/debug/examples/test_11 db/test_11.ring.wal.db" "./target/debug/examples/test_11 db/test_11.wal.db" +hyperfine -N --export-markdown md/test_12.md --show-output --warmup 3 "./target/debug/examples/test_12" "./target/debug/examples/test_12 db/test_12.db" "./target/debug/examples/test_12 db/test_12.ring.db" "./target/debug/examples/test_12 db/test_12.ring.wal.db" "./target/debug/examples/test_12 db/test_12.wal.db" +hyperfine -N --export-markdown md/test_13.md --show-output --warmup 3 "./target/debug/examples/test_13" "./target/debug/examples/test_13 db/test_13.db" "./target/debug/examples/test_13 db/test_13.ring.db" "./target/debug/examples/test_13 db/test_13.ring.wal.db" "./target/debug/examples/test_13 db/test_13.wal.db" +hyperfine -N --export-markdown md/test_14.md --show-output --warmup 3 "./target/debug/examples/test_14" "./target/debug/examples/test_14 db/test_14.db" "./target/debug/examples/test_14 db/test_14.ring.db" "./target/debug/examples/test_14 db/test_14.ring.wal.db" "./target/debug/examples/test_14 db/test_14.wal.db" +hyperfine -N --export-markdown md/test_15.md --show-output --warmup 3 "./target/debug/examples/test_15" "./target/debug/examples/test_15 db/test_15.db" "./target/debug/examples/test_15 db/test_15.ring.db" "./target/debug/examples/test_15 db/test_15.ring.wal.db" "./target/debug/examples/test_15 db/test_15.wal.db" +hyperfine -N --export-markdown md/test_16.md --show-output --warmup 3 "./target/debug/examples/test_16" "./target/debug/examples/test_16 db/test_16.db" "./target/debug/examples/test_16 db/test_16.ring.db" "./target/debug/examples/test_16 db/test_16.ring.wal.db" "./target/debug/examples/test_16 db/test_16.wal.db" + +# for i in `seq 16`; do +# echo "hyperfine --export-markdown md/test_${i}.md --show-output --warmup 3 \"./target/debug/examples/test_$i\" \"./target/debug/examples/test_$i db/test_${i}.db\" \"./target/debug/examples/test_$i db/test_${i}.ring.db\" \"./target/debug/examples/test_$i db/test_${i}.ring.wal.db\" \"./target/debug/examples/test_$i db/test_${i}.wal.db\""; \ +# done + diff --git a/benchmarks/vfs/io_uring/src/lib.rs b/benchmarks/vfs/io_uring/src/lib.rs new file mode 100644 index 0000000..041fc2a --- /dev/null +++ b/benchmarks/vfs/io_uring/src/lib.rs @@ -0,0 +1,234 @@ +#![allow(unused)] +pub mod lock; +pub mod open; +pub mod ops; + +use io_uring::IoUring; +use libc::name_t; + +use sqlite_loadable::ext::{ + sqlite3_file, sqlite3_io_methods, sqlite3_syscall_ptr, sqlite3_vfs, + sqlite3ext_context_db_handle, sqlite3ext_database_file_object, sqlite3ext_file_control, + sqlite3ext_vfs_find, sqlite3ext_vfs_register, +}; + +use sqlite_loadable::vfs::shim::{ShimFile, ShimVfs}; +use sqlite_loadable::vfs::vfs::create_vfs; + +use sqlite_loadable::vfs::file::{create_io_methods_boxed, prepare_file_ptr, FileWithAux}; +use sqlite_loadable::{ + api, define_scalar_function, prelude::*, register_boxed_vfs, vfs::traits::SqliteVfs, + SqliteIoMethods, +}; + +use std::cell::RefCell; +use std::ffi::{CStr, CString}; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::mem::MaybeUninit; +use std::os::raw::{c_char, c_void}; +use std::rc::Rc; +use std::sync::Arc; +use std::{mem, ptr}; + +// use sqlite3ext_sys::{sqlite3_file, sqlite3_io_methods, sqlite3_syscall_ptr, sqlite3_vfs}; +use sqlite3ext_sys::{SQLITE_CANTOPEN, SQLITE_IOERR_DELETE, SQLITE_OPEN_MAIN_DB, SQLITE_OPEN_WAL}; + +use std::io::{Error, ErrorKind, Result}; + +use crate::ops::OpsFd; + +pub const EXTENSION_NAME: &str = "iouring"; +pub const RING_SIZE: u32 = 32; + +// TODO alternate implementation: write to mmap + +struct IoUringVfs { + default_vfs: ShimVfs, + vfs_name: CString, + ring: Rc>, +} + +impl SqliteVfs for IoUringVfs { + fn open( + &mut self, + z_name: *const c_char, + p_file: *mut sqlite3_file, + flags: i32, + p_res_out: *mut i32, + ) -> Result<()> { + let mut uring_ops = OpsFd::from_rc_refcell_ring(z_name as *mut _, self.ring.clone()); + + uring_ops.open_file()?; + + unsafe { prepare_file_ptr(p_file, uring_ops) }; + + Ok(()) + } + + fn delete(&mut self, z_name: *const c_char, sync_dir: i32) -> Result<()> { + log::trace!("delete"); + + let f = unsafe { CStr::from_ptr(z_name) }; + + let file_path_str = f.to_str().expect("invalid UTF-8 string"); + + if let Ok(metadata) = fs::metadata(std::path::Path::new(file_path_str)) { + if metadata.is_file() { + self.default_vfs.delete(z_name, sync_dir)?; + } else { + return Err(Error::new( + ErrorKind::NotFound, + "pointer did not refer to valid file", + )); + } + } else { + return Err(Error::new( + ErrorKind::NotFound, + "failed to fetch metadata on file", + )); + } + + Ok(()) + } + + fn access(&mut self, z_name: *const c_char, flags: i32, p_res_out: *mut i32) -> Result<()> { + log::trace!("access, flags {}", flags); + + self.default_vfs.access(z_name, flags, p_res_out) + } + + fn full_pathname( + &mut self, + z_name: *const c_char, + n_out: i32, + z_out: *mut c_char, + ) -> Result<()> { + log::trace!("full_pathname"); + + let name = unsafe { CStr::from_ptr(z_name) }; + let src_ptr = name.as_ptr(); + let dst_ptr = z_out; + let len = name.to_bytes_with_nul().len(); + unsafe { ptr::copy_nonoverlapping(src_ptr, dst_ptr.cast(), len) }; + + Ok(()) + } + + /// From here onwards, all calls are redirected to the default vfs, e.g. default unix vfs + // fn dl_open(&mut self, z_filename: *const c_char) -> *mut c_void { + // self.default_vfs.dl_open(z_filename) + // } + + // fn dl_error(&mut self, n_byte: i32, z_err_msg: *mut c_char) { + // self.default_vfs.dl_error(n_byte, z_err_msg) + // } + + // fn dl_sym(&mut self, arg2: *mut c_void, z_symbol: *const c_char) + // -> Option { + // self.default_vfs.dl_sym(arg2, z_symbol) + // } + + // fn dl_close(&mut self, arg2: *mut c_void) { + // self.default_vfs.dl_close(arg2) + // } + + fn randomness(&mut self, n_byte: i32, z_out: *mut c_char) -> i32 { + log::trace!("randomness"); + self.default_vfs.randomness(n_byte, z_out) + } + + fn sleep(&mut self, microseconds: i32) -> i32 { + log::trace!("sleep"); + self.default_vfs.sleep(microseconds) + } + + fn current_time(&mut self, arg2: *mut f64) -> i32 { + log::trace!("current_time"); + self.default_vfs.current_time(arg2) + } + + fn get_last_error(&mut self, arg2: i32, arg3: *mut c_char) -> Result<()> { + if !arg3.is_null() { + let cstr = unsafe { CStr::from_ptr(arg3) }; + let err_str = cstr.to_string_lossy(); + log::trace!("get_last_error: {}", err_str); + } else { + log::trace!("get_last_error"); + } + self.default_vfs.get_last_error(arg2, arg3) + } + + fn current_time_int64(&mut self, arg2: *mut i64) -> i32 { + log::trace!("current_time_int64"); + self.default_vfs.current_time_int64(arg2) + } + + // fn set_system_call(&mut self, z_name: *const c_char, arg2: sqlite3_syscall_ptr) -> i32 { + // self.default_vfs.set_system_call(z_name, arg2) + // } + + // fn get_system_call(&mut self, z_name: *const c_char) -> sqlite3_syscall_ptr { + // self.default_vfs.get_system_call(z_name) + // } + + // fn next_system_call(&mut self, z_name: *const c_char) -> *const c_char { + // self.default_vfs.next_system_call(z_name) + // } +} + +/// Usage: "ATTACH io_uring_vfs_from_file('test.db') AS inring;" +fn vfs_from_file( + context: *mut sqlite3_context, + values: &[*mut sqlite3_value], +) -> sqlite_loadable::Result<()> { + let path = api::value_text(&values[0])?; + + let text_output = format!("file:{}?vfs={}", path, EXTENSION_NAME); + + api::result_text(context, text_output); + + Ok(()) +} + +// See Cargo.toml "[[lib]] name = ..." matches this function name +#[sqlite_entrypoint] +pub fn sqlite3_iouringvfs_init(db: *mut sqlite3) -> sqlite_loadable::Result<()> { + let vfs_name = CString::new(EXTENSION_NAME).expect("should be fine"); + + let shimmed_name = CString::new("unix").expect("cannot find the default linux vfs"); + let shimmed_vfs_char = shimmed_name.as_ptr() as *const c_char; + let shimmed_vfs = unsafe { sqlite3ext_vfs_find(shimmed_vfs_char) }; + + let mut ring = Rc::new(RefCell::new( + IoUring::new(RING_SIZE).expect("unable to create a ring"), + )); + + let ring_vfs = IoUringVfs { + default_vfs: unsafe { + // pass thru + ShimVfs::from_ptr(shimmed_vfs) + }, + vfs_name, + ring, + }; + + // allocation is bound to lifetime of struct + let name_ptr = ring_vfs.vfs_name.as_ptr(); + + // vfs_file_size == 0, fixes the stack smash, when Box does the clean up + let vfs: sqlite3_vfs = create_vfs( + ring_vfs, + name_ptr, + 1024, + // sqlite3 has ownership and thus manages the memory + std::mem::size_of::>() as i32, + ); + + register_boxed_vfs(vfs, false)?; + + let flags = FunctionFlags::UTF8 | FunctionFlags::DETERMINISTIC; + define_scalar_function(db, "io_uring_vfs_from_file", 1, vfs_from_file, flags)?; + + Ok(()) +} diff --git a/benchmarks/vfs/io_uring/src/lock/kind.rs b/benchmarks/vfs/io_uring/src/lock/kind.rs new file mode 100644 index 0000000..9c2a668 --- /dev/null +++ b/benchmarks/vfs/io_uring/src/lock/kind.rs @@ -0,0 +1,53 @@ +use strum::FromRepr; + +/// The access an object is opened with. +#[derive(FromRepr, Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum LockKind { + /// No locks are held. The database may be neither read nor written. Any internally cached data + /// is considered suspect and subject to verification against the database file before being + /// used. Other processes can read or write the database as their own locking states permit. + /// This is the default state. + None = 0, + + /// The database may be read but not written. Any number of processes can hold + /// [LockKind::Shared] locks at the same time, hence there can be many simultaneous readers. But + /// no other thread or process is allowed to write to the database file while one or more + /// [LockKind::Shared] locks are active. + Shared = 1, + + /// A [LockKind::Reserved] lock means that the process is planning on writing to the database + /// file at some point in the future but that it is currently just reading from the file. Only a + /// single [LockKind::Reserved] lock may be active at one time, though multiple + /// [LockKind::Shared] locks can coexist with a single [LockKind::Reserved] lock. + /// [LockKind::Reserved] differs from [LockKind::Pending] in that new [LockKind::Shared] locks + /// can be acquired while there is a [LockKind::Reserved] lock. + Reserved = 2, + + /// A [LockKind::Pending] lock means that the process holding the lock wants to write to the + /// database as soon as possible and is just waiting on all current [LockKind::Shared] locks to + /// clear so that it can get an [LockKind::Exclusive] lock. No new [LockKind::Shared] locks are + /// permitted against the database if a [LockKind::Pending] lock is active, though existing + /// [LockKind::Shared] locks are allowed to continue. + Pending = 3, + + /// An [LockKind::Exclusive] lock is needed in order to write to the database file. Only one + /// [LockKind::Exclusive] lock is allowed on the file and no other locks of any kind are allowed + /// to coexist with an [LockKind::Exclusive] lock. In order to maximize concurrency, SQLite + /// works to minimize the amount of time that [LockKind::Exclusive] locks are held. + Exclusive = 4, +} + +impl PartialOrd for LockKind { + fn partial_cmp(&self, other: &Self) -> Option { + let i: i32 = *self as i32; + let o: i32 = *other as i32; + i.partial_cmp(&o) + } +} + +impl Default for LockKind { + fn default() -> Self { + Self::None + } +} diff --git a/benchmarks/vfs/io_uring/src/lock/lock.rs b/benchmarks/vfs/io_uring/src/lock/lock.rs new file mode 100644 index 0000000..625d952 --- /dev/null +++ b/benchmarks/vfs/io_uring/src/lock/lock.rs @@ -0,0 +1,225 @@ +use std::fs::{File, OpenOptions}; +use std::os::unix::fs::MetadataExt; +use std::os::unix::io::{AsRawFd, IntoRawFd, RawFd}; +use std::os::unix::prelude::FromRawFd; +use std::path::Path; +use std::{env, io}; + +use super::kind::LockKind; + +/// SQLite's default locking on UNIX systems is quite involved to work around certain limitations +/// of POSIX locks. See https://github.com/sqlite/sqlite/blob/master/src/os_unix.c#L1026-L1114 for +/// details. +/// +/// Since I don't want to re-implement that, I am going with something simpler which should suffice +/// the use-case of a VFS only used for tests. The locking uses BSD locks instead of POSIX locks. +/// Since SQLite has five different lock states, one single BSD lock is not enough (one BSD lock is +/// either unlocked, shared or exclusive). This is why each database lock consists out of two BSD +/// locks. They work as follows to achieve the SQLite lock states: +/// +/// | {name}.db /tmp/{ino}.lck +/// +/// unlocked unlocked unlocked +/// +/// shared shared shared -> unlocked ¹ +/// +/// reserved shared exclusive -> shared ² +/// +/// pending shared exclusive +/// +/// exclusive exclusive exclusive +/// +/// +/// ¹ The shared lock is first acquired, but then unlocked again. The shared lock is not kept to +/// allow the creation of an exclusive lock for a reserved lock. +/// +/// ² The reserved lock must still allow new shared locks, which is why it is only tested for +/// exclusivity first (to make sure that there is neither a pending nor exclusive lock) and then +/// downgraded to a shared lock. Keeping the shared lock prevents any other reserved lock as there +/// can only be one. + +pub struct Lock { + fd1: RawFd, + fd1_owned: bool, + fd2: RawFd, + current: LockKind, +} + +impl Lock { + pub fn new(path: impl AsRef) -> io::Result { + let path = path.as_ref(); + let f1 = File::open(path)?; + + let f2 = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(env::temp_dir().join(format!("{}.lck", f1.metadata()?.ino())))?; + + Ok(Lock { + fd1: f1.into_raw_fd(), + fd1_owned: true, + fd2: f2.into_raw_fd(), + current: LockKind::None, + }) + } + + pub fn current(&self) -> LockKind { + self.current + } + + pub fn reserved(&self) -> bool { + if self.current > LockKind::Shared { + return true; + } + + if flock_exclusive(self.fd2) { + flock_unlock(self.fd2); + false + } else { + true + } + } + + /// Transition the lock to the given [LockKind]. + /// + /// # Panics + /// + /// Panics for invalid lock transitions or failed unlocks (which are not expected to happen). + pub fn lock(&mut self, to: LockKind) -> bool { + if self.current == to { + return true; + } + + // Never move from unlocked to anything higher than shared + if self.current == LockKind::None && to != LockKind::Shared { + panic!( + "cannot transition from unlocked to anything higher than shared (tried: {:?})", + to + ) + } + + match to { + LockKind::None => { + flock_unlock(self.fd1); + + if matches!(self.current, LockKind::Pending | LockKind::Exclusive) { + flock_unlock(self.fd2); + } + + self.current = LockKind::None; + + return true; + } + + LockKind::Shared => { + if self.current != LockKind::Reserved { + if !flock_shared(self.fd1) { + return false; + } + } + + if flock_shared(self.fd2) { + flock_unlock(self.fd2); + self.current = LockKind::Shared; + true + } else if matches!(self.current, LockKind::Pending | LockKind::Exclusive) { + panic!("failed to transition to shared from {:?}", self.current); + } else if self.current == LockKind::None { + flock_unlock(self.fd1); + false + } else { + false + } + } + + LockKind::Reserved => { + // A shared lock is always held when a reserved lock is requested + if self.current != LockKind::Shared { + panic!( + "must hold a shared lock when requesting a reserved lock (current: {:?})", + self.current + ) + } + + if flock_exclusive(self.fd2) { + flock_shared(self.fd2); + self.current = LockKind::Reserved; + true + } else { + false + } + } + + LockKind::Pending => { + panic!("cannot explicitly request pending lock (request explicit lock instead)") + } + + LockKind::Exclusive => { + if self.current != LockKind::Pending && !flock_exclusive(self.fd2) { + return false; + } + + if !flock_exclusive(self.fd1) { + self.current = LockKind::Pending; + return true; + } + + self.current = LockKind::Exclusive; + true + } + } + } +} + +fn flock_unlock(fd: RawFd) { + unsafe { + if libc::flock(fd, libc::LOCK_UN | libc::LOCK_NB) != 0 { + panic!("unlock failed: {}", std::io::Error::last_os_error()); + } + } +} + +fn flock_shared(fd: RawFd) -> bool { + unsafe { + if libc::flock(fd, libc::LOCK_SH | libc::LOCK_NB) == 0 { + return true; + } + } + + let err = std::io::Error::last_os_error(); + if err.raw_os_error().unwrap() == libc::EWOULDBLOCK { + return false; + } + + panic!("lock shared failed: {}", err); +} + +fn flock_exclusive(fd: RawFd) -> bool { + unsafe { + if libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) == 0 { + return true; + } + } + + let err = std::io::Error::last_os_error(); + if err.raw_os_error().unwrap() == libc::EWOULDBLOCK { + return false; + } + + panic!("lock exclusive failed: {}", err); +} + +impl Drop for Lock { + fn drop(&mut self) { + self.lock(LockKind::None); + + // Close file descriptors. + unsafe { + if self.fd1_owned { + File::from_raw_fd(self.fd1); + } + File::from_raw_fd(self.fd2); + } + } +} diff --git a/benchmarks/vfs/io_uring/src/lock/mod.rs b/benchmarks/vfs/io_uring/src/lock/mod.rs new file mode 100644 index 0000000..a442619 --- /dev/null +++ b/benchmarks/vfs/io_uring/src/lock/mod.rs @@ -0,0 +1,5 @@ +mod kind; +mod lock; + +pub use self::kind::LockKind; +pub use self::lock::Lock; diff --git a/benchmarks/vfs/io_uring/src/open.rs b/benchmarks/vfs/io_uring/src/open.rs new file mode 100644 index 0000000..264a4ef --- /dev/null +++ b/benchmarks/vfs/io_uring/src/open.rs @@ -0,0 +1,77 @@ +use std::borrow::Cow; +use std::collections::HashMap; +use std::ffi::{c_void, CStr, CString}; +use std::io::ErrorKind; +use std::mem::{size_of, ManuallyDrop, MaybeUninit}; +use std::ops::Range; +use std::os::raw::{c_char, c_int}; +use std::pin::Pin; +use std::ptr::null_mut; +use std::slice; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use strum::FromRepr; + +#[derive(Debug, Clone, PartialEq)] +pub struct OpenOptions { + /// The object type that is being opened. + pub kind: OpenKind, + + /// The access an object is opened with. + pub access: OpenAccess, +} + +/* +SQLITE_OPEN_MEMORY: i32 = 128; +SQLITE_OPEN_MAIN_DB: i32 = 256; +SQLITE_OPEN_TEMP_DB: i32 = 512; +SQLITE_OPEN_TRANSIENT_DB: i32 = 1024; +SQLITE_OPEN_MAIN_JOURNAL: i32 = 2048; +SQLITE_OPEN_TEMP_JOURNAL: i32 = 4096; +SQLITE_OPEN_SUBJOURNAL: i32 = 8192; +SQLITE_OPEN_SUPER_JOURNAL: i32 = 16384; +SQLITE_OPEN_NOMUTEX: i32 = 32768; +SQLITE_OPEN_FULLMUTEX: i32 = 65536; +SQLITE_OPEN_SHAREDCACHE: i32 = 131072; +SQLITE_OPEN_PRIVATECACHE: i32 = 262144; +SQLITE_OPEN_WAL: i32 = 524288; +SQLITE_OPEN_NOFOLLOW: i32 = 16777216; +SQLITE_OPEN_MASTER_JOURNAL: i32 = 16384; +*/ + +/// The object type that is being opened. +#[derive(FromRepr, Debug, Clone, Copy, PartialEq)] +#[repr(i32)] +pub enum OpenKind { + MainDb = 256, // SQLITE_OPEN_MAIN_DB, + MainJournal = 2048, // SQLITE_OPEN_MAIN_JOURNAL + TempDb = 512, // SQLITE_OPEN_TEMP_DB + TempJournal = 4096, // SQLITE_OPEN_TEMP_JOURNAL + TransientDb = 1024, // SQLITE_OPEN_TRANSIENT_DB + SubJournal = 8192, // SQLITE_OPEN_SUBJOURNAL + SuperJournal = 16384, // SQLITE_OPEN_SUPER_JOURNAL / SQLITE_OPEN_MASTER_JOURNAL + Wal = 524288, // SQLITE_OPEN_WAL +} + +/* +pub const SQLITE_OPEN_READONLY: i32 = 1; +pub const SQLITE_OPEN_READWRITE: i32 = 2; +pub const SQLITE_OPEN_CREATE: i32 = 4; +*/ +/// The access an object is opened with. +#[derive(FromRepr, Debug, Clone, Copy, PartialEq)] +#[repr(i32)] +pub enum OpenAccess { + /// Read access. + Read = 1, + + /// Write access (includes read access). + Write = 2, + + /// Create the file if it does not exist (includes write and read access). + Create = 6, + + /// Create the file, but throw if it it already exist (includes write and read access). + CreateNewThrowIfExists = 8, +} diff --git a/benchmarks/vfs/io_uring/src/ops/fd.rs b/benchmarks/vfs/io_uring/src/ops/fd.rs new file mode 100644 index 0000000..d6f6e2e --- /dev/null +++ b/benchmarks/vfs/io_uring/src/ops/fd.rs @@ -0,0 +1,493 @@ +use std::borrow::BorrowMut; +use std::cell::RefCell; +use std::ffi::{CStr, CString}; +use std::fs::File; +use std::os::fd::RawFd; +use std::os::raw::c_void; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::io::{AsRawFd, FromRawFd}; +use std::rc::Rc; + +use io_uring::types::Fd; +use libc::c_char; +use sqlite3ext_sys::{ + SQLITE_IOCAP_ATOMIC, SQLITE_IOCAP_POWERSAFE_OVERWRITE, SQLITE_IOCAP_SAFE_APPEND, + SQLITE_IOCAP_SEQUENTIAL, +}; +use sqlite3ext_sys::{SQLITE_IOERR_SHMLOCK, SQLITE_IOERR_SHMMAP}; +use sqlite_loadable::SqliteIoMethods; +use std::io::{Error, ErrorKind, Result}; + +use sqlite3ext_sys::{SQLITE_BUSY, SQLITE_LOCK_SHARED, SQLITE_OK}; + +// IO Uring errors: https://codebrowser.dev/linux/linux/include/uapi/asm-generic/errno-base.h.html + +use sqlite_loadable::ext::{sqlite3_file, sqlite3ext_vfs_find}; +use sqlite_loadable::vfs::shim::{ShimFile, ShimVfs}; +use std::{mem, ptr}; + +use io_uring::{opcode, register, types, IoUring}; +use std::io; + +use crate::lock::Lock; +use crate::lock::LockKind; + +const USER_DATA_OPEN: u64 = 0x1; +const USER_DATA_READ: u64 = 0x2; +const USER_DATA_STATX: u64 = 0x3; +const USER_DATA_WRITE: u64 = 0x4; +const USER_DATA_FALLOCATE: u64 = 0x5; +const USER_DATA_CLOSE: u64 = 0x6; +const USER_DATA_FSYNC: u64 = 0x7; + +// Tested on linux 5.15.49, 6.1.0, 6.3.13 +pub struct OpsFd { + ring: Rc>, + file_path: *const char, + file_fd: Option, + file: Option, + lock: Option, + file_name: String, +} + +/// I was tempted really often to convert file_path to Path with a lifetime, PathBuf, CString +/// but it quickly becomes awkward due to libc insistence on read pointers. +/// A purely oxidized project should work with Path. +/// Besides, the pointer memset that file_path is stored, is managed by C, +/// bad things will happen to sqlite3 if you try to take away ownership. +impl OpsFd { + // Used for tests + pub fn new(file_path: *const char, ring_size: u32) -> Self { + let mut ring = Rc::new(RefCell::new( + IoUring::new(ring_size).expect("unable to create a ring"), + )); + + Self::from_rc_refcell_ring(file_path, ring) + } + + pub fn from_rc_refcell_ring(file_path: *const char, ring: Rc>) -> Self { + OpsFd { + ring, + file_path, + file_fd: None, + file: None, + lock: None, + file_name: unsafe { + CStr::from_ptr(file_path as *const _) + .to_str() + .expect("invalid utf8") + .to_string() + }, + } + } + + // all tests pass + pub fn open_file(&mut self) -> Result<()> { + // This calls libc::open + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(self.file_name.as_str())?; + + let raw_fd = file.as_raw_fd(); + + self.file = Some(file); + self.file_fd = Some(raw_fd); + + Ok(()) + } + + // tests pass except anything that writes + /* + pub fn open_file(&mut self) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let dirfd = types::Fd(libc::AT_FDCWD); + + let flags = libc::O_CREAT as u64; + + let openhow = types::OpenHow::new() + .flags(flags) + .mode(libc::S_IRUSR as u64 | libc::S_IWUSR as u64); + + let open_e: opcode::OpenAt2 = + opcode::OpenAt2::new(dirfd, self.file_path as *const _, &openhow); + + unsafe { + ring.submission() + .push(&open_e.build().user_data(USER_DATA_OPEN)) + .expect("queue is full"); + } + + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("open_file: raw os error result: {}", -result as i32), + )) + } else { + self.file_fd = Some(result); + Ok(()) + } + } + */ + + pub unsafe fn o_read(&mut self, offset: u64, size: u32, buf_out: *mut c_void) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fd(self.file_fd.expect("missing fd")); + let mut op = opcode::Read::new(fd, buf_out as *mut _, size).offset(offset); + ring.submission() + .push(&op.build().user_data(USER_DATA_READ)) + .expect("queue is full"); + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("read: raw os error result: {}", -result as i32), + )) + } else { + Ok(()) + } + } + + pub unsafe fn o_write(&mut self, buf_in: *const c_void, offset: u64, size: u32) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fd(self.file_fd.expect("missing fd")); + let mut op = opcode::Write::new(fd, buf_in as _, size).offset(offset); + ring.submission() + .push(&op.build().user_data(USER_DATA_WRITE)) + .expect("queue is full"); + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("write: raw os error result: {}", -result as i32), + )) + } else { + Ok(()) + } + } + + // TODO implement with a read then write, also apply linking to guarantee read before write + // pub unsafe fn o_truncate(&mut self, size: i64) -> Result<()> { + // let mut file_size_box = Box::new(0 as u64); + // let mut file_size_ptr = Box::into_raw(file_size_box); + // self.o_file_size(file_size_ptr); + + // let mut ring = self.ring.as_ref().borrow_mut(); + + // let fd = types::Fd(self.file_fd.expect("missing fd")); + // let new_size: u64 = size.try_into().unwrap(); + // let mut op = opcode::Fallocate::new(fd, (*file_size_ptr) - new_size) + // .offset((size - 1).try_into().unwrap()) + // .mode(libc::FALLOC_FL_COLLAPSE_RANGE); + + // ring.submission() + // .push(&op.build().user_data(USER_DATA_FALLOCATE)) + // .expect("queue is full");; + + // ring.submit_and_wait(1) + // .expect("submit failed or timed out"); + + // let cqes: Vec = ring.completion().map(Into::into).collect(); + // let cqe = &cqes.as_slice()[0]; + // let result = cqe.result(); + + // Box::from_raw(file_size_ptr); + + // if result < 0 { + // Err(Error::new( + // ErrorKind::Other, + // format!("truncate: raw os error result: {}", -result as i32), + // )) + // }else { + // Ok(()) + // } + // } + + pub unsafe fn o_truncate(&mut self, size: i64) -> Result<()> { + // libc::ftruncate using self.file_fd returns -1 + let result = libc::truncate(self.file_path as *const _, size); + if result != 0 { + Err(Error::new( + ErrorKind::Other, + format!("truncate: raw os error result: {}", result), + )) + } else { + Ok(()) + } + } + + // SQLite Documentation: + // Implement this function to read data from the file at the specified offset and store it in `buf_out`. + // You can use the same pattern as in `read_file`. + pub unsafe fn o_fetch( + &mut self, + offset: u64, + size: u32, + buf_out: *mut *mut c_void, + ) -> Result<()> { + self.o_read(offset, size, *buf_out as *mut _) + } + + pub unsafe fn o_close(&mut self) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fd(self.file_fd.expect("missing fd")); + let mut op = opcode::Close::new(fd); + + ring.submission() + .push(&op.build().user_data(USER_DATA_CLOSE)) + .expect("queue is full"); + + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("close: raw os error result: {}", -result as i32), + )) + } else { + Ok(()) + } + } + + pub unsafe fn o_file_size(&mut self, out: *mut u64) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let mut statx_buf: libc::statx = unsafe { std::mem::zeroed() }; + let mut statx_buf_ptr: *mut libc::statx = &mut statx_buf; + + let dirfd = types::Fd(libc::AT_FDCWD); + let statx_op = + opcode::Statx::new(dirfd, self.file_path as *const _, statx_buf_ptr as *mut _) + .flags(libc::AT_EMPTY_PATH) + .mask(libc::STATX_ALL); + + ring.submission() + .push(&statx_op.build().user_data(USER_DATA_STATX)) + .expect("queue is full"); + + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("file_size: raw os error result: {}", -result as i32), + )) + } else { + unsafe { + *out = statx_buf.stx_size as u64; + } + + Ok(()) + } + } + + pub unsafe fn o_fsync(&mut self, flags: i32) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fd(self.file_fd.expect("missing fd")); + let op = opcode::Fsync::new(fd); + + ring.submission() + .push(&op.build().user_data(USER_DATA_FSYNC)) + .expect("queue is full"); + + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("fsync: raw os error result: {}", -result as i32), + )) + } else { + Ok(()) + } + } + + fn is_exclusive_requested_pending_acquired(&mut self, to: LockKind) -> bool { + if let Some(lock) = &mut self.lock { + lock.lock(to) && lock.current() == to + } else { + false + } + } + + fn init_lock(&mut self) -> Result<()> { + if self.lock.is_none() { + let cstr = unsafe { CStr::from_ptr(self.file_path as *const _) }; + + let str_result = cstr.to_str(); + + let err = Error::new(ErrorKind::Other, "bad file name"); + + // the fd from the ring, returns: os error 9 + let str = str_result.map_err(|_| err)?; + + let lock = Lock::new(str)?; + + self.lock = Some(lock); + } + Ok(()) + } + + pub fn lock_or_unlock(&mut self, lock_request: i32) -> Result { + self.init_lock()?; + LockKind::from_repr(lock_request) + .map(|kind| self.is_exclusive_requested_pending_acquired(kind)) + .map(|ok_or_busy| if ok_or_busy { SQLITE_OK } else { SQLITE_BUSY }) + .ok_or_else(|| Error::new(ErrorKind::Other, "Missing lock")) + } + + pub fn lock_reserved(&mut self) -> Result { + self.init_lock()?; + if let Some(lock) = &mut self.lock { + Ok(lock.reserved()) + } else { + Err(Error::new(ErrorKind::Other, "Missing lock")) + } + } +} + +// TODO remove *mut sqlite3_file +impl SqliteIoMethods for OpsFd { + fn close(&mut self) -> Result<()> { + log::trace!("file close"); + + unsafe { self.o_close() } + } + + fn read(&mut self, buf: *mut c_void, s: i32, ofst: i64) -> Result<()> { + log::trace!("file read"); + + unsafe { self.o_read(ofst as u64, s as u32, buf) } + } + + fn write(&mut self, buf: *const c_void, s: i32, ofst: i64) -> Result<()> { + log::trace!("file write"); + + unsafe { self.o_write(buf, ofst as u64, s as u32) } + } + + fn truncate(&mut self, size: i64) -> Result<()> { + log::trace!("file truncate"); + + unsafe { self.o_truncate(size) } + } + + fn sync(&mut self, flags: i32) -> Result<()> { + log::trace!("file sync"); + + unsafe { self.o_fsync(flags) } + } + + fn file_size(&mut self, p_size: *mut i64) -> Result<()> { + log::trace!("file size"); + + unsafe { self.o_file_size(p_size as *mut u64) } + } + + fn lock(&mut self, arg2: i32) -> Result { + log::trace!("file lock"); + self.lock_or_unlock(arg2) + } + + fn unlock(&mut self, arg2: i32) -> Result { + log::trace!("file unlock"); + self.lock_or_unlock(arg2) + } + + fn check_reserved_lock(&mut self, p_res_out: *mut i32) -> Result<()> { + log::trace!("file check reserved lock"); + + let lock_reserved = self.lock_reserved()?; + unsafe { + *p_res_out = if lock_reserved { 1 } else { 0 }; + } + Ok(()) + } + + /// See https://www.sqlite.org/c3ref/file_control.html + /// and also https://www.sqlite.org/c3ref/c_fcntl_begin_atomic_write.html + fn file_control(&mut self, op: i32, p_arg: *mut c_void) -> Result<()> { + log::trace!("file control"); + Ok(()) + } + + fn sector_size(&mut self) -> Result { + log::trace!("sector size"); + Ok(1024) + } + + fn device_characteristics(&mut self) -> Result { + log::trace!("device characteristics"); + let x = SQLITE_IOCAP_ATOMIC + | SQLITE_IOCAP_POWERSAFE_OVERWRITE + | SQLITE_IOCAP_SAFE_APPEND + | SQLITE_IOCAP_SEQUENTIAL; + Ok(x) + } + + fn shm_map(&mut self, i_pg: i32, pgsz: i32, arg2: i32, arg3: *mut *mut c_void) -> Result<()> { + log::trace!("shm map"); + Ok(()) + } + + fn shm_lock(&mut self, offset: i32, n: i32, flags: i32) -> Result<()> { + log::trace!("shm lock"); + Ok(()) + } + + fn shm_barrier(&mut self) -> Result<()> { + log::trace!("shm barrier"); + Ok(()) + } + + fn shm_unmap(&mut self, delete_flag: i32) -> Result<()> { + log::trace!("shm unmap"); + Ok(()) + } + + fn fetch(&mut self, ofst: i64, size: i32, pp: *mut *mut c_void) -> Result<()> { + unsafe { + log::trace!("file fetch"); + self.o_fetch(ofst as u64, size as u32, pp) + } + } + + fn unfetch(&mut self, i_ofst: i64, p: *mut c_void) -> Result<()> { + log::trace!("file unfetch"); + Ok(()) + } +} diff --git a/benchmarks/vfs/io_uring/src/ops/fixed.rs b/benchmarks/vfs/io_uring/src/ops/fixed.rs new file mode 100644 index 0000000..e07b05b --- /dev/null +++ b/benchmarks/vfs/io_uring/src/ops/fixed.rs @@ -0,0 +1,522 @@ +use std::borrow::BorrowMut; +use std::cell::RefCell; +use std::ffi::{CStr, CString}; +use std::fs::File; +use std::os::fd::RawFd; +use std::os::raw::c_void; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::io::{AsRawFd, FromRawFd}; +use std::rc::Rc; + +use io_uring::types::Fd; +use libc::c_char; +use sqlite3ext_sys::{ + SQLITE_IOCAP_ATOMIC, SQLITE_IOCAP_POWERSAFE_OVERWRITE, SQLITE_IOCAP_SAFE_APPEND, + SQLITE_IOCAP_SEQUENTIAL, +}; +use sqlite3ext_sys::{SQLITE_IOERR_SHMLOCK, SQLITE_IOERR_SHMMAP}; +use sqlite_loadable::SqliteIoMethods; +use std::io::{Error, ErrorKind, Result}; + +use sqlite3ext_sys::{SQLITE_BUSY, SQLITE_LOCK_SHARED, SQLITE_OK}; + +// IO Uring errors: https://codebrowser.dev/linux/linux/include/uapi/asm-generic/errno-base.h.html + +use sqlite_loadable::ext::{sqlite3_file, sqlite3ext_vfs_find}; +use sqlite_loadable::vfs::shim::{ShimFile, ShimVfs}; +use std::{mem, ptr}; + +use io_uring::{opcode, register, types, IoUring}; +use std::io; + +use crate::lock::Lock; +use crate::lock::LockKind; + +const USER_DATA_OPEN: u64 = 0x1; +const USER_DATA_READ: u64 = 0x2; +const USER_DATA_STATX: u64 = 0x3; +const USER_DATA_WRITE: u64 = 0x4; +const USER_DATA_FALLOCATE: u64 = 0x5; +const USER_DATA_CLOSE: u64 = 0x6; +const USER_DATA_FSYNC: u64 = 0x7; + +const FILE_INDEX_MAIN_DB: u32 = 0x0; +const FILE_INDEX_JOURNAL: u32 = 0x1; + +// Tested on linux 5.15.49, 6.1.0, 6.3.13 +pub struct OpsFixed { + ring: Rc>, + file_path: *const char, + file_index: Option, + lock: Option, + file_name: String, +} + +/// I was tempted really often to convert file_path to Path with a lifetime, PathBuf, CString +/// but it quickly becomes awkward due to libc insistence on read pointers. +/// A purely oxidized project should work with Path. +/// Besides, the pointer memset that file_path is stored, is managed by C, +/// bad things will happen to sqlite3 if you try to take away ownership. +impl OpsFixed { + // Used for tests + pub fn new(file_path: *const char, ring_size: u32) -> Self { + let mut ring = Rc::new(RefCell::new( + IoUring::new(ring_size).expect("unable to create a ring"), + )); + + Self::from_rc_refcell_ring(file_path, ring) + } + + pub fn from_rc_refcell_ring(file_path: *const char, ring: Rc>) -> Self { + OpsFixed { + ring, + file_path, + file_index: None, + lock: None, + file_name: unsafe { + CStr::from_ptr(file_path as *const _) + .to_str() + .expect("invalid utf-8") + .to_string() + }, + } + } + + fn get_file_index(&self) -> u32 { + let suffixes = vec!["-conch", "-journal", "-wal"]; // TODO investigate: what is a conch? + let ends_with_suffix = suffixes.iter().any(|s| self.file_name.ends_with(s)); + if ends_with_suffix { + FILE_INDEX_JOURNAL + } else { + FILE_INDEX_MAIN_DB + } + } + + fn get_dest_slot(&self) -> Option { + let result = types::DestinationSlot::try_from_slot_target(self.get_file_index()); + result.ok() + } + + // TODO investigate as premature optimization: add O_DIRECT and O_SYNC parameters for systems that actually support it + // TODO investigate o_TMPFILE for -journal, -wal etc. and disable vfs DELETE event + // Things I tried to avoid the -9, invalid fd, [EBADDF](https://www.javatpoint.com/linux-error-codes) + // * open twice + // * submitter().register_sparse ... 2, submitter().unregister_files() + pub fn open_file(&mut self) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + // Cleanup all fixed files (if any), then reserve two slots + let _ = ring.submitter().unregister_files(); + ring.submitter() + .register_files_sparse(2) + .expect("unable to register sparse files"); + + let dirfd = types::Fd(libc::AT_FDCWD); + + // source: https://stackoverflow.com/questions/5055859/how-are-the-o-sync-and-o-direct-flags-in-open2-different-alike + // file_size and open and close work + // let flags = libc::O_DIRECT as u64 | libc::O_SYNC as u64 | libc::O_CREAT as u64; + + // file_size and open and close work, but breaks other tests + // let flags = libc::O_DIRECT as u64 | libc::O_CREAT as u64; + + let flags = libc::O_CREAT as u64; + + let openhow = types::OpenHow::new() + .flags(flags) + .mode(libc::S_IRUSR as u64 | libc::S_IWUSR as u64); + + let open_e: opcode::OpenAt2 = + opcode::OpenAt2::new(dirfd, self.file_path as *const _, &openhow) + .file_index(self.get_dest_slot()); + + unsafe { + ring.submission() + .push(&open_e.build().user_data(USER_DATA_OPEN)) + .expect("queue is full"); + } + + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + // TODO turn on later + // unsafe { + // let path = CStr::from_ptr(self.file_path); + // log::trace!( + // "open {} with fd: {}", + // path.to_string_lossy().to_string(), + // result + // ) + // } + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("open_file: raw os error result: {}", -result as i32), + )) + } else { + // fixed + self.file_index = Some(self.get_file_index()); + Ok(()) + } + } + + pub unsafe fn o_read(&mut self, offset: u64, size: u32, buf_out: *mut c_void) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fixed(self.file_index.expect("missing fixed file index")); + let mut op = opcode::Read::new(fd, buf_out as *mut _, size).offset(offset); + ring.submission() + .push(&op.build().user_data(USER_DATA_READ)) + .expect("queue is full"); + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("read: raw os error result: {}", -result as i32), + )) + } else { + Ok(()) + } + } + + pub unsafe fn o_write(&mut self, buf_in: *const c_void, offset: u64, size: u32) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fixed(self.file_index.expect("missing fixed file index")); + let mut op = opcode::Write::new(fd, buf_in as _, size).offset(offset); + ring.submission() + .push(&op.build().user_data(USER_DATA_WRITE)) + .expect("queue is full"); + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("write: raw os error result: {}", -result as i32), + )) + } else { + Ok(()) + } + } + + /* + // This should work but it refuses the fd from open_file and returns -22 (EINVAL, invalid argument) + pub unsafe fn o_truncate(&mut self, size: i64) -> Result<()> { + let mut file_size_box = Box::new(0 as u64); + let mut file_size_ptr = Box::into_raw(file_size_box); + self.o_file_size(file_size_ptr); + + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fixed(self.file_index.expect("missing fixed file index")); + // let mut op = opcode::Fallocate::new(fd, size.try_into().unwrap()).offset(0); // before + let new_size: u64 = size.try_into().unwrap(); + let mut op = opcode::Fallocate::new(fd, (*file_size_ptr) - new_size) + .offset((size - 1).try_into().unwrap()) + .mode(libc::FALLOC_FL_COLLAPSE_RANGE); + + ring.submission() + .push(&op.build().user_data(USER_DATA_FALLOCATE)) + .expect("queue is full");; + + ring.submit_and_wait(1) + .expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + Box::from_raw(file_size_ptr); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("truncate: raw os error result: {}", -result as i32), + )) + }else { + Ok(()) + } + } + */ + + pub unsafe fn o_truncate(&mut self, size: i64) -> Result<()> { + // libc::ftruncate using self.file_fd returns -1 + let result = libc::truncate(self.file_path as *const _, size); + if result != 0 { + Err(Error::new( + ErrorKind::Other, + format!("truncate: raw os error result: {}", result), + )) + } else { + Ok(()) + } + } + + // SQLite Documentation: + // Implement this function to read data from the file at the specified offset and store it in `buf_out`. + // You can use the same pattern as in `read_file`. + pub unsafe fn o_fetch( + &mut self, + offset: u64, + size: u32, + buf_out: *mut *mut c_void, + ) -> Result<()> { + self.o_read(offset, size, *buf_out as *mut _) + } + + pub unsafe fn o_close(&mut self) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fixed(self.file_index.expect("missing fixed file index")); + let mut op = opcode::Close::new(fd); + + ring.submission() + .push(&op.build().user_data(USER_DATA_CLOSE)) + .expect("queue is full"); + + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("close: raw os error result: {}", -result as i32), + )) + } else { + Ok(()) + } + } + + pub unsafe fn o_file_size(&mut self, out: *mut u64) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let mut statx_buf: libc::statx = unsafe { std::mem::zeroed() }; + let mut statx_buf_ptr: *mut libc::statx = &mut statx_buf; + + let dirfd = types::Fd(libc::AT_FDCWD); + let statx_op = + opcode::Statx::new(dirfd, self.file_path as *const _, statx_buf_ptr as *mut _) + .flags(libc::AT_EMPTY_PATH) + .mask(libc::STATX_ALL); + + ring.submission() + .push(&statx_op.build().user_data(USER_DATA_STATX)) + .expect("queue is full"); + + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("file_size: raw os error result: {}", -result as i32), + )) + } else { + unsafe { + *out = statx_buf.stx_size as u64; + } + + Ok(()) + } + } + + pub unsafe fn o_fsync(&mut self, flags: i32) -> Result<()> { + let mut ring = self.ring.as_ref().borrow_mut(); + + let fd = types::Fixed(self.file_index.expect("missing fixed file index")); + let op = opcode::Fsync::new(fd); + + ring.submission() + .push(&op.build().user_data(USER_DATA_FSYNC)) + .expect("queue is full"); + + ring.submit_and_wait(1).expect("submit failed or timed out"); + + let cqes: Vec = ring.completion().map(Into::into).collect(); + let cqe = &cqes.as_slice()[0]; + let result = cqe.result(); + + if result < 0 { + Err(Error::new( + ErrorKind::Other, + format!("fsync: raw os error result: {}", -result as i32), + )) + } else { + Ok(()) + } + } + + fn is_exclusive_requested_pending_acquired(&mut self, to: LockKind) -> bool { + if let Some(lock) = &mut self.lock { + lock.lock(to) && lock.current() == to + } else { + false + } + } + + fn init_lock(&mut self) -> Result<()> { + if self.lock.is_none() { + let cstr = unsafe { CStr::from_ptr(self.file_path as *const _) }; + + let str_result = cstr.to_str(); + + let err = Error::new(ErrorKind::Other, "bad file name"); + + // the fd from the ring, returns: os error 9 + let str = str_result.map_err(|_| err)?; + + let lock = Lock::new(str)?; + + self.lock = Some(lock); + } + Ok(()) + } + + pub fn lock_or_unlock(&mut self, lock_request: i32) -> Result { + self.init_lock()?; + LockKind::from_repr(lock_request) + .map(|kind| self.is_exclusive_requested_pending_acquired(kind)) + .map(|ok_or_busy| if ok_or_busy { SQLITE_OK } else { SQLITE_BUSY }) + .ok_or_else(|| Error::new(ErrorKind::Other, "Missing lock")) + } + + pub fn lock_reserved(&mut self) -> Result { + self.init_lock()?; + if let Some(lock) = &mut self.lock { + Ok(lock.reserved()) + } else { + Err(Error::new(ErrorKind::Other, "Missing lock")) + } + } +} + +// TODO remove *mut sqlite3_file +impl SqliteIoMethods for OpsFixed { + fn close(&mut self) -> Result<()> { + log::trace!("file close"); + + unsafe { self.o_close() } + } + + fn read(&mut self, buf: *mut c_void, s: i32, ofst: i64) -> Result<()> { + log::trace!("file read"); + + unsafe { self.o_read(ofst as u64, s as u32, buf) } + } + + fn write(&mut self, buf: *const c_void, s: i32, ofst: i64) -> Result<()> { + log::trace!("file write"); + + unsafe { self.o_write(buf, ofst as u64, s as u32) } + } + + fn truncate(&mut self, size: i64) -> Result<()> { + log::trace!("file truncate"); + + unsafe { self.o_truncate(size) } + } + + fn sync(&mut self, flags: i32) -> Result<()> { + log::trace!("file sync"); + + unsafe { self.o_fsync(flags) } + } + + fn file_size(&mut self, p_size: *mut i64) -> Result<()> { + log::trace!("file size"); + + unsafe { self.o_file_size(p_size as *mut u64) } + } + + fn lock(&mut self, arg2: i32) -> Result { + log::trace!("file lock"); + self.lock_or_unlock(arg2) + } + + fn unlock(&mut self, arg2: i32) -> Result { + log::trace!("file unlock"); + self.lock_or_unlock(arg2) + } + + fn check_reserved_lock(&mut self, p_res_out: *mut i32) -> Result<()> { + log::trace!("file check reserved lock"); + + let lock_reserved = self.lock_reserved()?; + unsafe { + *p_res_out = if lock_reserved { 1 } else { 0 }; + } + Ok(()) + } + + /// See https://www.sqlite.org/c3ref/file_control.html + /// and also https://www.sqlite.org/c3ref/c_fcntl_begin_atomic_write.html + fn file_control(&mut self, op: i32, p_arg: *mut c_void) -> Result<()> { + log::trace!("file control"); + Ok(()) + } + + fn sector_size(&mut self) -> Result { + log::trace!("sector size"); + Ok(1024) + } + + fn device_characteristics(&mut self) -> Result { + log::trace!("device characteristics"); + let x = SQLITE_IOCAP_ATOMIC + | SQLITE_IOCAP_POWERSAFE_OVERWRITE + | SQLITE_IOCAP_SAFE_APPEND + | SQLITE_IOCAP_SEQUENTIAL; + Ok(x) + } + + fn shm_map(&mut self, i_pg: i32, pgsz: i32, arg2: i32, arg3: *mut *mut c_void) -> Result<()> { + log::trace!("shm map"); + Ok(()) + } + + fn shm_lock(&mut self, offset: i32, n: i32, flags: i32) -> Result<()> { + log::trace!("shm lock"); + Ok(()) + } + + fn shm_barrier(&mut self) -> Result<()> { + log::trace!("shm barrier"); + Ok(()) + } + + fn shm_unmap(&mut self, delete_flag: i32) -> Result<()> { + log::trace!("shm unmap"); + Ok(()) + } + + fn fetch(&mut self, ofst: i64, size: i32, pp: *mut *mut c_void) -> Result<()> { + unsafe { + log::trace!("file fetch"); + self.o_fetch(ofst as u64, size as u32, pp) + } + } + + fn unfetch(&mut self, i_ofst: i64, p: *mut c_void) -> Result<()> { + log::trace!("file unfetch"); + Ok(()) + } +} diff --git a/benchmarks/vfs/io_uring/src/ops/mod.rs b/benchmarks/vfs/io_uring/src/ops/mod.rs new file mode 100644 index 0000000..70a3165 --- /dev/null +++ b/benchmarks/vfs/io_uring/src/ops/mod.rs @@ -0,0 +1,9 @@ +mod fd; +mod fixed; + +// Not all IO Uring ops support fixed file indices, this is kept here for future use (>10-12-2023) +// e.g. Write does not support it. +// Fortunately, File creation and getting its raw fd is O(1), the perceived drawback +// is us not being able to use OpenAt/OpenAt2 to fetch the fd. +pub use fd::OpsFd; +pub use fixed::OpsFixed; diff --git a/benchmarks/vfs/io_uring/tests/test_locks.rs b/benchmarks/vfs/io_uring/tests/test_locks.rs new file mode 100644 index 0000000..c5e78ce --- /dev/null +++ b/benchmarks/vfs/io_uring/tests/test_locks.rs @@ -0,0 +1,181 @@ +#[cfg(test)] +mod tests { + use std::fs; + use std::path::PathBuf; + + use _iouringvfs::lock::*; + + fn test_file(name: &str) -> PathBuf { + let path = PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join(name) + .with_extension("txt"); + fs::write(&path, "").unwrap(); + path + } + + #[test] + fn test_lock_order() { + assert!(LockKind::None < LockKind::Shared); + assert!(LockKind::Shared < LockKind::Reserved); + assert!(LockKind::Reserved < LockKind::Pending); + assert!(LockKind::Pending < LockKind::Exclusive); + } + + #[test] + fn test_none() { + let path = test_file(".test_none"); + let lock = Lock::new(&path).unwrap(); + assert_eq!(lock.current(), LockKind::None); + } + + #[test] + fn test_shared() { + let path = test_file(".test_shared"); + let mut lock = Lock::new(&path).unwrap(); + assert!(lock.lock(LockKind::Shared)); + assert_eq!(lock.current(), LockKind::Shared); + } + + #[test] + fn test_reserved() { + let path = test_file(".test_reserved"); + let mut lock = Lock::new(&path).unwrap(); + assert!(lock.lock(LockKind::Shared)); + assert!(lock.lock(LockKind::Reserved)); + assert_eq!(lock.current(), LockKind::Reserved); + } + + #[test] + fn test_exclusive() { + let path = test_file(".test_exclusive"); + let mut lock = Lock::new(&path).unwrap(); + assert!(lock.lock(LockKind::Shared)); + assert!(lock.lock(LockKind::Exclusive)); + assert_eq!(lock.current(), LockKind::Exclusive); + } + + #[test] + fn test_exclusive_via_reserved() { + let path = test_file(".test_exclusive_via_reserved"); + let mut lock = Lock::new(&path).unwrap(); + assert!(lock.lock(LockKind::Shared)); + assert!(lock.lock(LockKind::Reserved)); + assert!(lock.lock(LockKind::Exclusive)); + assert_eq!(lock.current(), LockKind::Exclusive); + } + + #[test] + #[should_panic( + expected = "cannot transition from unlocked to anything higher than shared (tried: Reserved)" + )] + fn test_none_to_reserved_panic() { + let path = test_file(".test_none_to_reserved_panic"); + let mut lock = Lock::new(&path).unwrap(); + lock.lock(LockKind::Reserved); + } + + #[test] + #[should_panic( + expected = "cannot transition from unlocked to anything higher than shared (tried: Exclusive)" + )] + fn test_none_to_exclusive_panic() { + let path = test_file(".test_none_to_exclusive_panic"); + let mut lock = Lock::new(&path).unwrap(); + lock.lock(LockKind::Exclusive); + } + + #[test] + #[should_panic( + expected = "cannot explicitly request pending lock (request explicit lock instead)" + )] + fn test_shared_to_pending_panic() { + let path = test_file(".test_shared_to_pending_panic"); + let mut lock = Lock::new(&path).unwrap(); + assert!(lock.lock(LockKind::Shared)); + lock.lock(LockKind::Pending); + } + + #[test] + #[should_panic( + expected = "cannot explicitly request pending lock (request explicit lock instead)" + )] + fn test_reserved_to_pending_panic() { + let path = test_file(".test_reserved_to_pending_panic"); + let mut lock = Lock::new(&path).unwrap(); + assert!(lock.lock(LockKind::Shared)); + assert!(lock.lock(LockKind::Reserved)); + lock.lock(LockKind::Pending); + } + + #[test] + fn test_reserved_once() { + let path = test_file(".reserved_once"); + let mut lock1 = Lock::new(&path).unwrap(); + assert!(lock1.lock(LockKind::Shared)); + + let mut lock2 = Lock::new(&path).unwrap(); + assert!(lock2.lock(LockKind::Shared)); + + assert!(lock1.lock(LockKind::Reserved)); + assert!(!lock2.lock(LockKind::Reserved)); + + assert!(lock1.lock(LockKind::Shared)); + assert!(lock2.lock(LockKind::Reserved)); + } + + #[test] + fn test_shared_while_reserved() { + let path = test_file(".shared_while_reserved"); + let mut lock1 = Lock::new(&path).unwrap(); + assert!(lock1.lock(LockKind::Shared)); + assert!(lock1.lock(LockKind::Reserved)); + + let mut lock2 = Lock::new(&path).unwrap(); + assert!(lock2.lock(LockKind::Shared)); + } + + #[test] + fn test_pending() { + let path = test_file(".test_pending"); + let mut lock1 = Lock::new(&path).unwrap(); + assert!(lock1.lock(LockKind::Shared)); + + let mut lock2 = Lock::new(&path).unwrap(); + assert!(lock2.lock(LockKind::Shared)); + assert!(lock2.lock(LockKind::Exclusive)); + assert_eq!(lock2.current(), LockKind::Pending); + } + + #[test] + fn test_pending_once() { + let path = test_file(".test_pending_once"); + let mut lock1 = Lock::new(&path).unwrap(); + assert!(lock1.lock(LockKind::Shared)); + + let mut lock2 = Lock::new(&path).unwrap(); + assert!(lock2.lock(LockKind::Shared)); + assert!(lock2.lock(LockKind::Exclusive)); + + assert!(!lock1.lock(LockKind::Exclusive)); + + assert_eq!(lock1.current(), LockKind::Shared); + assert_eq!(lock2.current(), LockKind::Pending); + } + + #[test] + fn test_pending_to_exclusive() { + let path = test_file(".test_pending_to_exclusive"); + let mut lock1 = Lock::new(&path).unwrap(); + assert!(lock1.lock(LockKind::Shared)); + + let mut lock2 = Lock::new(&path).unwrap(); + assert!(lock2.lock(LockKind::Shared)); + assert!(lock2.lock(LockKind::Exclusive)); + + assert!(lock1.lock(LockKind::None)); + assert!(lock2.lock(LockKind::Exclusive)); + + assert_eq!(lock1.current(), LockKind::None); + assert_eq!(lock2.current(), LockKind::Exclusive); + } +} diff --git a/benchmarks/vfs/io_uring/tests/test_ops_fd.rs b/benchmarks/vfs/io_uring/tests/test_ops_fd.rs new file mode 100644 index 0000000..b504d6b --- /dev/null +++ b/benchmarks/vfs/io_uring/tests/test_ops_fd.rs @@ -0,0 +1,194 @@ +// Sometimes one test will fail randomly +// when the completion queue takes its sweet time +// Just run the test again + +#[cfg(test)] +mod tests { + use _iouringvfs::ops::OpsFd; + use std::ffi::CString; + use std::io::Result; + use std::io::Write; + use std::os::unix::ffi::OsStrExt; + use tempfile::TempDir; + + fn create_file(dir: &TempDir, file_name: &str, write: Option<&[u8]>) -> CString { + let path_buf = dir.path().join(file_name); + let path = CString::new(path_buf.as_os_str().as_bytes()).expect("bad path"); + if let Some(b) = write { + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(path_buf) + .expect("failed to create new file"); + let _ = file.write(b); + } + path + } + + #[test] + fn test_open_and_close_file() -> Result<()> { + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", None); + let mut ops = OpsFd::new(path.as_ptr() as _, 16); + + // Check if the operation was successful + ops.open_file()?; + + unsafe { + ops.o_close()?; + } + + Ok(()) + } + + #[test] + fn test_create_write_close_file() -> Result<()> { + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", None); + let mut ops = OpsFd::new(path.as_ptr() as _, 16); + + ops.open_file()?; + + // Write data to the file + let data_to_write = b"Hello, World!"; + unsafe { ops.o_write(data_to_write.as_ptr() as _, 0, 13) }?; + + unsafe { + ops.o_close()?; + } + + Ok(()) + } + + #[test] + fn test_read() -> Result<()> { + let data_to_write = b"Hello, World!"; + + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", Some(data_to_write)); + let mut ops = OpsFd::new(path.as_ptr() as _, 16); + + // Perform the open operation + ops.open_file()?; + + // Read the file + let mut buf: [u8; 13] = [0; 13]; + unsafe { + ops.o_read(0, 13, buf.as_mut_ptr() as _)?; + } + + // Check if the data read matches what was written + assert_eq!(buf[..], data_to_write[..]); + + Ok(()) + } + + #[test] + fn test_write() -> Result<()> { + let tmpfile = tempfile::NamedTempFile::new()?; + let file_path = CString::new(tmpfile.path().to_string_lossy().to_string())?; + + let mut ops = OpsFd::new(file_path.as_ptr() as _, 16); + + ops.open_file()?; + + let data_to_write = b"Hello, World!"; + unsafe { + ops.o_write( + data_to_write.as_ptr() as _, + 0, + data_to_write.len().try_into().unwrap(), + ) + }?; + + let file = tmpfile.as_file(); + + assert_eq!(file.metadata()?.len(), data_to_write.len() as u64); + + Ok(()) + } + + #[test] + fn test_write_then_read() -> Result<()> { + // Create a temporary file for testing + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", None); + let mut ops = OpsFd::new(path.as_ptr() as _, 16); + + // Perform the open operation + ops.open_file()?; + + // Write data to the file + let data_to_write = b"Hello, World!"; + let mut buf: [u8; 13] = [0; 13]; + unsafe { + ops.o_write(data_to_write.as_ptr() as _, 0, 13)?; + ops.o_fsync(0)?; + ops.o_read(0, 13, buf.as_mut_ptr() as _)?; + } + + // Check if the data read matches what was written + assert_eq!(buf[..], data_to_write[..]); + + Ok(()) + } + + #[test] + fn test_file_size() -> Result<()> { + let data_to_write = b"Hello, World!"; + + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", Some(data_to_write)); + + let mut ops = OpsFd::new(path.as_ptr() as _, 16); + + // Perform the open operation + ops.open_file()?; + + // Get the current file size + let mut file_size: u64 = 0; + unsafe { + ops.o_file_size(&mut file_size)?; + } + + assert_eq!(file_size, 13); + + Ok(()) + } + + #[test] + fn test_truncate_then_compare_file_size() -> Result<()> { + // Create a temporary file for testing + let mut tmpfile = tempfile::NamedTempFile::new()?; + let file_path = CString::new(tmpfile.path().to_string_lossy().to_string())?; + let mut ops = OpsFd::new(file_path.as_ptr() as _, 16); + + // Perform the open operation + ops.open_file()?; + + // Write some data to the file + let data_to_write = b"Hello, World!"; + tmpfile.write(data_to_write)?; + + // Truncate the file to a smaller size + let new_size = 5; // Set the new size to 5 bytes + unsafe { + ops.o_truncate(new_size)?; + } + + // Get the current file size + let mut file_size: u64 = 0; + unsafe { + ops.o_file_size(&mut file_size)?; + } + + // Check if the file size matches the expected size + assert_eq!(file_size, new_size as u64); + + // Cleanup + tmpfile.close()?; + + Ok(()) + } +} diff --git a/benchmarks/vfs/io_uring/tests/test_ops_fixed.rs b/benchmarks/vfs/io_uring/tests/test_ops_fixed.rs new file mode 100644 index 0000000..330e8cf --- /dev/null +++ b/benchmarks/vfs/io_uring/tests/test_ops_fixed.rs @@ -0,0 +1,193 @@ +#[cfg(test)] +mod tests { + use _iouringvfs::ops::OpsFixed; + use std::ffi::CString; + use std::io::Result; + use std::io::Write; + use std::os::unix::ffi::OsStrExt; + use tempfile::TempDir; + + fn create_file(dir: &TempDir, file_name: &str, write: Option<&[u8]>) -> CString { + let path_buf = dir.path().join(file_name); + let path = CString::new(path_buf.as_os_str().as_bytes()).expect("bad path"); + if let Some(b) = write { + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(path_buf) + .expect("failed to create new file"); + let _ = file.write(b); + } + path + } + + #[test] + fn test_open_and_close_file() -> Result<()> { + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", None); + let mut ops = OpsFixed::new(path.as_ptr() as _, 16); + + // Check if the operation was successful + ops.open_file()?; + + unsafe { + ops.o_close()?; + } + + Ok(()) + } + + #[test] + #[ignore] + fn test_create_write_close_file() -> Result<()> { + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", None); + let mut ops = OpsFixed::new(path.as_ptr() as _, 16); + + ops.open_file()?; + + // Write data to the file + let data_to_write = b"Hello, World!"; + unsafe { ops.o_write(data_to_write.as_ptr() as _, 0, 13) }?; + + unsafe { + ops.o_close()?; + } + + Ok(()) + } + + #[test] + fn test_read() -> Result<()> { + let data_to_write = b"Hello, World!"; + + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", Some(data_to_write)); + let mut ops = OpsFixed::new(path.as_ptr() as _, 16); + + // Perform the open operation + ops.open_file()?; + + // Read the file + let mut buf: [u8; 13] = [0; 13]; + unsafe { + ops.o_read(0, 13, buf.as_mut_ptr() as _)?; + } + + // Check if the data read matches what was written + assert_eq!(buf[..], data_to_write[..]); + + Ok(()) + } + + #[test] + #[ignore] + fn test_write() -> Result<()> { + let tmpfile = tempfile::NamedTempFile::new()?; + let file_path = CString::new(tmpfile.path().to_string_lossy().to_string())?; + + let mut ops = OpsFixed::new(file_path.as_ptr() as _, 16); + + ops.open_file()?; + + let data_to_write = b"Hello, World!"; + unsafe { + ops.o_write( + data_to_write.as_ptr() as _, + 0, + data_to_write.len().try_into().unwrap(), + ) + }?; + + let file = tmpfile.as_file(); + + assert_eq!(file.metadata()?.len(), data_to_write.len() as u64); + + Ok(()) + } + + #[test] + #[ignore] + fn test_write_then_read() -> Result<()> { + // Create a temporary file for testing + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", None); + let mut ops = OpsFixed::new(path.as_ptr() as _, 16); + + // Perform the open operation + ops.open_file()?; + + // Write data to the file + let data_to_write = b"Hello, World!"; + let mut buf: [u8; 13] = [0; 13]; + unsafe { + ops.o_write(data_to_write.as_ptr() as _, 0, 13)?; + ops.o_fsync(0)?; + ops.o_read(0, 13, buf.as_mut_ptr() as _)?; + } + + // Check if the data read matches what was written + assert_eq!(buf[..], data_to_write[..]); + + Ok(()) + } + + #[test] + fn test_file_size() -> Result<()> { + let data_to_write = b"Hello, World!"; + + let dir = tempfile::tempdir().expect("bad dir"); + let path = create_file(&dir, "main.db-journal", Some(data_to_write)); + + let mut ops = OpsFixed::new(path.as_ptr() as _, 16); + + // Perform the open operation + ops.open_file()?; + + // Get the current file size + let mut file_size: u64 = 0; + unsafe { + ops.o_file_size(&mut file_size)?; + } + + assert_eq!(file_size, 13); + + Ok(()) + } + + #[test] + fn test_truncate_then_compare_file_size() -> Result<()> { + // Create a temporary file for testing + let mut tmpfile = tempfile::NamedTempFile::new()?; + let file_path = CString::new(tmpfile.path().to_string_lossy().to_string())?; + let mut ops = OpsFixed::new(file_path.as_ptr() as _, 16); + + // Perform the open operation + ops.open_file()?; + + // Write some data to the file + let data_to_write = b"Hello, World!"; + tmpfile.write(data_to_write)?; + + // Truncate the file to a smaller size + let new_size = 5; // Set the new size to 5 bytes + unsafe { + ops.o_truncate(new_size)?; + } + + // Get the current file size + let mut file_size: u64 = 0; + unsafe { + ops.o_file_size(&mut file_size)?; + } + + // Check if the file size matches the expected size + assert_eq!(file_size, new_size as u64); + + // Cleanup + tmpfile.close()?; + + Ok(()) + } +} diff --git a/benchmarks/vfs/io_uring/tests/test_vfs.rs b/benchmarks/vfs/io_uring/tests/test_vfs.rs new file mode 100644 index 0000000..6c2d3c4 --- /dev/null +++ b/benchmarks/vfs/io_uring/tests/test_vfs.rs @@ -0,0 +1,41 @@ +include!("../include/conn.in.rs"); + +#[cfg(test)] +mod tests { + use _iouringvfs::sqlite3_iouringvfs_init; + use rusqlite::{self, ffi::sqlite3_auto_extension, Connection}; + + use crate::open_io_uring_connection; + + #[test] + fn test_io_uring_ext() -> rusqlite::Result<()> { + env_logger::init(); + + unsafe { + sqlite3_auto_extension(Some(std::mem::transmute( + sqlite3_iouringvfs_init as *const (), + ))); + } + + // Pre-load register function etc. See faux_sqlite_extension_init2 and its required global. + let _conn = Connection::open_in_memory()?; + _conn.close().expect("error occurred while closing"); + + let tmp_file = tempfile::NamedTempFile::new().unwrap(); + let out_path = tmp_file.path().to_string_lossy().to_string(); + + let conn = open_io_uring_connection(out_path.as_str())?; + + conn.execute("CREATE TABLE t3(x varchar(10), y integer)", ())?; + conn.execute( + "INSERT INTO t3 VALUES('a', 4),('b', 5),('c', 3),('d', 8),('e', 1)", + (), + )?; + + let result: String = conn.query_row("select x from t3 where y = 4", (), |x| x.get(0))?; + + assert_eq!(result, "a"); + + Ok(()) + } +} diff --git a/examples/characters.rs b/examples/characters.rs index 3ae9d43..bb49651 100644 --- a/examples/characters.rs +++ b/examples/characters.rs @@ -2,6 +2,7 @@ //! sqlite3 :memory: '.read examples/test.sql' use sqlite_loadable::prelude::*; + use sqlite_loadable::{ api, define_table_function, table::{BestIndexError, ConstraintOperator, IndexInfo, VTab, VTabArguments, VTabCursor}, diff --git a/examples/mem_vfs.rs b/examples/mem_vfs.rs new file mode 100644 index 0000000..bde5589 --- /dev/null +++ b/examples/mem_vfs.rs @@ -0,0 +1,3 @@ +#![allow(unused)] + +include!("../include/mem_vfs.in.rs"); diff --git a/include/mem_vfs.in.rs b/include/mem_vfs.in.rs new file mode 100644 index 0000000..fe665be --- /dev/null +++ b/include/mem_vfs.in.rs @@ -0,0 +1,294 @@ +use sqlite_loadable::ext::{sqlite3ext_vfs_find, sqlite3ext_context_db_handle, sqlite3ext_file_control}; +use sqlite_loadable::vfs::shim::ShimVfs; +use sqlite_loadable::vfs::vfs::create_vfs; +use sqlite_loadable::vfs::file::{FileWithAux, prepare_file_ptr}; + +use sqlite_loadable::{prelude::*, SqliteIoMethods, register_boxed_vfs, define_scalar_function, api, vfs::traits::SqliteVfs}; + +use std::ffi::{CString, CStr}; +use std::io::{Write, Read, self}; +use std::os::raw::{c_void, c_char}; +use std::{ptr, mem}; + +use sqlite_loadable::ext::{sqlite3_syscall_ptr, sqlite3_file, sqlite3_vfs, sqlite3_io_methods}; +use sqlite3ext_sys::{ + SQLITE_CANTOPEN, SQLITE_OPEN_MAIN_DB, SQLITE_IOCAP_ATOMIC, SQLITE_IOCAP_POWERSAFE_OVERWRITE, SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN, + SQLITE_IOCAP_SAFE_APPEND, SQLITE_IOCAP_SEQUENTIAL, SQLITE_LOCK_EXCLUSIVE, SQLITE_LOCK_SHARED, SQLITE_OK}; + +use std::io::{Error, Result, ErrorKind}; + +/// Inspired by https://www.sqlite.org/src/file/ext/misc/memvfs.c +/// See https://www.sqlite.org/debugging.html for debugging methods +struct MemVfs { + default_vfs: Option, + name: CString, +} + +const EXTENSION_NAME: &str = "memvfs"; + +fn write_file_to_vec_u8(path: &str, dest: &mut Vec) -> Result<()> { + let metadata = std::fs::metadata(path)?; + let file_size = metadata.len() as usize; + + let mut file = std::fs::File::open(path)?; + + file.read_to_end(dest)?; + + Ok(()) +} + +impl SqliteVfs for MemVfs { + fn open(&mut self, z_name: *const c_char, p_file: *mut sqlite3_file, flags: i32, p_res_out: *mut i32) -> Result<()> { + let mut mem_file = MemFile { + file_contents: vec![], + }; + + unsafe { + let file_name_cstr = CStr::from_ptr(z_name); + let file_name = file_name_cstr.to_str() + .map_err(|_| Error::new(ErrorKind::Other, "conversion to string failed"))?; + write_file_to_vec_u8(file_name, &mut mem_file.file_contents); + prepare_file_ptr(p_file, mem_file); + } + + Ok(()) + } + + fn delete(&mut self, z_name: *const c_char, sync_dir: i32) -> Result<()> { + Ok(()) + } + + fn access(&mut self, z_name: *const c_char, flags: i32, p_res_out: *mut i32) -> Result<()> { + unsafe { + *p_res_out = 0; + } + Ok(()) + } + + fn full_pathname(&mut self, z_name: *const c_char, n_out: i32, z_out: *mut c_char) -> Result<()> { + unsafe { + // don't rely on type conversion of n_out to determine the end line char + let name = CString::from_raw(z_name.cast_mut()); + let src_ptr = name.as_ptr(); + let dst_ptr = z_out; + let len = name.as_bytes().len() + 1; + ptr::copy_nonoverlapping(src_ptr, dst_ptr.cast(), len); + name.into_raw(); + } + + Ok(()) + } + + /// From here onwards, all calls are redirected to the default vfs + // fn dl_open(&mut self, z_filename: *const c_char) -> *mut c_void { + // self.default_vfs.dl_open(z_filename) + // } + + // fn dl_error(&mut self, n_byte: i32, z_err_msg: *mut c_char) { + // self.default_vfs.dl_error(n_byte, z_err_msg) + // } + + // fn dl_sym(&mut self, arg2: *mut c_void, z_symbol: *const c_char) + // -> Option { + // self.default_vfs.dl_sym(arg2, z_symbol) + // } + + // fn dl_close(&mut self, arg2: *mut c_void) { + // self.default_vfs.dl_close(arg2) + // } + + fn randomness(&mut self, n_byte: i32, z_out: *mut c_char) -> i32 { + if let Some(vfs) = &mut self.default_vfs { + return vfs.randomness(n_byte, z_out); + } + 0 + } + + fn sleep(&mut self, microseconds: i32) -> i32 { + if let Some(vfs) = &mut self.default_vfs { + return vfs.sleep(microseconds); + } + 0 + } + + fn current_time(&mut self, arg2: *mut f64) -> i32 { + if let Some(vfs) = &mut self.default_vfs { + return vfs.current_time(arg2); + } + 0 + } + + fn get_last_error(&mut self, arg2: i32, arg3: *mut c_char) -> Result<()> { + if let Some(vfs) = &mut self.default_vfs { + vfs.get_last_error(arg2, arg3); + } + Ok(()) + } + + fn current_time_int64(&mut self, arg2: *mut i64) -> i32 { + if let Some(vfs) = &mut self.default_vfs { + return vfs.current_time_int64(arg2); + } + 0 + } + + // fn set_system_call(&mut self, z_name: *const c_char, arg2: sqlite3_syscall_ptr) -> i32 { + // self.default_vfs.set_system_call(z_name, arg2) + // } + + // fn get_system_call(&mut self, z_name: *const c_char) -> sqlite3_syscall_ptr { + // self.default_vfs.get_system_call(z_name) + // } + + // fn next_system_call(&mut self, z_name: *const c_char) -> *const c_char { + // self.default_vfs.next_system_call(z_name) + // } +} + +struct MemFile { + file_contents: Vec, +} + +impl SqliteIoMethods for MemFile { + fn close(&mut self) -> Result<()> { + Ok(()) + } + + fn read(&mut self, buf: *mut c_void, s: i32, ofst: i64) -> Result<()> { + let size: usize = s.try_into().unwrap(); + let offset = ofst.try_into().unwrap(); + let source = &mut self.file_contents; + if source.len() < size { + let new_len = offset + size; + let prev_len = source.len(); + source.resize(new_len, 0); + source.extend(vec![0; new_len - prev_len]); + } + + let src_ptr = source[offset..(offset + size-1)].as_ptr(); + unsafe { ptr::copy_nonoverlapping(src_ptr, buf.cast(), size) } + + Ok(()) + } + + fn write(&mut self, buf: *const c_void, s: i32, ofst: i64) -> Result<()> { + let size = s.try_into().unwrap(); + let offset = ofst.try_into().unwrap(); + let new_length = size + offset; + if new_length > self.file_contents.len() { + self.file_contents.resize(new_length, 0); + } + + let dest = &mut self.file_contents; + + let src_slice = unsafe { std::slice::from_raw_parts(buf as *const u8, size) }; + + dest[offset..offset + src_slice.len()].copy_from_slice(src_slice); + + Ok(()) + } + + fn truncate(&mut self, size: i64) -> Result<()> { + self.file_contents.resize(size.try_into().unwrap(), 0); + + Ok(()) + } + + fn sync(&mut self, flags: i32) -> Result<()> { + Ok(()) + } + + fn file_size(&mut self, p_size: *mut i64) -> Result<()> { + unsafe { *p_size = self.file_contents.len().try_into().unwrap(); } + Ok(()) + } + + fn lock(&mut self, arg2: i32) -> Result { + Ok(0) // or SQLITE_LOCK_BUSY + } + + fn unlock(&mut self, arg2: i32) -> Result { + Ok(0) + } + + fn check_reserved_lock(&mut self, p_res_out: *mut i32) -> Result<()> { + unsafe{ *p_res_out = 0; } + Ok(()) + } + + fn file_control(&mut self, op: i32, p_arg: *mut c_void) -> Result<()> { + Ok(()) + } + + fn sector_size(&mut self) -> Result { + Ok(1024) + } + + fn device_characteristics(&mut self) -> Result { + let settings = SQLITE_IOCAP_ATOMIC | + SQLITE_IOCAP_POWERSAFE_OVERWRITE | + SQLITE_IOCAP_SAFE_APPEND | + // SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + SQLITE_IOCAP_SEQUENTIAL; + Ok(settings) + } + + fn shm_map(&mut self, i_pg: i32, pgsz: i32, arg2: i32, arg3: *mut *mut c_void) -> Result<()> { + // SQLITE_IOERR_SHMMAP + Err(Error::new(ErrorKind::Other, "Unsupported")) + } + + fn shm_lock(&mut self, offset: i32, n: i32, flags: i32) -> Result<()> { + // SQLITE_IOERR_SHMLOCK + Err(Error::new(ErrorKind::Other, "Unsupported")) + } + + fn shm_barrier(&mut self) -> Result<()> { + Ok(()) + } + + fn shm_unmap(&mut self, delete_flag: i32) -> Result<()> { + Ok(()) + } + + fn fetch(&mut self, ofst: i64, size: i32, pp: *mut *mut c_void) -> Result<()> { + let memory_location = self.file_contents.as_mut_ptr(); + unsafe { *pp = memory_location.add(ofst.try_into().unwrap()).cast(); } + Ok(()) + } + + fn unfetch(&mut self, i_ofst: i64, p: *mut c_void) -> Result<()> { + Ok(()) + } +} + +fn print_uri(context: *mut sqlite3_context, _: &[*mut sqlite3_value]) -> sqlite_loadable::Result<()> { + let text_output = format!("file:___mem___?vfs={}", EXTENSION_NAME); + + api::result_text(context, text_output); + + Ok(()) +} + +#[sqlite_entrypoint] +pub fn sqlite3_memvfs_init(db: *mut sqlite3) -> sqlite_loadable::Result<()> { + let name = CString::new(EXTENSION_NAME).expect("should be a valid utf-8 string"); + let mem_vfs = MemVfs { + default_vfs: unsafe { + // pass thru + // Some(ShimVfs::from_ptr(sqlite3ext_vfs_find(ptr::null()))) + None + }, + name + }; + let name_ptr = mem_vfs.name.as_ptr(); + + let vfs: sqlite3_vfs = create_vfs(mem_vfs, name_ptr, 1024, std::mem::size_of::>() as i32); + + register_boxed_vfs(vfs, true)?; + + let flags = FunctionFlags::UTF8 | FunctionFlags::DETERMINISTIC; + define_scalar_function(db, "mem_vfs_uri", 0, print_uri, flags)?; + + Ok(()) +} diff --git a/run-docker.sh b/run-docker.sh new file mode 100644 index 0000000..5bd9b51 --- /dev/null +++ b/run-docker.sh @@ -0,0 +1,8 @@ +#!/bin/sh +NAME="sqlite-loadable-rs:1.0" +docker image inspect "$NAME" || docker build -t "$NAME" . +docker run -it -p 2222:22 -v $PWD:/root -w /root $NAME + +# see https://github.com/jfrimmel/cargo-valgrind/pull/58/commits/1c168f296e0b3daa50279c642dd37aecbd85c5ff#L59 +# scan for double frees and leaks +# VALGRINDFLAGS="--leak-check=yes --trace-children=yes" cargo valgrind test diff --git a/sqlite-loadable-macros/src/lib.rs b/sqlite-loadable-macros/src/lib.rs index b2c02a4..bb12f99 100644 --- a/sqlite-loadable-macros/src/lib.rs +++ b/sqlite-loadable-macros/src/lib.rs @@ -33,7 +33,7 @@ pub fn sqlite_entrypoint(_attr: TokenStream, item: TokenStream) -> TokenStream { db: *mut sqlite3, pz_err_msg: *mut *mut c_char, p_api: *mut sqlite3_api_routines, - ) -> c_uint { + ) -> i32 { register_entrypoint(db, pz_err_msg, p_api, #prefixed_original_function) } @@ -73,7 +73,7 @@ pub fn sqlite_entrypoint_permanent(_attr: TokenStream, item: TokenStream) -> Tok db: *mut sqlite3, pz_err_msg: *mut *mut c_char, p_api: *mut sqlite3_api_routines, - ) -> c_uint { + ) -> i32 { register_entrypoint_load_permanently(db, pz_err_msg, p_api, #prefixed_original_function) } diff --git a/sqlite3ext-sys/build.rs b/sqlite3ext-sys/build.rs index 5ba27d3..3f3a74f 100644 --- a/sqlite3ext-sys/build.rs +++ b/sqlite3ext-sys/build.rs @@ -17,6 +17,7 @@ fn main() { println!("cargo:rerun-if-changed=wrapper.h"); let bindings = bindgen::Builder::default() + .default_macro_constant_type(bindgen::MacroTypeVariation::Signed) .header("sqlite3/sqlite3ext.h") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .generate() diff --git a/src/api.rs b/src/api.rs index 11e444f..18a97a3 100644 --- a/src/api.rs +++ b/src/api.rs @@ -5,7 +5,6 @@ //! Useful when working with sqlite3_value or sqlite3_context. #![allow(clippy::not_unsafe_ptr_arg_deref)] -use crate::constants::SQLITE_OKAY; use crate::ext::{ sqlite3, sqlite3_context, sqlite3_value, sqlite3ext_context_db_handle, sqlite3ext_get_auxdata, sqlite3ext_mprintf, sqlite3ext_overload_function, sqlite3ext_result_blob, @@ -17,7 +16,9 @@ use crate::ext::{ sqlite3ext_value_subtype, sqlite3ext_value_text, sqlite3ext_value_type, }; use crate::Error; -use sqlite3ext_sys::{SQLITE_BLOB, SQLITE_FLOAT, SQLITE_INTEGER, SQLITE_NULL, SQLITE_TEXT}; +use sqlite3ext_sys::{ + SQLITE_BLOB, SQLITE_FLOAT, SQLITE_INTEGER, SQLITE_NULL, SQLITE_OK, SQLITE_TEXT, +}; use std::os::raw::c_int; use std::slice::from_raw_parts; use std::str::Utf8Error; @@ -211,7 +212,7 @@ pub enum ValueType { pub fn value_type(value: &*mut sqlite3_value) -> ValueType { let raw_type = unsafe { sqlite3ext_value_type(value.to_owned()) }; // "as u32" because bindings for constants are u32 for some reason??? - match raw_type as u32 { + match raw_type { SQLITE_TEXT => ValueType::Text, SQLITE_INTEGER => ValueType::Integer, SQLITE_FLOAT => ValueType::Float, @@ -225,7 +226,7 @@ pub fn value_type(value: &*mut sqlite3_value) -> ValueType { } pub fn value_is_null(value: &*mut sqlite3_value) -> bool { let raw_type = unsafe { sqlite3ext_value_type(value.to_owned()) }; - (raw_type as u32) == SQLITE_NULL + raw_type == SQLITE_NULL } pub fn value_subtype(value: &*mut sqlite3_value) -> u32 { @@ -397,7 +398,7 @@ pub fn context_db_handle(context: *mut sqlite3_context) -> *mut sqlite3 { pub fn overload_function(db: *mut sqlite3, func_name: &str, n_args: i32) -> crate::Result<()> { let cname = CString::new(func_name)?; let result = unsafe { sqlite3ext_overload_function(db, cname.as_ptr(), n_args) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { return Err(Error::new_message("TODO")); } Ok(()) diff --git a/src/collation.rs b/src/collation.rs index b234b0c..db53639 100644 --- a/src/collation.rs +++ b/src/collation.rs @@ -2,13 +2,12 @@ #![allow(clippy::not_unsafe_ptr_arg_deref)] use crate::{ - constants::SQLITE_OKAY, errors::{Error, ErrorKind, Result}, ext::{sqlite3, sqlite3ext_collation_v2}, }; use std::{ffi::CString, os::raw::c_void}; -use sqlite3ext_sys::SQLITE_UTF8; +use sqlite3ext_sys::{SQLITE_UTF8, SQLITE_OK}; pub fn define_collation(db: *mut sqlite3, name: &str, x_func: F) -> Result<()> where @@ -43,7 +42,7 @@ where ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { Err(Error::new(ErrorKind::DefineScalarFunction(result))) } else { Ok(()) diff --git a/src/constants.rs b/src/constants.rs deleted file mode 100644 index 338f700..0000000 --- a/src/constants.rs +++ /dev/null @@ -1,21 +0,0 @@ -/// rust bindgen for some reason is defining many SQLite constants -/// as u32, which can't safely be casted into i32. So, here we -/// hardcode some of those codes to avoid unwrapping - -/// https://www.sqlite.org/rescode.html#ok -pub const SQLITE_OKAY: i32 = 0; - -/// https://www.sqlite.org/rescode.html#internal -pub const SQLITE_INTERNAL: i32 = 2; - -/// https://www.sqlite.org/rescode.html#row -pub const SQLITE_ROW: i32 = 100; - -/// https://www.sqlite.org/rescode.html#done -pub const SQLITE_DONE: i32 = 101; - -/// https://www.sqlite.org/rescode.html#error -pub const SQLITE_ERROR: i32 = 1; - -/// https://www.sqlite.org/rescode.html#constraint -pub const SQLITE_CONSTRAINT: i32 = 19; diff --git a/src/entrypoints.rs b/src/entrypoints.rs index 1301e41..caac6c5 100644 --- a/src/entrypoints.rs +++ b/src/entrypoints.rs @@ -7,7 +7,7 @@ use crate::{ use sqlite3ext_sys::SQLITE_OK; -use std::os::raw::{c_char, c_uint}; +use std::os::raw::c_char; /// Low-level wrapper around a typical entrypoint to a SQLite extension. /// You shouldn't have to use this directly - the sqlite_entrypoint @@ -17,7 +17,7 @@ pub fn register_entrypoint( _pz_err_msg: *mut *mut c_char, p_api: *mut sqlite3_api_routines, callback: F, -) -> c_uint +) -> i32 where F: Fn(*mut sqlite3) -> Result<()>, { @@ -38,7 +38,7 @@ pub fn register_entrypoint_load_permanently( _pz_err_msg: *mut *mut c_char, p_api: *mut sqlite3_api_routines, callback: F, -) -> c_uint +) -> i32 where F: Fn(*mut sqlite3) -> Result<()>, { diff --git a/src/errors.rs b/src/errors.rs index 31135a3..a95715a 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -2,7 +2,6 @@ use std::{ ffi::NulError, fmt, - os::raw::{c_int, c_uint}, result, }; @@ -32,10 +31,10 @@ impl Error { *self.0 } - pub fn code(self) -> c_int { + pub fn code(self) -> i32 { 1 } - pub fn code_extended(self) -> c_uint { + pub fn code_extended(self) -> i32 { 1 } pub fn result_error_message(self) -> String { @@ -52,10 +51,10 @@ impl Error { /// The specific type of an error. #[derive(Debug, PartialEq, Eq)] pub enum ErrorKind { - DefineScalarFunction(c_int), + DefineScalarFunction(i32), CStringError(NulError), CStringUtf8Error(std::str::Utf8Error), - TableFunction(c_int), + TableFunction(i32), Message(String), } diff --git a/src/ext.rs b/src/ext.rs index 7326cd5..d98af6f 100644 --- a/src/ext.rs +++ b/src/ext.rs @@ -17,23 +17,27 @@ use std::{ #[cfg(feature = "static")] pub use libsqlite3_sys::{ - sqlite3, sqlite3_api_routines, sqlite3_context, - sqlite3_index_constraint as sqlite3_index_info_sqlite3_index_constraint, + sqlite3, sqlite3_api_routines, sqlite3_context, sqlite3_database_file_object, sqlite3_file, + sqlite3_file_control, sqlite3_index_constraint as sqlite3_index_info_sqlite3_index_constraint, sqlite3_index_constraint_usage as sqlite3_index_info_sqlite3_index_constraint_usage, sqlite3_index_info, sqlite3_index_orderby as sqlite3_index_info_sqlite3_index_orderby, - sqlite3_module, sqlite3_stmt, sqlite3_value, sqlite3_vtab, sqlite3_vtab_cursor, + sqlite3_int64, sqlite3_io_methods, sqlite3_module, sqlite3_stmt, sqlite3_syscall_ptr, + sqlite3_value, sqlite3_vfs, sqlite3_vfs_find, sqlite3_vfs_register, sqlite3_vfs_unregister, + sqlite3_vtab, sqlite3_vtab_cursor, }; #[cfg(not(feature = "static"))] pub use sqlite3ext_sys::{ - sqlite3, sqlite3_api_routines, sqlite3_context, sqlite3_index_info, - sqlite3_index_info_sqlite3_index_constraint, sqlite3_index_info_sqlite3_index_constraint_usage, - sqlite3_index_info_sqlite3_index_orderby, sqlite3_module, sqlite3_stmt, sqlite3_value, + sqlite3, sqlite3_api_routines, sqlite3_context, sqlite3_file, sqlite3_file_control, + sqlite3_index_info, sqlite3_index_info_sqlite3_index_constraint, + sqlite3_index_info_sqlite3_index_constraint_usage, sqlite3_index_info_sqlite3_index_orderby, + sqlite3_int64, sqlite3_io_methods, sqlite3_module, sqlite3_stmt, sqlite3_syscall_ptr, + sqlite3_value, sqlite3_vfs, sqlite3_vfs_find, sqlite3_vfs_register, sqlite3_vfs_unregister, sqlite3_vtab, sqlite3_vtab_cursor, }; /// If creating a dynmically loadable extension, this MUST be redefined to point -/// to a proper sqlite3_api_rountines module (from a entrypoint function). +/// to a proper sqlite3_api_routines module (from a entrypoint function). /// The "sqlite_entrypoint" macro will do this for you usually. static mut SQLITE3_API: *mut sqlite3_api_routines = std::ptr::null_mut(); @@ -591,6 +595,62 @@ pub unsafe fn sqlite3ext_context_db_handle(context: *mut sqlite3_context) -> *mu ((*SQLITE3_API).context_db_handle.expect(EXPECT_MESSAGE))(context) } +#[cfg(feature = "static")] +pub unsafe fn sqlite3ext_vfs_unregister(vfs_ptr: *mut sqlite3_vfs) -> i32 { + return libsqlite3_sys::sqlite3_vfs_unregister(vfs_ptr); +} + +#[cfg(not(feature = "static"))] +pub unsafe fn sqlite3ext_vfs_unregister(vfs_ptr: *mut sqlite3_vfs) -> i32 { + ((*SQLITE3_API).vfs_unregister.expect(EXPECT_MESSAGE))(vfs_ptr) +} + +#[cfg(feature = "static")] +pub unsafe fn sqlite3ext_vfs_register( + vfs_ptr: *mut sqlite3_vfs, + make_default: i32, +) -> i32 { + return libsqlite3_sys::sqlite3_vfs_register(vfs_ptr, make_default); +} + +#[cfg(not(feature = "static"))] +pub unsafe fn sqlite3ext_vfs_register( + vfs_ptr: *mut sqlite3_vfs, + make_default: i32, +) -> i32 { + ((*SQLITE3_API).vfs_register.expect(EXPECT_MESSAGE))(vfs_ptr, make_default) +} + +#[cfg(feature = "static")] +pub unsafe fn sqlite3ext_vfs_find(name: *const c_char) -> *mut sqlite3_vfs { + return libsqlite3_sys::sqlite3_vfs_find(name); +} + +#[cfg(not(feature = "static"))] +pub unsafe fn sqlite3ext_vfs_find(name: *const c_char) -> *mut sqlite3_vfs { + ((*SQLITE3_API).vfs_find.expect(EXPECT_MESSAGE))(name) +} + +#[cfg(feature = "static")] +pub unsafe fn sqlite3ext_file_control( + db: *mut sqlite3, + name: *const c_char, + option: c_int, + data: *mut c_void, +) -> c_int { + return libsqlite3_sys::sqlite3_file_control(db, name, option, data); +} + +#[cfg(not(feature = "static"))] +pub unsafe fn sqlite3ext_file_control( + db: *mut sqlite3, + name: *const c_char, + option: c_int, + data: *mut c_void, +) -> c_int { + ((*SQLITE3_API).file_control.expect(EXPECT_MESSAGE))(db, name, option, data) +} + #[cfg(feature = "static")] pub unsafe fn sqlite3ext_user_data(context: *mut sqlite3_context) -> *mut c_void { libsqlite3_sys::sqlite3_user_data(context) @@ -616,3 +676,12 @@ pub unsafe fn sqlite3ext_auto_extension(f: unsafe extern "C" fn()) -> i32 { pub unsafe fn sqlite3ext_auto_extension(f: unsafe extern "C" fn()) -> i32 { ((*SQLITE3_API).auto_extension.expect(EXPECT_MESSAGE))(Some(f)) } + +#[cfg(feature = "static")] +pub unsafe fn sqlite3ext_database_file_object(s: *const c_char) -> *mut sqlite3_file { + libsqlite3_sys::sqlite3_database_file_object(s) +} +#[cfg(not(feature = "static"))] +pub unsafe fn sqlite3ext_database_file_object(s: *const c_char) -> *mut sqlite3_file { + ((*SQLITE3_API).database_file_object.expect(EXPECT_MESSAGE))(s) +} diff --git a/src/lib.rs b/src/lib.rs index 3df5c80..7584e85 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,6 @@ pub mod api; pub mod collation; -mod constants; pub mod entrypoints; pub mod errors; @@ -13,6 +12,7 @@ pub mod ext; // TODO dont expose pub mod prelude; pub mod scalar; pub mod table; +pub mod vfs; pub mod vtab_argparse; #[doc(inline)] @@ -30,4 +30,6 @@ pub use table::{ define_virtual_table_writeable, define_virtual_table_writeablex, BestIndexError, }; -pub use constants::*; +pub use vfs::traits::{SqliteIoMethods, SqliteVfs}; +#[doc(inline)] +pub use vfs::vfs::register_boxed_vfs; diff --git a/src/scalar.rs b/src/scalar.rs index db2982e..d4c88e8 100644 --- a/src/scalar.rs +++ b/src/scalar.rs @@ -9,7 +9,6 @@ use std::{ use crate::{ api, - constants::{SQLITE_INTERNAL, SQLITE_OKAY}, errors::{Error, ErrorKind, Result}, ext::{ sqlite3, sqlite3_context, sqlite3_value, sqlite3ext_create_function_v2, @@ -21,7 +20,7 @@ use bitflags::bitflags; use sqlite3ext_sys::{ SQLITE_DETERMINISTIC, SQLITE_DIRECTONLY, SQLITE_INNOCUOUS, SQLITE_SUBTYPE, SQLITE_UTF16, - SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8, + SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8, SQLITE_OK, SQLITE_INTERNAL, }; bitflags! { @@ -71,7 +70,7 @@ fn create_function_v2( ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { Err(Error::new(ErrorKind::DefineScalarFunction(result))) } else { Ok(()) diff --git a/src/table.rs b/src/table.rs index 9a88d87..50bc31c 100644 --- a/src/table.rs +++ b/src/table.rs @@ -1,8 +1,6 @@ //! Defining virtual tables and table functions on sqlite3 database connections. // ![allow(clippy::not_unsafe_ptr_arg_deref)] - -use crate::constants::*; use std::ffi::CString; use std::marker::PhantomData; use std::marker::Sync; @@ -20,7 +18,9 @@ use crate::ext::{ sqlite3ext_declare_vtab, sqlite3ext_vtab_distinct, sqlite3ext_vtab_in, sqlite3ext_vtab_in_first, sqlite3ext_vtab_in_next, }; + use serde::{Deserialize, Serialize}; +use sqlite3ext_sys::{SQLITE_ERROR, SQLITE_OK, SQLITE_CONSTRAINT, SQLITE_DONE}; /// Possible operators for a given constraint, found and used in xBestIndex and xFilter. /// @@ -368,7 +368,7 @@ pub fn define_table_function<'vtab, T: VTab<'vtab> + 'vtab>( Some(destroy_aux::), ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { return Err(Error::new(ErrorKind::TableFunction(result))); } Ok(()) @@ -424,7 +424,7 @@ pub fn define_table_function_with_find<'vtab, T: VTabFind<'vtab> + 'vtab>( Some(destroy_aux::), ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { return Err(Error::new(ErrorKind::TableFunction(result))); } Ok(()) @@ -487,7 +487,7 @@ pub fn define_virtual_table<'vtab, T: VTab<'vtab> + 'vtab>( Some(destroy_aux::), ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { return Err(Error::new(ErrorKind::TableFunction(result))); } Ok(()) @@ -544,7 +544,7 @@ pub fn define_virtual_table_with_find<'vtab, T: VTabFind<'vtab> + 'vtab>( Some(destroy_aux::), ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { return Err(Error::new(ErrorKind::TableFunction(result))); } Ok(()) @@ -600,7 +600,7 @@ pub fn define_virtual_table_writeable<'vtab, T: VTabWriteable<'vtab> + 'vtab>( Some(destroy_aux::), ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { return Err(Error::new(ErrorKind::TableFunction(result))); } Ok(()) @@ -660,7 +660,7 @@ pub fn define_virtual_table_writeable_with_transactions< Some(destroy_aux::), ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { return Err(Error::new(ErrorKind::TableFunction(result))); } Ok(()) @@ -717,7 +717,7 @@ pub fn define_virtual_table_writeablex<'vtab, T: VTabWriteable<'vtab> + 'vtab>( Some(destroy_aux::), ) }; - if result != SQLITE_OKAY { + if result != SQLITE_OK { return Err(Error::new(ErrorKind::TableFunction(result))); } Ok(()) @@ -881,10 +881,10 @@ where Ok((sql, vtab)) => match CString::new(sql) { Ok(c_sql) => { let rc = sqlite3ext_declare_vtab(db, c_sql.as_ptr()); - if rc == SQLITE_OKAY { + if rc == SQLITE_OK { let boxed_vtab: *mut T = Box::into_raw(Box::new(vtab)); *pp_vtab = boxed_vtab.cast::(); - SQLITE_OKAY + SQLITE_OK } else { rc } @@ -924,10 +924,10 @@ where Ok((sql, vtab)) => match CString::new(sql) { Ok(c_sql) => { let rc = sqlite3ext_declare_vtab(db, c_sql.as_ptr()); - if rc == SQLITE_OKAY { + if rc == SQLITE_OK { let boxed_vtab: *mut T = Box::into_raw(Box::new(vtab)); *pp_vtab = boxed_vtab.cast::(); - SQLITE_OKAY + SQLITE_OK } else { rc } @@ -956,7 +956,7 @@ where { let vt = vtab.cast::(); match (*vt).best_index(IndexInfo { index_info }) { - Ok(_) => SQLITE_OKAY, + Ok(_) => SQLITE_OK, Err(e) => match e { BestIndexError::Constraint => SQLITE_CONSTRAINT, BestIndexError::Error => SQLITE_ERROR, @@ -971,11 +971,11 @@ where T: VTab<'vtab>, { if vtab.is_null() { - return SQLITE_OKAY; + return SQLITE_OK; } let vtab = vtab.cast::(); drop(Box::from_raw(vtab)); - SQLITE_OKAY + SQLITE_OK } /// @@ -985,11 +985,11 @@ where T: VTab<'vtab>, { if vtab.is_null() { - return SQLITE_OKAY; + return SQLITE_OK; } let vt = vtab.cast::(); match (*vt).destroy() { - Ok(_) => SQLITE_OKAY, + Ok(_) => SQLITE_OK, Err(err) => err.code(), } } @@ -1008,7 +1008,7 @@ where Ok(cursor) => { let boxed_cursor: *mut T::Cursor = Box::into_raw(Box::new(cursor)); *pp_cursor = boxed_cursor.cast::(); - SQLITE_OKAY + SQLITE_OK } Err(err) => err.code(), } @@ -1096,7 +1096,7 @@ where let vt = vtab.cast::(); match (*vt).update(determine_update_operation(argc, argv), p_rowid) { - Ok(_) => SQLITE_OKAY, + Ok(_) => SQLITE_OK, Err(err) => err.code(), } } @@ -1109,7 +1109,7 @@ where { let vt = vtab.cast::(); match (*vt).begin() { - Ok(_) => SQLITE_OKAY, + Ok(_) => SQLITE_OK, Err(err) => err.code(), } } @@ -1122,7 +1122,7 @@ where { let vt = vtab.cast::(); match (*vt).sync() { - Ok(_) => SQLITE_OKAY, + Ok(_) => SQLITE_OK, Err(err) => err.code(), } } @@ -1135,7 +1135,7 @@ where { let vt = vtab.cast::(); match (*vt).rollback() { - Ok(_) => SQLITE_OKAY, + Ok(_) => SQLITE_OK, Err(err) => err.code(), } } @@ -1148,7 +1148,7 @@ where { let vt = vtab.cast::(); match (*vt).commit() { - Ok(_) => SQLITE_OKAY, + Ok(_) => SQLITE_OK, Err(err) => err.code(), } } @@ -1189,7 +1189,7 @@ where { let cr = cursor.cast::(); drop(Box::from_raw(cr)); - SQLITE_OKAY + SQLITE_OK } /// @@ -1215,7 +1215,7 @@ where //cursor_error(cursor, ) let args = slice::from_raw_parts_mut(argv, argc as usize); match (*cr).filter(idx_num, idx_name, args) { - Ok(()) => SQLITE_OKAY, + Ok(()) => SQLITE_OK, Err(err) => { if let ErrorKind::Message(msg) = err.kind() { if let Ok(err) = mprintf(msg) { @@ -1236,7 +1236,7 @@ where let cr = cursor.cast::(); //cursor_error(cursor, (*cr).next()) match (*cr).next() { - Ok(()) => SQLITE_OKAY, + Ok(()) => SQLITE_OK, Err(err) => { if let ErrorKind::Message(msg) = err.kind() { if let Ok(err) = mprintf(msg) { @@ -1271,7 +1271,7 @@ where let cr = cursor.cast::(); //result_error(ctx, (*cr).column(&mut ctxt, i)) match (*cr).column(ctx, i) { - Ok(()) => SQLITE_OKAY, + Ok(()) => SQLITE_OK, Err(err) => { if let ErrorKind::Message(msg) = err.kind() { if let Ok(err) = mprintf(msg) { @@ -1285,7 +1285,7 @@ where /// "A successful invocation of this method will cause *pRowid to be filled with the rowid of row /// that the virtual table cursor pCur is currently pointing at. -/// This method returns SQLITE_OKAY on success. It returns an appropriate error code on failure." +/// This method returns SQLITE_OK on success. It returns an appropriate error code on failure." /// // TODO set error message properly unsafe extern "C" fn rust_rowid(cursor: *mut sqlite3_vtab_cursor, p_rowid: *mut i64) -> c_int @@ -1296,7 +1296,7 @@ where match (*cr).rowid() { Ok(rowid) => { *p_rowid = rowid; - SQLITE_OKAY + SQLITE_OK } Err(err) => err.code(), } diff --git a/src/vfs/file.rs b/src/vfs/file.rs new file mode 100644 index 0000000..7e7716c --- /dev/null +++ b/src/vfs/file.rs @@ -0,0 +1,244 @@ +#![allow(non_snake_case)] +#![allow(unused)] + +use crate::ext::{sqlite3_file, sqlite3_int64, sqlite3_io_methods}; + +use sqlite3ext_sys::{ + SQLITE_IOERR_CLOSE, SQLITE_IOERR_FSTAT, SQLITE_IOERR_FSYNC, SQLITE_IOERR_LOCK, + SQLITE_IOERR_MMAP, SQLITE_IOERR_READ, SQLITE_IOERR_SHMLOCK, SQLITE_IOERR_SHMMAP, + SQLITE_IOERR_TRUNCATE, SQLITE_IOERR_UNLOCK, SQLITE_IOERR_WRITE, +}; + +use std::{ + fs::File, + mem::MaybeUninit, + os::raw::{c_int, c_void}, +}; + +use crate::vfs::traits::SqliteIoMethods; +use crate::vfs::vfs::handle_error; +use std::io::{Error, ErrorKind, Result}; + +use super::vfs::handle_int; + +// TODO keep a pointer of f and m, then +// This should just close the file, and not do gc +unsafe extern "C" fn x_close(file: *mut sqlite3_file) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.close(); + handle_error(result, Some(SQLITE_IOERR_CLOSE)) +} + +unsafe extern "C" fn x_read( + file: *mut sqlite3_file, + buf: *mut c_void, + iAmt: c_int, + iOfst: sqlite3_int64, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.read(buf, iAmt, iOfst); + handle_error(result, Some(SQLITE_IOERR_READ)) +} + +unsafe extern "C" fn x_write( + file: *mut sqlite3_file, + buf: *const c_void, + iAmt: c_int, + iOfst: sqlite3_int64, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.write(buf, iAmt, iOfst); + handle_error(result, Some(SQLITE_IOERR_WRITE)) +} + +unsafe extern "C" fn x_truncate( + file: *mut sqlite3_file, + size: sqlite3_int64, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.truncate(size); + handle_error(result, Some(SQLITE_IOERR_TRUNCATE)) +} + +unsafe extern "C" fn x_sync(file: *mut sqlite3_file, flags: c_int) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.sync(flags); + handle_error(result, Some(SQLITE_IOERR_FSYNC)) +} + +unsafe extern "C" fn x_file_size( + file: *mut sqlite3_file, + pSize: *mut sqlite3_int64, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.file_size(pSize); + handle_error(result, Some(SQLITE_IOERR_FSTAT)) +} + +unsafe extern "C" fn x_lock(file: *mut sqlite3_file, arg2: c_int) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.lock(arg2); + handle_int(result, Some(SQLITE_IOERR_LOCK)) +} + +unsafe extern "C" fn x_unlock(file: *mut sqlite3_file, arg2: c_int) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.unlock(arg2); + handle_int(result, Some(SQLITE_IOERR_UNLOCK)) +} + +unsafe extern "C" fn x_check_reserved_lock( + file: *mut sqlite3_file, + pResOut: *mut c_int, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.check_reserved_lock(pResOut); + handle_error(result, None) +} + +unsafe extern "C" fn x_file_control( + file: *mut sqlite3_file, + op: c_int, + pArg: *mut c_void, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.file_control(op, pArg); + handle_error(result, None) +} + +unsafe extern "C" fn x_sector_size(file: *mut sqlite3_file) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.sector_size(); + handle_int(result, None) +} + +unsafe extern "C" fn x_device_characteristics( + file: *mut sqlite3_file, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.device_characteristics(); + handle_int(result, None) +} + +unsafe extern "C" fn x_shm_map( + file: *mut sqlite3_file, + iPg: c_int, + pgsz: c_int, + arg2: c_int, + arg3: *mut *mut c_void, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.shm_map(iPg, pgsz, arg2, arg3); + handle_error(result, Some(SQLITE_IOERR_SHMMAP)) +} + +unsafe extern "C" fn x_shm_lock( + file: *mut sqlite3_file, + offset: c_int, + n: c_int, + flags: c_int, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.shm_lock(offset, n, flags); + handle_error(result, Some(SQLITE_IOERR_SHMLOCK)) +} + +unsafe extern "C" fn x_shm_barrier(file: *mut sqlite3_file) { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.shm_barrier(); +} + +unsafe extern "C" fn x_shm_unmap( + file: *mut sqlite3_file, + deleteFlag: c_int, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.shm_unmap(deleteFlag); + handle_error(result, Some(SQLITE_IOERR_SHMMAP)) +} + +unsafe extern "C" fn x_fetch( + file: *mut sqlite3_file, + iOfst: sqlite3_int64, + iAmt: c_int, + pp: *mut *mut c_void, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.fetch(iOfst, iAmt, pp); + handle_error(result, None) +} + +unsafe extern "C" fn x_unfetch( + file: *mut sqlite3_file, + iOfst: sqlite3_int64, + p: *mut c_void, +) -> c_int { + let mut f = &mut *file.cast::>(); + let mut aux = f.aux.assume_init_mut(); + let result = aux.unfetch(iOfst, p); + handle_error(result, None) +} + +#[repr(C)] +pub struct FileWithAux { + pub pMethods: Box, + pub aux: MaybeUninit, +} + +/// See sqlite3OsOpenMalloc and sqlite3OsCloseFree dependency on szOsFile on sqlite3_vfs, +/// this implies that ownership of sqlite3_file and any "sub-type", is with sqlite3 +pub unsafe fn prepare_file_ptr( + file_ptr: *mut sqlite3_file, + aux: T, +) -> *const sqlite3_file { + let f = (file_ptr as *mut FileWithAux).as_mut().unwrap(); + f.pMethods = create_io_methods_boxed::(); + f.aux.write(aux); + + file_ptr +} + +pub fn create_io_methods_boxed() -> Box { + let m = sqlite3_io_methods { + iVersion: 3, // this library targets version 3? + xClose: Some(x_close::), + xRead: Some(x_read::), + xWrite: Some(x_write::), + xTruncate: Some(x_truncate::), + xSync: Some(x_sync::), + xFileSize: Some(x_file_size::), + xLock: Some(x_lock::), + xUnlock: Some(x_unlock::), + xCheckReservedLock: Some(x_check_reserved_lock::), + xFileControl: Some(x_file_control::), + xSectorSize: Some(x_sector_size::), + xDeviceCharacteristics: Some(x_device_characteristics::), + xShmMap: Some(x_shm_map::), + xShmLock: Some(x_shm_lock::), + xShmBarrier: Some(x_shm_barrier::), + xShmUnmap: Some(x_shm_unmap::), + xFetch: Some(x_fetch::), + xUnfetch: Some(x_unfetch::), + }; + Box::new(m) +} + +// TODO determine false positive: Valgrind reports mismatch malloc/free? 16B +// VALGRINDFLAGS="--leak-check=full --trace-children=yes --verbose --log-file=leaky.txt" cargo valgrind test diff --git a/src/vfs/mod.rs b/src/vfs/mod.rs new file mode 100644 index 0000000..c269fbc --- /dev/null +++ b/src/vfs/mod.rs @@ -0,0 +1,4 @@ +pub mod file; +pub mod shim; +pub mod traits; +pub mod vfs; diff --git a/src/vfs/shim.rs b/src/vfs/shim.rs new file mode 100644 index 0000000..12bed5b --- /dev/null +++ b/src/vfs/shim.rs @@ -0,0 +1,631 @@ +#![allow(unused)] +#![allow(non_snake_case)] + +use crate::SqliteIoMethods; + +use std::io::{Error, ErrorKind, Result}; + +use super::super::vfs::traits::SqliteVfs; + +use std::{ + os::raw::{c_char, c_int, c_void}, + ptr, +}; + +use crate::ext::{ + sqlite3_file, sqlite3_int64, sqlite3_io_methods, sqlite3_syscall_ptr, sqlite3_vfs, + sqlite3_vfs_find, +}; + +use sqlite3ext_sys::{SQLITE_ERROR, SQLITE_LOCK_NONE, SQLITE_OK}; + +pub struct ShimVfs { + default_vfs: *mut sqlite3_vfs, +} + +impl ShimVfs { + pub fn from_ptr(vfs: *mut sqlite3_vfs) -> Self { + ShimVfs { default_vfs: vfs } + } +} + +impl SqliteVfs for ShimVfs { + fn open( + &mut self, + z_name: *const c_char, + p_file: *mut sqlite3_file, + flags: c_int, + p_out_flags: *mut c_int, + ) -> Result<()> { + unsafe { + if let Some(xOpen) = (*self.default_vfs).xOpen { + let result = xOpen(self.default_vfs, z_name, p_file, flags, p_out_flags); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while opening a file", + )) + } + } else { + Ok(()) + } + } + } + + fn delete(&mut self, z_name: *const c_char, sync_dir: c_int) -> Result<()> { + unsafe { + if let Some(xDelete) = (*self.default_vfs).xDelete { + let result = xDelete(self.default_vfs, z_name, sync_dir); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while removing a file", + )) + } + } else { + Ok(()) + } + } + } + + fn access(&mut self, z_name: *const c_char, flags: c_int, p_res_out: *mut c_int) -> Result<()> { + unsafe { + if let Some(xAccess) = (*self.default_vfs).xAccess { + let result = xAccess(self.default_vfs, z_name, flags, p_res_out); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while determining the permissions of a file", + )) + } + } else { + Ok(()) + } + } + } + + fn full_pathname( + &mut self, + z_name: *const c_char, + n_out: c_int, + z_out: *mut c_char, + ) -> Result<()> { + unsafe { + if let Some(xFullPathname) = (*self.default_vfs).xFullPathname { + let result = xFullPathname(self.default_vfs, z_name, n_out, z_out); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while determining the full pathname of a file", + )) + } + } else { + Ok(()) + } + } + } + + #[cfg(feature = "vfs_loadext")] + fn dl_open(&mut self, z_filename: *const c_char) -> *mut c_void { + unsafe { + if let Some(xDlOpen) = (*self.default_vfs).xDlOpen { + xDlOpen(self.default_vfs, z_filename) + } else { + ptr::null_mut() + } + } + } + + #[cfg(feature = "vfs_loadext")] + fn dl_error(&mut self, n_byte: c_int, z_err_msg: *mut c_char) { + unsafe { + if let Some(xDlError) = (*self.default_vfs).xDlError { + xDlError(self.default_vfs, n_byte, z_err_msg); + } + } + } + + #[cfg(feature = "vfs_loadext")] + fn dl_sym( + &mut self, + arg2: *mut c_void, + z_symbol: *const c_char, + ) -> Option { + unsafe { + if let Some(func) = (*self.default_vfs).xDlSym { + func(self.default_vfs, arg2, z_symbol); + } + None + } + } + + #[cfg(feature = "vfs_loadext")] + fn dl_close(&mut self, arg2: *mut c_void) { + unsafe { + if let Some(xDlClose) = (*self.default_vfs).xDlClose { + xDlClose(self.default_vfs, arg2); + } + } + } + + fn randomness(&mut self, n_byte: c_int, z_out: *mut c_char) -> c_int { + unsafe { + if let Some(xRandomness) = (*self.default_vfs).xRandomness { + xRandomness(self.default_vfs, n_byte, z_out) + } else { + SQLITE_ERROR + } + } + } + + fn sleep(&mut self, microseconds: c_int) -> c_int { + unsafe { + if let Some(xSleep) = (*self.default_vfs).xSleep { + xSleep(self.default_vfs, microseconds) + } else { + SQLITE_ERROR + } + } + } + + fn current_time(&mut self, arg2: *mut f64) -> c_int { + unsafe { + if let Some(xCurrentTime) = (*self.default_vfs).xCurrentTime { + xCurrentTime(self.default_vfs, arg2) + } else { + SQLITE_ERROR + } + } + } + + fn get_last_error(&mut self, arg2: c_int, arg3: *mut c_char) -> Result<()> { + unsafe { + if let Some(xGetLastError) = (*self.default_vfs).xGetLastError { + let result = xGetLastError(self.default_vfs, arg2, arg3); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while determining the last internal error", + )) + } + } else { + Ok(()) + } + } + } + + fn current_time_int64(&mut self, arg2: *mut sqlite3_int64) -> c_int { + unsafe { + if let Some(xCurrentTimeInt64) = (*self.default_vfs).xCurrentTimeInt64 { + xCurrentTimeInt64(self.default_vfs, arg2) + } else { + SQLITE_ERROR + } + } + } + + #[cfg(feature = "vfs_syscall")] + fn set_system_call(&mut self, z_name: *const c_char, arg2: sqlite3_syscall_ptr) -> c_int { + unsafe { + if let Some(xSetSystemCall) = (*self.default_vfs).xSetSystemCall { + xSetSystemCall(self.default_vfs, z_name, arg2) + } else { + SQLITE_ERROR + } + } + } + + #[cfg(feature = "vfs_syscall")] + fn get_system_call(&mut self, z_name: *const c_char) -> sqlite3_syscall_ptr { + unsafe { + if let Some(xGetSystemCall) = (*self.default_vfs).xGetSystemCall { + xGetSystemCall(self.default_vfs, z_name) + } else { + None + } + } + } + + #[cfg(feature = "vfs_syscall")] + fn next_system_call(&mut self, z_name: *const c_char) -> *const c_char { + unsafe { + if let Some(xNextSystemCall) = (*self.default_vfs).xNextSystemCall { + xNextSystemCall(self.default_vfs, z_name) + } else { + ptr::null() + } + } + } +} + +/// See ORIGFILE https://www.sqlite.org/src/file?name=ext/misc/cksumvfs.c +pub struct ShimFile { + methods_ptr: *mut sqlite3_io_methods, + file_ptr: *mut sqlite3_file, +} + +impl ShimFile { + pub fn from_ptr(file_ptr: *mut sqlite3_file) -> Self { + if file_ptr.is_null() { + return Self { + file_ptr, + methods_ptr: ptr::null_mut(), + }; + } + Self { + file_ptr, + methods_ptr: (unsafe { *file_ptr }).pMethods.cast_mut(), + } + } +} + +impl SqliteIoMethods for ShimFile { + fn close(&mut self) -> Result<()> { + unsafe { + if let Some(xClose) = ((*self.methods_ptr).xClose) { + let result = xClose(self.file_ptr); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while attempting to close a file", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn read(&mut self, buf: *mut c_void, s: i32, ofst: i64) -> Result<()> { + unsafe { + if let Some(xRead) = ((*self.methods_ptr).xRead) { + let result = xRead(self.file_ptr, buf, s, ofst); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while reading from a file", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn write( + &mut self, + buf: *const c_void, + i_amt: i32, + i_ofst: i64, + ) -> Result<()> { + unsafe { + if let Some(xWrite) = ((*self.methods_ptr).xWrite) { + let result = xWrite(self.file_ptr, buf, i_amt, i_ofst); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while writing to a file", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn truncate(&mut self, size: i64) -> Result<()> { + unsafe { + if let Some(xTruncate) = ((*self.methods_ptr).xTruncate) { + let result = xTruncate(self.file_ptr, size); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while truncating a file", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn sync(&mut self, flags: c_int) -> Result<()> { + unsafe { + if let Some(xSync) = ((*self.methods_ptr).xSync) { + let result = xSync(self.file_ptr, flags); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while updating the metadata of a file", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn file_size(&mut self, p_size: *mut sqlite3_int64) -> Result<()> { + unsafe { + if let Some(xFileSize) = ((*self.methods_ptr).xFileSize) { + let result = xFileSize(self.file_ptr, p_size); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while determining the size a file", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn lock(&mut self, arg2: c_int) -> Result { + unsafe { + if let Some(xLock) = ((*self.methods_ptr).xLock) { + Ok(xLock(self.file_ptr, arg2)) + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn unlock(&mut self, arg2: c_int) -> Result { + unsafe { + if let Some(xUnlock) = ((*self.methods_ptr).xUnlock) { + Ok(xUnlock(self.file_ptr, arg2)) + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn check_reserved_lock( + &mut self, + p_res_out: *mut c_int, + ) -> Result<()> { + unsafe { + if let Some(xCheckReservedLock) = ((*self.methods_ptr).xCheckReservedLock) { + xCheckReservedLock(self.file_ptr, p_res_out); + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn file_control( + &mut self, + op: c_int, + p_arg: *mut c_void, + ) -> Result<()> { + unsafe { + if let Some(xFileControl) = ((*self.methods_ptr).xFileControl) { + let result = xFileControl(self.file_ptr, op, p_arg); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while setting file parameters", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn sector_size(&mut self) -> Result { + unsafe { + if let Some(xSectorSize) = ((*self.methods_ptr).xSectorSize) { + Ok(xSectorSize(self.file_ptr)) + } else { + Err(Error::new(ErrorKind::Other, "Missing sector size")) + } + } + } + + fn device_characteristics(&mut self) -> Result { + unsafe { + if let Some(xDeviceCharacteristics) = ((*self.methods_ptr).xDeviceCharacteristics) { + Ok(xDeviceCharacteristics(self.file_ptr)) + } else { + Err(Error::new( + ErrorKind::Other, + "Missing device characteristics", + )) + } + } + } + + fn shm_map( + &mut self, + i_pg: c_int, + pgsz: c_int, + arg2: c_int, + arg3: *mut *mut c_void, + ) -> Result<()> { + unsafe { + if let Some(xShmMap) = ((*self.methods_ptr).xShmMap) { + let result = xShmMap(self.file_ptr, i_pg, pgsz, arg2, arg3); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while using mmap", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn shm_lock( + &mut self, + + offset: c_int, + n: c_int, + flags: c_int, + ) -> Result<()> { + unsafe { + if let Some(xShmLock) = ((*self.methods_ptr).xShmLock) { + let result = xShmLock(self.file_ptr, offset, n, flags); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while applying a lock to mmap", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn shm_barrier(&mut self) -> Result<()> { + unsafe { + if let Some(xShmBarrier) = ((*self.methods_ptr).xShmBarrier) { + xShmBarrier(self.file_ptr); + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn shm_unmap(&mut self, delete_flag: c_int) -> Result<()> { + unsafe { + if let Some(xShmUnmap) = ((*self.methods_ptr).xShmUnmap) { + let result = xShmUnmap(self.file_ptr, delete_flag); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while unmapping", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn fetch( + &mut self, + + i_ofst: i64, + i_amt: i32, + pp: *mut *mut c_void, + ) -> Result<()> { + unsafe { + if let Some(xFetch) = ((*self.methods_ptr).xFetch) { + let result = xFetch(self.file_ptr, i_ofst, i_amt, pp); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while fetching", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } + + fn unfetch(&mut self, i_ofst: i64, p: *mut c_void) -> Result<()> { + unsafe { + if let Some(xUnfetch) = ((*self.methods_ptr).xUnfetch) { + let result = xUnfetch(self.file_ptr, i_ofst, p); + if result == SQLITE_OK { + Ok(()) + } else { + Err(Error::new( + ErrorKind::Other, + "An error occurred while unfetching", + )) + } + } else { + Err(Error::new( + ErrorKind::Other, + "An undefined function was called", + )) + } + } + } +} diff --git a/src/vfs/traits.rs b/src/vfs/traits.rs new file mode 100644 index 0000000..b09b3eb --- /dev/null +++ b/src/vfs/traits.rs @@ -0,0 +1,134 @@ +use std::io::Result; + +use std::os::raw::{c_char, c_int, c_void}; + +use crate::ext::sqlite3_file; + +#[cfg(feature = "vfs_loadext")] +use sqlite3ext_sys::sqlite3_vfs; + +#[cfg(feature = "vfs_syscall")] +use sqlite3ext_sys::{sqlite3_syscall_ptr, sqlite3_vfs}; + +/// There was no attempt to rustify the parameters or functions, because the shims, that reuse +/// existing sqlite3 C-based vfs functionality, e.g. unix vfs, also must conform to how the C parameters work. +/// +/// Also, maintaining the function signatures as-is, help keep the existing documentation relevant: +/// e.g.: https://www.sqlite.org/vfs.html +/// +/// Rustifying functions / parameters / types, or "oxidizing" them is left to the user. + +// TODO compare performance of dynamic (indirection via trait) vs static dispatch (just callbacks) +// TODO even better read the asm and verify that this extra indirection was removed +/// See https://www.sqlite.org/c3ref/io_methods.html for hints on how to implement +pub trait SqliteIoMethods { + fn close(&mut self) -> Result<()>; + fn read( + &mut self, + buf: *mut c_void, + i_amt: i32, + i_ofst: i64, + ) -> Result<()>; + fn write( + &mut self, + buf: *const c_void, + i_amt: i32, + i_ofst: i64, + ) -> Result<()>; + fn truncate(&mut self, size: i64) -> Result<()>; + fn sync(&mut self, flags: c_int) -> Result<()>; + fn file_size(&mut self, p_size: *mut i64) -> Result<()>; + + /// Lock the database. Returns whether the requested lock could be acquired. + /// Locking sequence: + /// - The lock is never moved from [LockKind::None] to anything higher than [LockKind::Shared]. + /// - A [LockKind::Pending] is never requested explicitly. + /// - A [LockKind::Shared] is always held when a [LockKind::Reserved] lock is requested + fn lock(&mut self, arg2: c_int) -> Result; + + /// Unlock the database. + fn unlock(&mut self, arg2: c_int) -> Result; + + /// Check if the database this handle points to holds a [LockKind::Reserved], + /// [LockKind::Pending] or [LockKind::Exclusive] lock. + fn check_reserved_lock(&mut self, p_res_out: *mut c_int) + -> Result<()>; + fn file_control( + &mut self, + op: c_int, + p_arg: *mut c_void, + ) -> Result<()>; + fn sector_size(&mut self) -> Result; + fn device_characteristics(&mut self) -> Result; + fn shm_map( + &mut self, + i_pg: c_int, + pgsz: c_int, + arg2: c_int, + arg3: *mut *mut c_void, + ) -> Result<()>; + fn shm_lock(&mut self, offset: c_int, n: c_int, flags: c_int) -> Result<()>; + fn shm_barrier(&mut self) -> Result<()>; + fn shm_unmap(&mut self, delete_flag: c_int) -> Result<()>; + fn fetch(&mut self, i_ofst: i64, i_amt: c_int, pp: *mut *mut c_void) -> Result<()>; + fn unfetch(&mut self, i_ofst: i64, p: *mut c_void) -> Result<()>; +} + +// TODO compare dynamic (indirection via trait) vs static dispatch (just callbacks), same as upstairs +pub trait SqliteVfs { + fn open( + &mut self, + z_name: *const c_char, + p_file: *mut sqlite3_file, + flags: c_int, + p_out_flags: *mut c_int, + ) -> Result<()>; + + fn delete(&mut self, z_name: *const c_char, sync_dir: c_int) -> Result<()>; + + fn access(&mut self, z_name: *const c_char, flags: c_int, p_res_out: *mut c_int) -> Result<()>; + + fn full_pathname( + &mut self, + z_name: *const c_char, + n_out: c_int, + z_out: *mut c_char, + ) -> Result<()>; + + #[cfg(feature = "vfs_loadext")] + fn dl_open(&mut self, z_filename: *const c_char) -> *mut c_void; + + #[cfg(feature = "vfs_loadext")] + fn dl_error(&mut self, n_byte: c_int, z_err_msg: *mut c_char); + + #[cfg(feature = "vfs_loadext")] + fn dl_sym( + &mut self, + arg2: *mut c_void, + z_symbol: *const c_char, + ) -> Option< + unsafe extern "C" fn(arg1: *mut sqlite3_vfs, arg2: *mut c_void, z_symbol: *const c_char), + >; + + #[cfg(feature = "vfs_loadext")] + fn dl_close(&mut self, arg2: *mut c_void); + + fn randomness(&mut self, n_byte: c_int, z_out: *mut c_char) -> c_int; + + fn sleep(&mut self, microseconds: c_int) -> c_int; + + fn current_time(&mut self, arg2: *mut f64) -> c_int; + + fn get_last_error(&mut self, arg2: c_int, arg3: *mut c_char) -> Result<()>; + + fn current_time_int64(&mut self, arg2: *mut i64) -> c_int; + + #[cfg(feature = "vfs_syscall")] + fn set_system_call(&mut self, z_name: *const c_char, arg2: sqlite3_syscall_ptr) -> c_int; + + #[cfg(feature = "vfs_syscall")] + fn get_system_call(&mut self, z_name: *const c_char) -> sqlite3_syscall_ptr; + + #[cfg(feature = "vfs_syscall")] + fn next_system_call(&mut self, z_name: *const c_char) -> *const c_char; +} diff --git a/src/vfs/vfs.rs b/src/vfs/vfs.rs new file mode 100644 index 0000000..f8d139a --- /dev/null +++ b/src/vfs/vfs.rs @@ -0,0 +1,381 @@ +#![allow(non_snake_case)] +#![allow(unused)] + +use crate::ext::{sqlite3_file, sqlite3_int64, sqlite3_syscall_ptr, sqlite3_vfs}; + +use sqlite3ext_sys::{ + SQLITE_CANTOPEN, SQLITE_CANTOPEN_FULLPATH, SQLITE_ERROR, SQLITE_IOERR_ACCESS, + SQLITE_IOERR_DELETE, SQLITE_OK, +}; + +use std::ffi::{CStr, CString}; +use std::os::raw::{c_char, c_int, c_void}; +use std::ptr; +use std::rc::Rc; + +use crate::ext::{sqlite3ext_vfs_find, sqlite3ext_vfs_register}; +use std::io::{Error, ErrorKind, Result}; + +use super::traits::SqliteVfs; + +pub(crate) fn handle_int(result: Result, ext_io_err: Option) -> c_int { + match result { + Ok(i) => i, + Err(e) => { + if let Some(inner_err) = e.into_inner() { + println!("error: {inner_err}"); + } + if let Some(extended) = ext_io_err { + extended + } else { + SQLITE_ERROR + } + } + } +} + +pub(crate) fn handle_error(result: Result<()>, ext_io_err: Option) -> c_int { + match result { + Ok(()) => SQLITE_OK, + Err(e) => { + if let Some(inner_err) = e.into_inner() { + println!("error: {inner_err}"); + } + if let Some(extended) = ext_io_err { + extended + } else { + SQLITE_ERROR + } + } + } +} + +unsafe extern "C" fn x_open( + p_vfs: *mut sqlite3_vfs, + z_name: *const c_char, + p_file: *mut sqlite3_file, + flags: c_int, + p_out_flags: *mut c_int, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.open(z_name, p_file, flags, p_out_flags); + Box::into_raw(b); + Box::into_raw(vfs); + + handle_error(result, Some(SQLITE_CANTOPEN)) +} + +unsafe extern "C" fn x_delete( + p_vfs: *mut sqlite3_vfs, + z_name: *const c_char, + sync_dir: c_int, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.delete(z_name, sync_dir); + Box::into_raw(b); + Box::into_raw(vfs); + + handle_error(result, Some(SQLITE_IOERR_DELETE)) +} + +unsafe extern "C" fn x_access( + p_vfs: *mut sqlite3_vfs, + z_name: *const c_char, + flags: c_int, + p_res_out: *mut c_int, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.access(z_name, flags, p_res_out); + Box::into_raw(b); + Box::into_raw(vfs); + + handle_error(result, Some(SQLITE_IOERR_ACCESS)) +} + +unsafe extern "C" fn x_full_pathname( + p_vfs: *mut sqlite3_vfs, + z_name: *const c_char, + n_out: c_int, + z_out: *mut c_char, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.full_pathname(z_name, n_out, z_out); + Box::into_raw(b); + Box::into_raw(vfs); + + handle_error(result, Some(SQLITE_CANTOPEN_FULLPATH)) +} + +#[cfg(feature = "vfs_loadext")] +unsafe extern "C" fn x_dl_open( + p_vfs: *mut sqlite3_vfs, + z_filename: *const c_char, +) -> *mut c_void { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let out = b.dl_open(z_filename); + Box::into_raw(b); + Box::into_raw(vfs); + + out +} + +#[cfg(feature = "vfs_loadext")] +unsafe extern "C" fn x_dl_error( + p_vfs: *mut sqlite3_vfs, + n_byte: c_int, + z_err_msg: *mut c_char, +) { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + b.dl_error(n_byte, z_err_msg); + Box::into_raw(b); + Box::into_raw(vfs); +} + +#[cfg(feature = "vfs_loadext")] +unsafe extern "C" fn x_dl_sym( + p_vfs: *mut sqlite3_vfs, + p_handle: *mut c_void, + z_symbol: *const c_char, +) -> Option { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + b.dl_sym(p_handle, z_symbol); + Box::into_raw(b); + Box::into_raw(vfs); + + None +} + +#[cfg(feature = "vfs_loadext")] +/// Let Boxes go out of scope, thus drop +unsafe extern "C" fn x_dl_close(p_vfs: *mut sqlite3_vfs, p_handle: *mut c_void) { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + b.dl_close(p_handle); +} + +unsafe extern "C" fn x_randomness( + p_vfs: *mut sqlite3_vfs, + n_byte: c_int, + z_out: *mut c_char, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.randomness(n_byte, z_out); + Box::into_raw(b); + Box::into_raw(vfs); + + result +} + +unsafe extern "C" fn x_sleep(p_vfs: *mut sqlite3_vfs, microseconds: c_int) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.sleep(microseconds); + Box::into_raw(b); + Box::into_raw(vfs); + + result +} + +unsafe extern "C" fn x_current_time( + p_vfs: *mut sqlite3_vfs, + p_time: *mut f64, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.current_time(p_time); + Box::into_raw(b); + Box::into_raw(vfs); + + result +} + +unsafe extern "C" fn x_get_last_error( + p_vfs: *mut sqlite3_vfs, + err_code: c_int, + z_err_msg: *mut c_char, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.get_last_error(err_code, z_err_msg); + Box::into_raw(b); + Box::into_raw(vfs); + + handle_error(result, None) +} + +unsafe extern "C" fn x_current_time_int64( + p_vfs: *mut sqlite3_vfs, + p_time: *mut sqlite3_int64, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.current_time_int64(p_time); + Box::into_raw(b); + Box::into_raw(vfs); + + result +} + +#[cfg(feature = "vfs_syscall")] +unsafe extern "C" fn x_set_system_call( + p_vfs: *mut sqlite3_vfs, + z_name: *const c_char, + p_call: sqlite3_syscall_ptr, +) -> c_int { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.set_system_call(z_name, p_call); + Box::into_raw(b); + Box::into_raw(vfs); + + result +} + +#[cfg(feature = "vfs_syscall")] +unsafe extern "C" fn x_get_system_call( + p_vfs: *mut sqlite3_vfs, + z_name: *const c_char, +) -> sqlite3_syscall_ptr { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.get_system_call(z_name); + Box::into_raw(b); + Box::into_raw(vfs); + + result +} + +#[cfg(feature = "vfs_syscall")] +unsafe extern "C" fn x_next_system_call( + p_vfs: *mut sqlite3_vfs, + z_name: *const c_char, +) -> *const c_char { + let mut vfs = Box::::from_raw(p_vfs); + let mut b = Box::::from_raw(vfs.pAppData.cast::()); + + let result = b.next_system_call(z_name); + Box::into_raw(b); + Box::into_raw(vfs); + + result +} + +pub fn create_vfs( + aux: T, + name_ptr: *const c_char, + max_path_name_size: i32, + vfs_file_size: i32, +) -> sqlite3_vfs { + unsafe { + let vfs_ptr = Box::into_raw(Box::::new(aux)); + + /// According to the documentation: + /// At least vfs_file_size bytes of memory are allocated by SQLite to hold the sqlite3_file + /// structure passed as the third argument to xOpen. The xOpen method does not have to + /// allocate the structure; it should just fill it in. + sqlite3_vfs { + iVersion: 3, + pNext: ptr::null_mut(), + pAppData: vfs_ptr.cast(), + // raw box pointers sizes are all the same + szOsFile: vfs_file_size, + mxPathname: max_path_name_size, + zName: name_ptr, + + xOpen: Some(x_open::), + xDelete: Some(x_delete::), + xAccess: Some(x_access::), + xFullPathname: Some(x_full_pathname::), + + /// The following four VFS methods: + /// + /// xDlOpen + /// xDlError + /// xDlSym + /// xDlClose + /// + /// are supposed to implement the functionality needed by SQLite to load + /// extensions compiled as shared objects. + #[cfg(feature = "vfs_loadext")] + xDlOpen: Some(x_dl_open::), + #[cfg(feature = "vfs_loadext")] + xDlError: Some(x_dl_error::), + #[cfg(feature = "vfs_loadext")] + xDlSym: Some(x_dl_sym::), + #[cfg(feature = "vfs_loadext")] + xDlClose: Some(x_dl_close::), + + #[cfg(not(feature = "vfs_loadext"))] + xDlOpen: None, + #[cfg(not(feature = "vfs_loadext"))] + xDlError: None, + #[cfg(not(feature = "vfs_loadext"))] + xDlSym: None, + #[cfg(not(feature = "vfs_loadext"))] + xDlClose: None, + + xRandomness: Some(x_randomness::), + xSleep: Some(x_sleep::), + xCurrentTime: Some(x_current_time::), + xGetLastError: Some(x_get_last_error::), + xCurrentTimeInt64: Some(x_current_time_int64::), + + #[cfg(feature = "vfs_syscall")] + xSetSystemCall: Some(x_set_system_call::), + #[cfg(feature = "vfs_syscall")] + xGetSystemCall: Some(x_get_system_call::), + #[cfg(feature = "vfs_syscall")] + xNextSystemCall: Some(x_next_system_call::), + + #[cfg(not(feature = "vfs_syscall"))] + xSetSystemCall: None, + #[cfg(not(feature = "vfs_syscall"))] + xGetSystemCall: None, + #[cfg(not(feature = "vfs_syscall"))] + xNextSystemCall: None, + } + } +} + +fn handle_vfs_result(result: i32) -> crate::Result<()> { + if result == SQLITE_OK { + Ok(()) + } else { + Err(crate::errors::Error::new_message(format!( + "sqlite3_vfs_register failed with error code: {}", + result + ))) + } +} + +pub fn register_boxed_vfs(vfs: sqlite3_vfs, make_default: bool) -> crate::Result<()> { + let translate_to_int = if make_default { 1 } else { 0 }; + + let boxed_vfs = Box::into_raw(Box::new(vfs)); + + let result = unsafe { sqlite3ext_vfs_register(boxed_vfs, translate_to_int) }; + + handle_vfs_result(result) +} diff --git a/test.sql b/test.sql deleted file mode 100644 index 0ceaba3..0000000 --- a/test.sql +++ /dev/null @@ -1,8 +0,0 @@ -.mode box -.header on - -.load target/debug/examples/libin - -select * -from vtab_in('xxx') -where y in (select value from json_each('[1,2,"alex", "puppy"]')); diff --git a/tests/test_mem_vfs.rs b/tests/test_mem_vfs.rs new file mode 100644 index 0000000..4558e15 --- /dev/null +++ b/tests/test_mem_vfs.rs @@ -0,0 +1,79 @@ +#![allow(unused)] + +include!("../include/mem_vfs.in.rs"); + +#[derive(Debug)] +struct Person { + id: i32, + name: String, + data: Option>, +} + +#[cfg(test)] +mod tests { + use super::*; + + use rusqlite::{ffi::sqlite3_auto_extension, Connection, OpenFlags, Result}; + + #[test] + fn test_rusqlite_auto_extension() -> Result<()> { + unsafe { + sqlite3_auto_extension(Some(std::mem::transmute(sqlite3_memvfs_init as *const ()))); + } + + // let conn = Connection::open_in_memory()?; + // conn.execute("ATTACH DATABASE mem_vfs_uri() AS inmem;", ()); + + // open in memory first to run faux_sqlite_extension_init2, + // to register the new vfs, in the auto extension callback + // in the following open_with_flags_and_vfs connection + // this workaround would be unnecessary if a Option would + // be passed down to the callback function besides the sqlite3 ptr + // also large parts of ext.rs would be unnecessary + // if the end user could decide whether to use rusqlite or libsqlite3 + // by placing a conditional cfg there instead + // in any case, this should be documented properly -- TODO + let _conn = Connection::open_in_memory()?; + + _conn.close(); + + let conn = Connection::open_with_flags_and_vfs( + "db/100-bytes.db", + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE, + EXTENSION_NAME, + )?; + + conn.execute( + "CREATE TABLE person ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + data BLOB + )", + (), // empty list of parameters. + )?; + let me = Person { + id: 0, + name: "Batman".to_string(), + data: None, + }; + conn.execute( + "INSERT INTO person (name, data) VALUES (?1, ?2)", + (&me.name, &me.data), + )?; + + let mut stmt = conn.prepare("SELECT id, name, data FROM person")?; + let person_iter = stmt.query_map([], |row| { + Ok(Person { + id: row.get(0)?, + name: row.get(1)?, + data: row.get(2)?, + }) + })?; + + for person in person_iter { + println!("Found person {:?}", person.unwrap()); + } + + Ok(()) + } +}