From 916e732ec9339b4aa1f2a9d433eddd44c94d0130 Mon Sep 17 00:00:00 2001 From: b Date: Thu, 27 Feb 2025 16:16:06 +0100 Subject: [PATCH 1/4] Added WAL Index format --- database/sqlite3-wal-index.ksy | 120 +++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 database/sqlite3-wal-index.ksy diff --git a/database/sqlite3-wal-index.ksy b/database/sqlite3-wal-index.ksy new file mode 100644 index 000000000..580170e83 --- /dev/null +++ b/database/sqlite3-wal-index.ksy @@ -0,0 +1,120 @@ +meta: + id: sqlite3_wal_index + file-extension: + - sqlite-shm + - db-shm + - db3-shm + - sqlite3-shm +doc: | + SQLite3 is a popular serverless SQL engine, implemented as a library + to be used within other applications. It keeps its databases as + regular disk files. + + Write-Ahead Logging (WAL) is a journaling mode that enhances + performance and allow more concurrency than the default rollback + journal. + + The WAL Index is a shared memory object that allows faster lookup of + pages stored in the WAL journal. +doc-ref: + - https://www.sqlite.org/wal.html + - https://sqlite.org/src/file?name=src/wal.c&ci=trunk +seq: + - id: header + type: header + - id: header2 + type: header + - id: ckpt_info + type: ckpt_info + - id: first_block + type: index_block_one + - id: index_blocks + type: index_block + repeat: eos + +types: + header: + seq: + - id: version + type: b32 + - id: unused + type: b32 + - id: change + type: b32 + - id: init + type: b8 + - id: big_end_cksum + type: b8 + - id: page + type: b16 + - id: mx_frame + type: b32 + - id: n_page + type: b32 + - id: frame_cksum_0 + type: b32 + - id: frame_cksum_1 + type: b32 + - id: salt_0 + type: b32 + - id: salt_1 + type: b32 + - id: cksum_0 + type: b32 + - id: cksum_1 + type: b32 + ckpt_info: + seq: + - id: n_backfill + type: b32 + - id: read_mark + type: read_mark_array + - id: lock + type: lock_array + - id: n_backfill_attempted + type: b32 + - id: not_used + type: b32 + lock_array: + seq: + - id: lock + type: b8 + repeat: expr + repeat-expr: 8 # SQLITE_SHM_NLOCK + read_mark_array: + seq: + - id: read_mark + type: b32 + repeat: expr + repeat-expr: 5 # WAL_NREADER + index_block_one: + seq: + - id: page_mapping + type: page_mapping_one + - id: hash_table + type: hash_table + page_mapping_one: + seq: + - id: page_number + type: b32 + repeat: expr + repeat-expr: 4062 # HASHTABLE_NPAGE_ONE + index_block: + seq: + - id: page_mapping + type: page_mapping_one + - id: hash_table + type: hash_table + page_mapping: + seq: + - id: page_number + type: b32 + repeat: expr + repeat-expr: 4096 # HASHTABLE_NPAGE + hash_table: + seq: + - id: hash + type: b16 + repeat: expr + repeat-expr: 2*4096 # HASHTABLE_NSLOT + From 5d354d1c90b7f1caa822aa9a062a86f6691cbf61 Mon Sep 17 00:00:00 2001 From: b Date: Thu, 27 Feb 2025 16:30:34 +0100 Subject: [PATCH 2/4] Added Sqlite3 WAL format --- database/sqlite3-wal.ksy | 67 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 database/sqlite3-wal.ksy diff --git a/database/sqlite3-wal.ksy b/database/sqlite3-wal.ksy new file mode 100644 index 000000000..217c2946b --- /dev/null +++ b/database/sqlite3-wal.ksy @@ -0,0 +1,67 @@ +meta: + id: sqlite3_wal + file-extension: + - sqlite-wal + - db-wal + - db3-wal + - sqlite3-wal + endian: be + license: CC0-1.0 +doc: | + SQLite3 is a popular serverless SQL engine, implemented as a library + to be used within other applications. It keeps its databases as + regular disk files. + + Write-Ahead Logging (WAL) is a journaling mode that enhances + performance and allow more concurrency than the default rollback + journal. +doc-ref: + - https://www.sqlite.org/wal.html + - https://sqlite.org/src/file?name=src/wal.c&ci=trunk +seq: + - id: header + type: header + - id: frames + type: frame + repeat: eos +types: + header: + seq: + - id: magic + type: u4 + valid: + any-of: [0x377f0682, 0x377f0683] + - id: file_format_version + type: u4 + - id: database_page_size + type: u4 + - id: checkpoint_sequence_number + type: u4 + - id: salt1 + type: u4 + - id: salt2 + type: u4 + - id: checksum1 + type: u4 + - id: checksum2 + type: u4 + frame: + seq: + - id: frame_header + type: frame_header + - id: page_data + size: _root.header.database_page_size + frame_header: + seq: + - id: page_number + type: u4 + - id: commit_db_pages + type: u4 + - id: salt1 + type: u4 + - id: salt2 + type: u4 + - id: checksum1 + type: u4 + - id: checksum2 + type: u4 From e490667fbea5141ec058986bcfdcca68bd37b40a Mon Sep 17 00:00:00 2001 From: b Date: Thu, 27 Feb 2025 16:30:52 +0100 Subject: [PATCH 3/4] Documentation about WAL endianness --- database/sqlite3-wal-index.ksy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/database/sqlite3-wal-index.ksy b/database/sqlite3-wal-index.ksy index 580170e83..3db590229 100644 --- a/database/sqlite3-wal-index.ksy +++ b/database/sqlite3-wal-index.ksy @@ -5,6 +5,7 @@ meta: - db-shm - db3-shm - sqlite3-shm + license: CC0-1.0 doc: | SQLite3 is a popular serverless SQL engine, implemented as a library to be used within other applications. It keeps its databases as @@ -15,7 +16,8 @@ doc: | journal. The WAL Index is a shared memory object that allows faster lookup of - pages stored in the WAL journal. + pages stored in the WAL journal. Endianness of the stored values is + platform-dependent. doc-ref: - https://www.sqlite.org/wal.html - https://sqlite.org/src/file?name=src/wal.c&ci=trunk @@ -31,7 +33,6 @@ seq: - id: index_blocks type: index_block repeat: eos - types: header: seq: From 0721b8475735b7d23634bbece5ad6f8f75aae9ab Mon Sep 17 00:00:00 2001 From: b Date: Thu, 27 Feb 2025 16:42:24 +0100 Subject: [PATCH 4/4] SQLite3 WAL - Style Guide modifications --- database/{sqlite3-wal.ksy => sqlite3_wal.ksy} | 4 ++-- database/{sqlite3-wal-index.ksy => sqlite3_wal_index.ksy} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename database/{sqlite3-wal.ksy => sqlite3_wal.ksy} (94%) rename database/{sqlite3-wal-index.ksy => sqlite3_wal_index.ksy} (99%) diff --git a/database/sqlite3-wal.ksy b/database/sqlite3_wal.ksy similarity index 94% rename from database/sqlite3-wal.ksy rename to database/sqlite3_wal.ksy index 217c2946b..a3457f273 100644 --- a/database/sqlite3-wal.ksy +++ b/database/sqlite3_wal.ksy @@ -33,7 +33,7 @@ types: any-of: [0x377f0682, 0x377f0683] - id: file_format_version type: u4 - - id: database_page_size + - id: len_database_page type: u4 - id: checkpoint_sequence_number type: u4 @@ -50,7 +50,7 @@ types: - id: frame_header type: frame_header - id: page_data - size: _root.header.database_page_size + size: _root.header.len_database_page frame_header: seq: - id: page_number diff --git a/database/sqlite3-wal-index.ksy b/database/sqlite3_wal_index.ksy similarity index 99% rename from database/sqlite3-wal-index.ksy rename to database/sqlite3_wal_index.ksy index 3db590229..d34a47936 100644 --- a/database/sqlite3-wal-index.ksy +++ b/database/sqlite3_wal_index.ksy @@ -38,7 +38,7 @@ types: seq: - id: version type: b32 - - id: unused + - id: reserved type: b32 - id: change type: b32