Only pass filtered columns to powersync_diff

Author: simolus3

crates/core/src/diff.rs

@@ -3,6 +3,7 @@ extern crate alloc;
 use alloc::format;
 use alloc::string::{String, ToString};
 use core::ffi::c_int;
+use core::slice;
 
 use sqlite::ResultCode;
 use sqlite_nostd as sqlite;
@@ -19,20 +20,11 @@ fn powersync_diff_impl(
 ) -> Result<String, SQLiteError> {
     let data_old = args[0].text();
     let data_new = args[1].text();
-    let ignore_removed = args.get(2).map_or(false, |v| v.int() != 0);
 
-    diff_objects_with_options(data_old, data_new, ignore_removed)
+    diff_objects(data_old, data_new)
 }
 
-/// Returns a JSON object containing entries from [data_new] that are not present in [data_old].
-///
-/// When [ignore_removed_columns] is set, columns that are present in [data_old] but not in
-/// [data_new] will not be present in the returned object. Otherwise, they will be set to `null`.
-fn diff_objects_with_options(
-    data_old: &str,
-    data_new: &str,
-    ignore_removed_columns: bool,
-) -> Result<String, SQLiteError> {
+pub fn diff_objects(data_old: &str, data_new: &str) -> Result<String, SQLiteError> {
     let v_new: json::Value = json::from_str(data_new)?;
     let v_old: json::Value = json::from_str(data_old)?;
 
@@ -47,11 +39,9 @@ fn diff_objects_with_options(
         }
 
         // Add missing nulls to left
-        if !ignore_removed_columns {
-            for key in right.keys() {
-                if !left.contains_key(key) {
-                    left.insert(key.clone(), json::Value::Null);
-                }
+        for key in right.keys() {
+            if !left.contains_key(key) {
+                left.insert(key.clone(), json::Value::Null);
             }
         }
 
@@ -86,28 +76,13 @@ pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> {
         None,
     )?;
 
-    db.create_function_v2(
-        "powersync_diff",
-        3,
-        sqlite::UTF8 | sqlite::DETERMINISTIC,
-        None,
-        Some(powersync_diff),
-        None,
-        None,
-        None,
-    )?;
-
     Ok(())
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    fn diff_objects(data_old: &str, data_new: &str) -> Result<String, SQLiteError> {
-        diff_objects_with_options(data_old, data_new, false)
-    }
-
     #[test]
     fn basic_diff_test() {
         assert_eq!(diff_objects("{}", "{}").unwrap(), "{}");

crates/core/src/schema/table_info.rs

@@ -50,7 +50,7 @@ impl TableInfo {
             flags = flags.set_flag(TableInfoFlags::INSERT_ONLY, insert_only);
             flags = flags.set_flag(TableInfoFlags::INCLUDE_METADATA, include_metadata);
             flags = flags.set_flag(
-                TableInfoFlags::INCLUDE_OLD_ONLY_IF_CHANGED,
+                TableInfoFlags::INCLUDE_OLD_ONLY_WHEN_CHANGED,
                 include_old_only_when_changed,
             );
 
@@ -104,7 +104,7 @@ impl TableInfoFlags {
     pub const LOCAL_ONLY: u32 = 1;
     pub const INSERT_ONLY: u32 = 2;
     pub const INCLUDE_METADATA: u32 = 4;
-    pub const INCLUDE_OLD_ONLY_IF_CHANGED: u32 = 8;
+    pub const INCLUDE_OLD_ONLY_WHEN_CHANGED: u32 = 8;
 
     pub const fn local_only(self) -> bool {
         self.0 & Self::LOCAL_ONLY != 0
@@ -118,8 +118,8 @@ impl TableInfoFlags {
         self.0 & Self::INCLUDE_METADATA != 0
     }
 
-    pub const fn include_old_only_if_changed(self) -> bool {
-        self.0 & Self::INCLUDE_OLD_ONLY_IF_CHANGED != 0
+    pub const fn include_old_only_when_changed(self) -> bool {
+        self.0 & Self::INCLUDE_OLD_ONLY_WHEN_CHANGED != 0
     }
 
     const fn with_flag(self, flag: u32) -> Self {

crates/core/src/views.rs

@@ -291,11 +291,30 @@ fn powersync_trigger_update_sql_impl(
         }
     };
 
-    if table_info.flags.include_old_only_if_changed() {
-        // We're setting ignore_removed_columns here because values that are present in the new
-        // values but not in the old values should not lead to a null entry.
-        old_values_fragment = old_values_fragment
-            .map(|f| format!("json(powersync_diff({json_fragment_new}, {f}, TRUE))"));
+    if table_info.flags.include_old_only_when_changed() {
+        old_values_fragment = match old_values_fragment {
+            None => None,
+            Some(f) => {
+                let filtered_new_fragment = match &table_info.diff_include_old {
+                    // When include_old_only_when_changed is combined with a column filter, make sure we
+                    // only include the powersync_diff of columns matched by the filter.
+                    Some(DiffIncludeOld::OnlyForColumns { columns }) => {
+                        let mut iterator = columns.iter();
+                        let mut columns =
+                            streaming_iterator::from_fn(|| -> Option<Result<&str, ResultCode>> {
+                                Some(Ok(iterator.next()?.as_str()))
+                            });
+
+                        Cow::Owned(json_object_fragment("NEW", &mut columns)?)
+                    }
+                    _ => Cow::Borrowed(json_fragment_new.as_str()),
+                };
+
+                Some(format!(
+                    "json(powersync_diff({filtered_new_fragment}, {f}))"
+                ))
+            }
+        }
     }
 
     let old_fragment: Cow<'static, str> = match old_values_fragment {

dart/pubspec.lock

@@ -5,39 +5,34 @@ packages:
     dependency: transitive
     description:
       name: _fe_analyzer_shared
-      sha256: "03f6da266a27a4538a69295ec142cb5717d7d4e5727b84658b63e1e1509bac9c"
+      sha256: e55636ed79578b9abca5fecf9437947798f5ef7456308b5cb85720b793eac92f
       url: "https://pub.dev"
     source: hosted
-    version: "79.0.0"
-  _macros:
-    dependency: transitive
-    description: dart
-    source: sdk
-    version: "0.3.3"
+    version: "82.0.0"
   analyzer:
     dependency: transitive
     description:
       name: analyzer
-      sha256: c9040fc56483c22a5e04a9f6a251313118b1a3c42423770623128fa484115643
+      sha256: "13c1e6c6fd460522ea840abec3f677cc226f5fec7872c04ad7b425517ccf54f7"
       url: "https://pub.dev"
     source: hosted
-    version: "7.2.0"
+    version: "7.4.4"
   args:
     dependency: transitive
     description:
       name: args
-      sha256: bf9f5caeea8d8fe6721a9c358dd8a5c1947b27f1cfaa18b39c301273594919e6
+      sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04
       url: "https://pub.dev"
     source: hosted
-    version: "2.6.0"
+    version: "2.7.0"
   async:
     dependency: transitive
     description:
       name: async
-      sha256: d2872f9c19731c2e5f10444b14686eb7cc85c76274bd6c16e1816bff9a3bab63
+      sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb"
       url: "https://pub.dev"
     source: hosted
-    version: "2.12.0"
+    version: "2.13.0"
   boolean_selector:
     dependency: transitive
     description:
@@ -46,6 +41,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.1.2"
+  cli_config:
+    dependency: transitive
+    description:
+      name: cli_config
+      sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.2.0"
   collection:
     dependency: transitive
     description:
@@ -66,10 +69,10 @@ packages:
     dependency: transitive
     description:
       name: coverage
-      sha256: e3493833ea012784c740e341952298f1cc77f1f01b1bbc3eb4eecf6984fb7f43
+      sha256: "9086475ef2da7102a0c0a4e37e1e30707e7fb7b6d28c209f559a9c5f8ce42016"
       url: "https://pub.dev"
     source: hosted
-    version: "1.11.1"
+    version: "1.12.0"
   crypto:
     dependency: transitive
     description:
@@ -82,10 +85,10 @@ packages:
     dependency: transitive
     description:
       name: ffi
-      sha256: "16ed7b077ef01ad6170a3d0c57caa4a112a38d7a2ed5602e0aca9ca6f3d98da6"
+      sha256: "289279317b4b16eb2bb7e271abccd4bf84ec9bdcbe999e278a94b804f5630418"
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.3"
+    version: "2.1.4"
   file:
     dependency: transitive
     description:
@@ -138,10 +141,10 @@ packages:
     dependency: transitive
     description:
       name: js
-      sha256: c1b2e9b5ea78c45e1a0788d29606ba27dc5f71f019f32ca5140f61ef071838cf
+      sha256: "53385261521cc4a0c4658fd0ad07a7d14591cf8fc33abbceae306ddb974888dc"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.1"
+    version: "0.7.2"
   logging:
     dependency: transitive
     description:
@@ -150,14 +153,6 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.3.0"
-  macros:
-    dependency: transitive
-    description:
-      name: macros
-      sha256: "1d9e801cd66f7ea3663c45fc708450db1fa57f988142c64289142c9b7ee80656"
-      url: "https://pub.dev"
-    source: hosted
-    version: "0.1.3-main.0"
   matcher:
     dependency: transitive
     description:
@@ -194,10 +189,10 @@ packages:
     dependency: transitive
     description:
       name: package_config
-      sha256: "92d4488434b520a62570293fbd33bb556c7d49230791c1b4bbd973baf6d2dc67"
+      sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.1"
+    version: "2.2.0"
   path:
     dependency: transitive
     description:
@@ -218,10 +213,10 @@ packages:
     dependency: transitive
     description:
       name: pub_semver
-      sha256: "7b3cfbf654f3edd0c6298ecd5be782ce997ddf0e00531b9464b55245185bbbbd"
+      sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585"
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.5"
+    version: "2.2.0"
   shelf:
     dependency: transitive
     description:
@@ -250,10 +245,10 @@ packages:
     dependency: transitive
     description:
       name: shelf_web_socket
-      sha256: cc36c297b52866d203dbf9332263c94becc2fe0ceaa9681d07b6ef9807023b67
+      sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925"
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.1"
+    version: "3.0.0"
   source_map_stack_trace:
     dependency: transitive
     description:
@@ -282,10 +277,10 @@ packages:
     dependency: "direct main"
     description:
       name: sqlite3
-      sha256: "35d3726fe18ab1463403a5cc8d97dbc81f2a0b08082e8173851363fcc97b6627"
+      sha256: "310af39c40dd0bb2058538333c9d9840a2725ae0b9f77e4fd09ad6696aa8f66e"
       url: "https://pub.dev"
     source: hosted
-    version: "2.7.2"
+    version: "2.7.5"
   stack_trace:
     dependency: transitive
     description:
@@ -322,10 +317,10 @@ packages:
     dependency: "direct dev"
     description:
       name: test
-      sha256: "8391fbe68d520daf2314121764d38e37f934c02fd7301ad18307bd93bd6b725d"
+      sha256: "301b213cd241ca982e9ba50266bd3f5bd1ea33f1455554c5abb85d1be0e2d87e"
       url: "https://pub.dev"
     source: hosted
-    version: "1.25.14"
+    version: "1.25.15"
   test_api:
     dependency: transitive
     description:
@@ -370,26 +365,26 @@ packages:
     dependency: transitive
     description:
       name: web
-      sha256: cd3543bd5798f6ad290ea73d210f423502e71900302dde696f8bff84bf89a1cb
+      sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a"
       url: "https://pub.dev"
     source: hosted
-    version: "1.1.0"
+    version: "1.1.1"
   web_socket:
     dependency: transitive
     description:
       name: web_socket
-      sha256: "3c12d96c0c9a4eec095246debcea7b86c0324f22df69893d538fcc6f1b8cce83"
+      sha256: bfe6f435f6ec49cb6c01da1e275ae4228719e59a6b067048c51e72d9d63bcc4b
       url: "https://pub.dev"
     source: hosted
-    version: "0.1.6"
+    version: "1.0.0"
   web_socket_channel:
     dependency: transitive
     description:
       name: web_socket_channel
-      sha256: "0b8e2457400d8a859b7b2030786835a28a8e80836ef64402abef392ff4f1d0e5"
+      sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8
       url: "https://pub.dev"
     source: hosted
-    version: "3.0.2"
+    version: "3.0.3"
   webkit_inspection_protocol:
     dependency: transitive
     description:
@@ -407,4 +402,4 @@ packages:
     source: hosted
     version: "3.1.3"
 sdks:
-  dart: ">=3.5.0 <4.0.0"
+  dart: ">=3.7.0 <4.0.0"