From 8605f4e01cecde7ab94a571088268fd65663cc53 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 9 Jul 2025 15:29:51 +0200 Subject: [PATCH 01/33] Basic stream API --- .../powersync_core/lib/src/sync/stream.dart | 28 +++++++++++++++++++ .../lib/src/sync/sync_status.dart | 16 +++++++++++ 2 files changed, 44 insertions(+) create mode 100644 packages/powersync_core/lib/src/sync/stream.dart diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart new file mode 100644 index 00000000..fde95101 --- /dev/null +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -0,0 +1,28 @@ +import 'sync_status.dart'; + +abstract interface class SyncStreamDescription { + String get name; + Map? get parameters; +} + +abstract interface class SyncSubscriptionDefinition + extends SyncStreamDescription { + bool get active; + DateTime? get expiresAt; + bool get hasSynced; + DateTime? lastSyncedAt; +} + +abstract interface class SyncStream extends SyncStreamDescription { + Future subscribe({ + Duration? ttl, + BucketPriority? priority, + Map? parameters, + }); +} + +abstract interface class SyncStreamSubscription + implements SyncStreamDescription, SyncSubscriptionDefinition { + Future waitForFirstSync(); + Future unsubscribe({bool immediately = false}); +} diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index 62c48df1..3c3f89cf 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -5,6 +5,7 @@ import 'package:meta/meta.dart'; import 'bucket_storage.dart'; import 'protocol.dart'; +import 'stream.dart'; final class SyncStatus { /// true if currently connected. @@ -54,6 +55,8 @@ final class SyncStatus { final List priorityStatusEntries; + final List? activeSubscriptions; + const SyncStatus({ this.connected = false, this.connecting = false, @@ -65,6 +68,7 @@ final class SyncStatus { this.downloadError, this.uploadError, this.priorityStatusEntries = const [], + this.activeSubscriptions, }); @override @@ -174,6 +178,18 @@ final class SyncStatus { static const _statusEquality = ListEquality(); } +final class SyncStreamStatus { + final SyncSubscriptionDefinition subscription; + final BucketPriority priority; + + final bool isDefault; + final ProgressWithOperations progress; + + @internal + SyncStreamStatus( + this.subscription, this.priority, this.isDefault, this.progress); +} + /// The priority of a PowerSync bucket. extension type const BucketPriority._(int priorityNumber) { static const _highest = 0; From 0b248a224201eb2c72c66499d33d27cb781b47f1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 9 Jul 2025 17:34:41 +0200 Subject: [PATCH 02/33] Map stream status from core --- .../lib/src/database/powersync_db_mixin.dart | 6 ++ .../lib/src/sync/instruction.dart | 7 ++ .../lib/src/sync/mutable_sync_status.dart | 2 + .../powersync_core/lib/src/sync/stream.dart | 57 +++++++++++- .../lib/src/sync/sync_status.dart | 88 ++++++++++++++----- 5 files changed, 138 insertions(+), 22 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index a53b5049..e6134293 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -17,6 +17,8 @@ import 'package:powersync_core/src/schema_logic.dart' as schema_logic; import 'package:powersync_core/src/sync/options.dart'; import 'package:powersync_core/src/sync/sync_status.dart'; +import '../sync/stream.dart'; + mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Schema used for the local database. Schema get schema; @@ -141,6 +143,10 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { return isInitialized; } + Future> get activeSubscriptions { + throw UnimplementedError(); + } + Future _updateHasSynced() async { // Query the database to see if any data has been synced. final result = await database.getAll( diff --git a/packages/powersync_core/lib/src/sync/instruction.dart b/packages/powersync_core/lib/src/sync/instruction.dart index f0146e8e..6344e017 100644 --- a/packages/powersync_core/lib/src/sync/instruction.dart +++ b/packages/powersync_core/lib/src/sync/instruction.dart @@ -1,3 +1,4 @@ +import 'stream.dart'; import 'sync_status.dart'; /// An internal instruction emitted by the sync client in the core extension in @@ -62,12 +63,14 @@ final class CoreSyncStatus { final bool connecting; final List priorityStatus; final DownloadProgress? downloading; + final List? streams; CoreSyncStatus({ required this.connected, required this.connecting, required this.priorityStatus, required this.downloading, + required this.streams, }); factory CoreSyncStatus.fromJson(Map json) { @@ -82,6 +85,10 @@ final class CoreSyncStatus { null => null, final raw as Map => DownloadProgress.fromJson(raw), }, + streams: (json['stream'] as List?) + ?.map((e) => + CoreActiveStreamSubscription.fromJson(e as Map)) + .toList(), ); } diff --git a/packages/powersync_core/lib/src/sync/mutable_sync_status.dart b/packages/powersync_core/lib/src/sync/mutable_sync_status.dart index 23e3becb..2707cb58 100644 --- a/packages/powersync_core/lib/src/sync/mutable_sync_status.dart +++ b/packages/powersync_core/lib/src/sync/mutable_sync_status.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:collection/collection.dart'; import 'instruction.dart'; +import 'stream.dart'; import 'sync_status.dart'; import 'bucket_storage.dart'; import 'protocol.dart'; @@ -15,6 +16,7 @@ final class MutableSyncStatus { InternalSyncDownloadProgress? downloadProgress; List priorityStatusEntries = const []; + List streams = const []; DateTime? lastSyncedAt; diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index fde95101..6eada49c 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -1,3 +1,5 @@ +import 'package:meta/meta.dart'; + import 'sync_status.dart'; abstract interface class SyncStreamDescription { @@ -10,7 +12,7 @@ abstract interface class SyncSubscriptionDefinition bool get active; DateTime? get expiresAt; bool get hasSynced; - DateTime? lastSyncedAt; + DateTime? get lastSyncedAt; } abstract interface class SyncStream extends SyncStreamDescription { @@ -26,3 +28,56 @@ abstract interface class SyncStreamSubscription Future waitForFirstSync(); Future unsubscribe({bool immediately = false}); } + +/// An `ActiveStreamSubscription` as part of the sync status in Rust. +@internal +final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { + @override + final String name; + @override + final Map? parameters; + final BucketPriority priority; + final List associatedBuckets; + @override + final bool active; + final bool isDefault; + @override + final DateTime? expiresAt; + @override + final DateTime? lastSyncedAt; + + @override + bool get hasSynced => lastSyncedAt != null; + + CoreActiveStreamSubscription._({ + required this.name, + required this.parameters, + required this.priority, + required this.associatedBuckets, + required this.active, + required this.isDefault, + required this.expiresAt, + required this.lastSyncedAt, + }); + + factory CoreActiveStreamSubscription.fromJson(Map json) { + return CoreActiveStreamSubscription._( + name: json['name'] as String, + parameters: json['parameters'] as Map, + priority: BucketPriority(json['priority'] as int), + associatedBuckets: (json['associated_buckets'] as List).cast(), + active: json['active'] as bool, + isDefault: json['is_default'] as bool, + expiresAt: switch (json['expires_at']) { + null => null, + final timestamp as int => + DateTime.fromMillisecondsSinceEpoch(timestamp * 1000), + }, + lastSyncedAt: switch (json['last_synced_at']) { + null => null, + final timestamp as int => + DateTime.fromMillisecondsSinceEpoch(timestamp * 1000), + }, + ); + } +} diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index 3c3f89cf..df616988 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -2,6 +2,7 @@ import 'dart:math'; import 'package:collection/collection.dart'; import 'package:meta/meta.dart'; +import '../database/powersync_database.dart'; import 'bucket_storage.dart'; import 'protocol.dart'; @@ -55,7 +56,7 @@ final class SyncStatus { final List priorityStatusEntries; - final List? activeSubscriptions; + final List? _internalSubscriptions; const SyncStatus({ this.connected = false, @@ -68,8 +69,8 @@ final class SyncStatus { this.downloadError, this.uploadError, this.priorityStatusEntries = const [], - this.activeSubscriptions, - }); + List? streamSubscriptions, + }) : _internalSubscriptions = streamSubscriptions; @override bool operator ==(Object other) { @@ -82,8 +83,10 @@ final class SyncStatus { other.uploadError == uploadError && other.lastSyncedAt == lastSyncedAt && other.hasSynced == hasSynced && - _statusEquality.equals( + _listEquality.equals( other.priorityStatusEntries, priorityStatusEntries) && + _listEquality.equals( + other._internalSubscriptions, _internalSubscriptions) && other.downloadProgress == downloadProgress); } @@ -114,6 +117,18 @@ final class SyncStatus { ); } + /// All sync streams currently being tracked in this subscription. + /// + /// This returns null when the sync stream is currently being opened and we + /// don't have reliable information about all included streams yet (in that + /// state, [PowerSyncDatabase.activeSubscriptions] can still be used to + /// resolve known subscriptions locally). + Iterable? get activeSubscriptions { + return _internalSubscriptions?.map((subscription) { + return SyncStreamStatus._(subscription, downloadProgress); + }); + } + /// Get the current [downloadError] or [uploadError]. Object? get anyError { return downloadError ?? uploadError; @@ -153,6 +168,21 @@ final class SyncStatus { ); } + /// If the [stream] appears in [activeSubscriptions], returns the current + /// status for that stream. + SyncStreamStatus? statusFor(SyncStreamDescription stream) { + final raw = _internalSubscriptions?.firstWhereOrNull( + (e) => + e.name == stream.name && + _mapEquality.equals(e.parameters, stream.parameters), + ); + + if (raw == null) { + return null; + } + return SyncStreamStatus._(raw, downloadProgress); + } + @override int get hashCode { return Object.hash( @@ -163,8 +193,9 @@ final class SyncStatus { uploadError, downloadError, lastSyncedAt, - _statusEquality.hash(priorityStatusEntries), + _listEquality.hash(priorityStatusEntries), downloadProgress, + _listEquality.hash(_internalSubscriptions), ); } @@ -173,21 +204,20 @@ final class SyncStatus { return "SyncStatus"; } - // This should be a ListEquality, but that appears to - // cause weird type errors with DDC (but only after hot reloads?!) - static const _statusEquality = ListEquality(); + static const _listEquality = ListEquality(); + static const _mapEquality = MapEquality(); } final class SyncStreamStatus { - final SyncSubscriptionDefinition subscription; - final BucketPriority priority; + final ProgressWithOperations? progress; + final CoreActiveStreamSubscription _internal; - final bool isDefault; - final ProgressWithOperations progress; + SyncSubscriptionDefinition get subscription => _internal; + BucketPriority get priority => _internal.priority; + bool get isDefault => _internal.isDefault; - @internal - SyncStreamStatus( - this.subscription, this.priority, this.isDefault, this.progress); + SyncStreamStatus._(this._internal, SyncDownloadProgress? progress) + : progress = progress?._internal._forStream(_internal); } /// The priority of a PowerSync bucket. @@ -304,13 +334,23 @@ final class InternalSyncDownloadProgress extends ProgressWithOperations { /// Sums the total target and completed operations for all buckets up until /// the given [priority] (inclusive). ProgressWithOperations untilPriority(BucketPriority priority) { - final (total, downloaded) = - buckets.values.where((e) => e.priority >= priority).fold( + final (total, downloaded) = buckets.values + .where((e) => e.priority >= priority) + .fold((0, 0), _addProgress); + + return ProgressWithOperations._(total, downloaded); + } + + ProgressWithOperations _forStream(CoreActiveStreamSubscription subscription) { + final (total, downloaded) = subscription.associatedBuckets.fold( (0, 0), - (prev, entry) { - final downloaded = entry.sinceLast; - final total = entry.targetCount - entry.atLast; - return (prev.$1 + total, prev.$2 + downloaded); + (prev, bucket) { + final foundProgress = buckets[bucket]; + if (foundProgress == null) { + return prev; + } + + return _addProgress(prev, foundProgress); }, ); @@ -356,6 +396,12 @@ final class InternalSyncDownloadProgress extends ProgressWithOperations { } static const _mapEquality = MapEquality(); + + (int, int) _addProgress((int, int) prev, BucketProgress entry) { + final downloaded = entry.sinceLast; + final total = entry.targetCount - entry.atLast; + return (prev.$1 + total, prev.$2 + downloaded); + } } /// Information about a progressing download. From b8c66221c5696985c022ec38d119a8f19e1a370e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 14 Jul 2025 17:48:09 +0200 Subject: [PATCH 03/33] Refactor connection logic into separate class --- .../lib/src/database/powersync_db_mixin.dart | 145 ++-------------- .../lib/src/sync/connection_manager.dart | 162 ++++++++++++++++++ 2 files changed, 174 insertions(+), 133 deletions(-) create mode 100644 packages/powersync_core/lib/src/sync/connection_manager.dart diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index e6134293..43ed352b 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -14,6 +14,7 @@ import 'package:powersync_core/src/powersync_update_notification.dart'; import 'package:powersync_core/src/schema.dart'; import 'package:powersync_core/src/schema_logic.dart'; import 'package:powersync_core/src/schema_logic.dart' as schema_logic; +import 'package:powersync_core/src/sync/connection_manager.dart'; import 'package:powersync_core/src/sync/options.dart'; import 'package:powersync_core/src/sync/sync_status.dart'; @@ -44,16 +45,13 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { @Deprecated("This field is unused, pass params to connect() instead") Map? clientParams; + late final ConnectionManager _connections; + /// Current connection status. - SyncStatus currentStatus = - const SyncStatus(connected: false, lastSyncedAt: null); + SyncStatus get currentStatus => _connections.currentStatus; /// Use this stream to subscribe to connection status updates. - late final Stream statusStream; - - @protected - StreamController statusStreamController = - StreamController.broadcast(); + Stream get statusStream => _connections.statusStream; late final ActiveDatabaseGroup _activeGroup; @@ -83,15 +81,6 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { @protected Future get isInitialized; - /// The abort controller for the current sync iteration. - /// - /// null when disconnected, present when connecting or connected. - /// - /// The controller must only be accessed from within a critical section of the - /// sync mutex. - @protected - AbortController? _abortActiveSync; - @protected Future baseInit() async { String identifier = 'memory'; @@ -109,8 +98,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { 'instantiation logic if this is not intentional', ); } - - statusStream = statusStreamController.stream; + _connections = ConnectionManager(this); updates = powerSyncUpdateNotifications(database.updates); await database.initialize(); @@ -217,33 +205,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { @protected @visibleForTesting void setStatus(SyncStatus status) { - if (status != currentStatus) { - final newStatus = SyncStatus( - connected: status.connected, - downloading: status.downloading, - uploading: status.uploading, - connecting: status.connecting, - uploadError: status.uploadError, - downloadError: status.downloadError, - priorityStatusEntries: status.priorityStatusEntries, - downloadProgress: status.downloadProgress, - // Note that currently the streaming sync implementation will never set - // hasSynced. lastSyncedAt implies that syncing has completed at some - // point (hasSynced = true). - // The previous values of hasSynced should be preserved here. - lastSyncedAt: status.lastSyncedAt ?? currentStatus.lastSyncedAt, - hasSynced: status.lastSyncedAt != null - ? true - : status.hasSynced ?? currentStatus.hasSynced, - ); - - // If the absence of hasSynced was the only difference, the new states - // would be equal and don't require an event. So, check again. - if (newStatus != currentStatus) { - currentStatus = newStatus; - statusStreamController.add(currentStatus); - } - } + _connections.manuallyChangeSyncStatus(status); } @override @@ -270,7 +232,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { // If there are paused subscriptionso n the status stream, don't delay // closing the database because of that. - unawaited(statusStreamController.close()); + _connections.close(); await _activeGroup.close(); } } @@ -304,67 +266,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { params: params, ); - if (schema.rawTables.isNotEmpty && - resolvedOptions.source.syncImplementation != - SyncClientImplementation.rust) { - throw UnsupportedError( - 'Raw tables are only supported by the Rust client.'); - } - - // ignore: deprecated_member_use_from_same_package - clientParams = params; - var thisConnectAborter = AbortController(); - final zone = Zone.current; - - late void Function() retryHandler; - - Future connectWithSyncLock() async { - // Ensure there has not been a subsequent connect() call installing a new - // sync client. - assert(identical(_abortActiveSync, thisConnectAborter)); - assert(!thisConnectAborter.aborted); - - await connectInternal( - connector: connector, - options: resolvedOptions, - abort: thisConnectAborter, - // Run follow-up async tasks in the parent zone, a new one is introduced - // while we hold the lock (and async tasks won't hold the sync lock). - asyncWorkZone: zone, - ); - - thisConnectAborter.onCompletion.whenComplete(retryHandler); - } - - // If the sync encounters a failure without being aborted, retry - retryHandler = Zone.current.bindCallback(() async { - _activeGroup.syncConnectMutex.lock(() async { - // Is this still supposed to be active? (abort is only called within - // mutex) - if (!thisConnectAborter.aborted) { - // We only change _abortActiveSync after disconnecting, which resets - // the abort controller. - assert(identical(_abortActiveSync, thisConnectAborter)); - - // We need a new abort controller for this attempt - _abortActiveSync = thisConnectAborter = AbortController(); - - logger.warning('Sync client failed, retrying...'); - await connectWithSyncLock(); - } - }); - }); - - await _activeGroup.syncConnectMutex.lock(() async { - // Disconnect a previous sync client, if one is active. - await _abortCurrentSync(); - assert(_abortActiveSync == null); - - // Install the abort controller for this particular connect call, allowing - // it to be disconnected. - _abortActiveSync = thisConnectAborter; - await connectWithSyncLock(); - }); + await _connections.connect(connector: connector, options: resolvedOptions); } /// Internal method to establish a sync client connection. @@ -386,27 +288,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// /// Use [connect] to connect again. Future disconnect() async { - // Also wrap this in the sync mutex to ensure there's no race between us - // connecting and disconnecting. - await _activeGroup.syncConnectMutex.lock(_abortCurrentSync); - - setStatus( - SyncStatus(connected: false, lastSyncedAt: currentStatus.lastSyncedAt)); - } - - Future _abortCurrentSync() async { - if (_abortActiveSync case final disconnector?) { - /// Checking `disconnecter.aborted` prevents race conditions - /// where multiple calls to `disconnect` can attempt to abort - /// the controller more than once before it has finished aborting. - if (disconnector.aborted == false) { - await disconnector.abort(); - _abortActiveSync = null; - } else { - /// Wait for the abort to complete. Continue updating the sync status after completed - await disconnector.onCompletion; - } - } + await _connections.disconnect(); } /// Disconnect and clear the database. @@ -424,8 +306,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { await tx.execute('select powersync_clear(?)', [clearLocal ? 1 : 0]); }); // The data has been deleted - reset these - currentStatus = SyncStatus(lastSyncedAt: null, hasSynced: false); - statusStreamController.add(currentStatus); + setStatus(SyncStatus(lastSyncedAt: null, hasSynced: false)); } @Deprecated('Use [disconnectAndClear] instead.') @@ -447,9 +328,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { schema.validate(); await _activeGroup.syncConnectMutex.lock(() async { - if (_abortActiveSync != null) { - throw AssertionError('Cannot update schema while connected'); - } + _connections.checkNotConnected(); this.schema = schema; await database.writeLock((tx) => schema_logic.updateSchema(tx, schema)); diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart new file mode 100644 index 00000000..148c8447 --- /dev/null +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -0,0 +1,162 @@ +import 'dart:async'; + +import 'package:meta/meta.dart'; +import 'package:powersync_core/powersync_core.dart'; +import 'package:powersync_core/src/abort_controller.dart'; +import 'package:powersync_core/src/database/active_instances.dart'; +import 'package:powersync_core/src/database/powersync_db_mixin.dart'; +import 'package:powersync_core/src/sync/options.dart'; + +@internal +final class ConnectionManager { + final PowerSyncDatabaseMixin db; + final StreamController _statusController = StreamController(); + + SyncStatus _currentStatus = + const SyncStatus(connected: false, lastSyncedAt: null); + + SyncStatus get currentStatus => _currentStatus; + Stream get statusStream => _statusController.stream; + + final ActiveDatabaseGroup _activeGroup; + + ConnectionManager(this.db) : _activeGroup = db.group; + + /// The abort controller for the current sync iteration. + /// + /// null when disconnected, present when connecting or connected. + /// + /// The controller must only be accessed from within a critical section of the + /// sync mutex. + @protected + AbortController? _abortActiveSync; + + void checkNotConnected() { + if (_abortActiveSync != null) { + throw StateError('Cannot update schema while connected'); + } + } + + Future _abortCurrentSync() async { + if (_abortActiveSync case final disconnector?) { + /// Checking `disconnecter.aborted` prevents race conditions + /// where multiple calls to `disconnect` can attempt to abort + /// the controller more than once before it has finished aborting. + if (disconnector.aborted == false) { + await disconnector.abort(); + _abortActiveSync = null; + } else { + /// Wait for the abort to complete. Continue updating the sync status after completed + await disconnector.onCompletion; + } + } + } + + Future disconnect() async { + // Also wrap this in the sync mutex to ensure there's no race between us + // connecting and disconnecting. + await _activeGroup.syncConnectMutex.lock(_abortCurrentSync); + + manuallyChangeSyncStatus( + SyncStatus(connected: false, lastSyncedAt: currentStatus.lastSyncedAt)); + } + + Future connect({ + required PowerSyncBackendConnector connector, + required ResolvedSyncOptions options, + }) async { + if (db.schema.rawTables.isNotEmpty && + options.source.syncImplementation != SyncClientImplementation.rust) { + throw UnsupportedError( + 'Raw tables are only supported by the Rust client.'); + } + + var thisConnectAborter = AbortController(); + final zone = Zone.current; + + late void Function() retryHandler; + + Future connectWithSyncLock() async { + // Ensure there has not been a subsequent connect() call installing a new + // sync client. + assert(identical(_abortActiveSync, thisConnectAborter)); + assert(!thisConnectAborter.aborted); + + // ignore: invalid_use_of_protected_member + await db.connectInternal( + connector: connector, + options: options, + abort: thisConnectAborter, + // Run follow-up async tasks in the parent zone, a new one is introduced + // while we hold the lock (and async tasks won't hold the sync lock). + asyncWorkZone: zone, + ); + + thisConnectAborter.onCompletion.whenComplete(retryHandler); + } + + // If the sync encounters a failure without being aborted, retry + retryHandler = Zone.current.bindCallback(() async { + _activeGroup.syncConnectMutex.lock(() async { + // Is this still supposed to be active? (abort is only called within + // mutex) + if (!thisConnectAborter.aborted) { + // We only change _abortActiveSync after disconnecting, which resets + // the abort controller. + assert(identical(_abortActiveSync, thisConnectAborter)); + + // We need a new abort controller for this attempt + _abortActiveSync = thisConnectAborter = AbortController(); + + db.logger.warning('Sync client failed, retrying...'); + await connectWithSyncLock(); + } + }); + }); + + await _activeGroup.syncConnectMutex.lock(() async { + // Disconnect a previous sync client, if one is active. + await _abortCurrentSync(); + assert(_abortActiveSync == null); + + // Install the abort controller for this particular connect call, allowing + // it to be disconnected. + _abortActiveSync = thisConnectAborter; + await connectWithSyncLock(); + }); + } + + void manuallyChangeSyncStatus(SyncStatus status) { + if (status != currentStatus) { + final newStatus = SyncStatus( + connected: status.connected, + downloading: status.downloading, + uploading: status.uploading, + connecting: status.connecting, + uploadError: status.uploadError, + downloadError: status.downloadError, + priorityStatusEntries: status.priorityStatusEntries, + downloadProgress: status.downloadProgress, + // Note that currently the streaming sync implementation will never set + // hasSynced. lastSyncedAt implies that syncing has completed at some + // point (hasSynced = true). + // The previous values of hasSynced should be preserved here. + lastSyncedAt: status.lastSyncedAt ?? currentStatus.lastSyncedAt, + hasSynced: status.lastSyncedAt != null + ? true + : status.hasSynced ?? currentStatus.hasSynced, + ); + + // If the absence of hasSynced was the only difference, the new states + // would be equal and don't require an event. So, check again. + if (newStatus != currentStatus) { + _currentStatus = newStatus; + _statusController.add(currentStatus); + } + } + } + + void close() { + _statusController.close(); + } +} From ae477b0235c785903c667aed1e093d95b156baee Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 14 Jul 2025 20:17:57 +0200 Subject: [PATCH 04/33] Initial API --- .../lib/src/database/powersync_db_mixin.dart | 13 +- .../lib/src/sync/connection_manager.dart | 196 +++++++++++++++++- .../powersync_core/lib/src/sync/stream.dart | 2 + 3 files changed, 196 insertions(+), 15 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 43ed352b..784a93bb 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -192,14 +192,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { } } - if (matches(currentStatus)) { - return; - } - await for (final result in statusStream) { - if (matches(result)) { - break; - } - } + return _connections.firstStatusMatching(matches); } @protected @@ -551,6 +544,10 @@ SELECT * FROM crud_entries; Future refreshSchema() async { await database.refreshSchema(); } + + SyncStream syncStream(String name, [Map? parameters]) { + return _connections.syncStream(name, parameters); + } } Stream powerSyncUpdateNotifications( diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 148c8447..e2e812c3 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'package:meta/meta.dart'; import 'package:powersync_core/powersync_core.dart'; @@ -6,31 +7,33 @@ import 'package:powersync_core/src/abort_controller.dart'; import 'package:powersync_core/src/database/active_instances.dart'; import 'package:powersync_core/src/database/powersync_db_mixin.dart'; import 'package:powersync_core/src/sync/options.dart'; +import 'package:powersync_core/src/sync/stream.dart'; @internal final class ConnectionManager { final PowerSyncDatabaseMixin db; - final StreamController _statusController = StreamController(); + final ActiveDatabaseGroup _activeGroup; + final StreamController _statusController = StreamController(); SyncStatus _currentStatus = const SyncStatus(connected: false, lastSyncedAt: null); SyncStatus get currentStatus => _currentStatus; Stream get statusStream => _statusController.stream; - final ActiveDatabaseGroup _activeGroup; - - ConnectionManager(this.db) : _activeGroup = db.group; - /// The abort controller for the current sync iteration. /// /// null when disconnected, present when connecting or connected. /// /// The controller must only be accessed from within a critical section of the /// sync mutex. - @protected AbortController? _abortActiveSync; + /// Only to be called in the sync mutex. + Future Function()? _connectWithLastOptions; + + ConnectionManager(this.db) : _activeGroup = db.group; + void checkNotConnected() { if (_abortActiveSync != null) { throw StateError('Cannot update schema while connected'); @@ -55,12 +58,39 @@ final class ConnectionManager { Future disconnect() async { // Also wrap this in the sync mutex to ensure there's no race between us // connecting and disconnecting. - await _activeGroup.syncConnectMutex.lock(_abortCurrentSync); + await _activeGroup.syncConnectMutex.lock(() async { + await _abortCurrentSync(); + _connectWithLastOptions = null; + }); manuallyChangeSyncStatus( SyncStatus(connected: false, lastSyncedAt: currentStatus.lastSyncedAt)); } + Future firstStatusMatching(bool Function(SyncStatus) predicate) async { + if (predicate(currentStatus)) { + return; + } + await for (final result in statusStream) { + if (predicate(result)) { + break; + } + } + } + + Future reconnect() async { + // Also wrap this in the sync mutex to ensure there's no race between us + // connecting and disconnecting. + await _activeGroup.syncConnectMutex.lock(() async { + if (_connectWithLastOptions case final activeSync?) { + await _abortCurrentSync(); + assert(_abortActiveSync == null); + + await activeSync(); + } + }); + } + Future connect({ required PowerSyncBackendConnector connector, required ResolvedSyncOptions options, @@ -118,6 +148,7 @@ final class ConnectionManager { // Disconnect a previous sync client, if one is active. await _abortCurrentSync(); assert(_abortActiveSync == null); + _connectWithLastOptions = connectWithSyncLock; // Install the abort controller for this particular connect call, allowing // it to be disconnected. @@ -156,7 +187,158 @@ final class ConnectionManager { } } + Future _subscriptionsCommand(Object? command) async { + await db.writeTransaction((tx) { + return db.execute( + 'SELECT powersync_control(?, ?)', + ['subscriptions', json.encode(command)], + ); + }); + + await reconnect(); + } + + Future subscribe({ + required String stream, + required Object? parameters, + Duration? ttl, + BucketPriority? priority, + }) async { + await _subscriptionsCommand({ + 'subscribe': { + 'stream': stream, + 'params': parameters, + 'ttl': ttl?.inSeconds, + 'priority': priority, + }, + }); + } + + Future unsubscribe({ + required String stream, + required Object? parameters, + required bool immediate, + }) async { + await _subscriptionsCommand({ + 'unsubscribe': { + 'stream': stream, + 'params': parameters, + 'immediate': immediate, + }, + }); + } + + Future resolveCurrent( + String name, Map? parameters) async { + final row = await db.getOptional( + 'SELECT stream_name, active, is_default, local_priority, local_params, expires_at, last_synced_at FROM ps_stream_subscriptions WHERE stream_name = ? AND local_params = ?', + [name, json.encode(parameters)], + ); + + if (row == null) { + return null; + } + + return _SyncStreamSubscription( + this, + name: name, + parameters: + json.decode(row['local_params'] as String) as Map?, + active: row['active'] != 0, + expiresAt: switch (row['expires_at']) { + null => null, + final expiresAt as int => + DateTime.fromMicrosecondsSinceEpoch(expiresAt * 1000), + }, + hasSynced: row['has_synced'] != 0, + lastSyncedAt: switch (row['last_synced_at']) { + null => null, + final lastSyncedAt as int => + DateTime.fromMicrosecondsSinceEpoch(lastSyncedAt * 1000), + }, + ); + } + + SyncStream syncStream(String name, Map? parameters) { + return _SyncStreamImplementation(this, name, parameters); + } + void close() { _statusController.close(); } } + +final class _SyncStreamImplementation implements SyncStream { + @override + final String name; + + @override + final Map? parameters; + + final ConnectionManager _connections; + + _SyncStreamImplementation(this._connections, this.name, this.parameters); + + @override + Future get current { + return _connections.resolveCurrent(name, parameters); + } + + @override + Future subscribe( + {Duration? ttl, + BucketPriority? priority, + Map? parameters}) async { + await _connections.subscribe( + stream: name, + parameters: parameters, + ttl: ttl, + priority: priority, + ); + } +} + +final class _SyncStreamSubscription implements SyncStreamSubscription { + final ConnectionManager _connections; + + @override + final String name; + @override + final Map? parameters; + + @override + final bool active; + @override + final DateTime? expiresAt; + @override + final bool hasSynced; + @override + final DateTime? lastSyncedAt; + + _SyncStreamSubscription( + this._connections, { + required this.name, + required this.parameters, + required this.active, + required this.expiresAt, + required this.hasSynced, + required this.lastSyncedAt, + }); + + @override + Future unsubscribe({bool immediately = false}) async { + await _connections.unsubscribe( + stream: name, parameters: parameters, immediate: immediately); + } + + @override + Future waitForFirstSync() async { + if (hasSynced) { + return; + } + return _connections.firstStatusMatching((status) { + final currentProgress = status.statusFor(this); + return currentProgress?.subscription.hasSynced ?? false; + }); + } +} diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index 6eada49c..bfab929f 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -21,6 +21,8 @@ abstract interface class SyncStream extends SyncStreamDescription { BucketPriority? priority, Map? parameters, }); + + Future get current; } abstract interface class SyncStreamSubscription From 0e27592422264fccd98dc06b38751ccf9434e453 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 15 Jul 2025 17:30:17 +0200 Subject: [PATCH 05/33] Expose more information --- .../lib/src/sync/connection_manager.dart | 18 ++++-- .../powersync_core/lib/src/sync/stream.dart | 62 ++++++++++++++++++- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index e2e812c3..0b662a70 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -231,7 +231,7 @@ final class ConnectionManager { Future resolveCurrent( String name, Map? parameters) async { final row = await db.getOptional( - 'SELECT stream_name, active, is_default, local_priority, local_params, expires_at, last_synced_at FROM ps_stream_subscriptions WHERE stream_name = ? AND local_params = ?', + 'SELECT stream_name, active, is_default, local_priority, local_params, expires_at, last_synced_at, ttl FROM ps_stream_subscriptions WHERE stream_name = ? AND local_params = ?', [name, json.encode(parameters)], ); @@ -245,6 +245,8 @@ final class ConnectionManager { parameters: json.decode(row['local_params'] as String) as Map?, active: row['active'] != 0, + isDefault: row['is_default'] != 0, + hasExplicitSubscription: row['ttl'] != null, expiresAt: switch (row['expires_at']) { null => null, final expiresAt as int => @@ -285,10 +287,10 @@ final class _SyncStreamImplementation implements SyncStream { } @override - Future subscribe( - {Duration? ttl, - BucketPriority? priority, - Map? parameters}) async { + Future subscribe({ + Duration? ttl, + BucketPriority? priority, + }) async { await _connections.subscribe( stream: name, parameters: parameters, @@ -309,6 +311,10 @@ final class _SyncStreamSubscription implements SyncStreamSubscription { @override final bool active; @override + final bool isDefault; + @override + final bool hasExplicitSubscription; + @override final DateTime? expiresAt; @override final bool hasSynced; @@ -320,6 +326,8 @@ final class _SyncStreamSubscription implements SyncStreamSubscription { required this.name, required this.parameters, required this.active, + required this.isDefault, + required this.hasExplicitSubscription, required this.expiresAt, required this.hasSynced, required this.lastSyncedAt, diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index bfab929f..7f279256 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -1,33 +1,88 @@ import 'package:meta/meta.dart'; import 'sync_status.dart'; +import '../database/powersync_database.dart'; +/// A description of a sync stream, consisting of its [name] and the +/// [parameters] used when subscribing. abstract interface class SyncStreamDescription { + /// The name of the stream as it appears in the stream definition for the + /// PowerSync service. String get name; + + /// The parameters used to subscribe to the stream, if any. + /// + /// The same stream can be subscribed to multiple times with different + /// parameters. Map? get parameters; } +/// Information about a subscribed sync stream. +/// +/// This includes the [SyncStreamDescription] along with information about the +/// current sync status. abstract interface class SyncSubscriptionDefinition extends SyncStreamDescription { + /// Whether this stream is active, meaning that the subscription has been + /// acknownledged by the sync serivce. bool get active; + + /// Whether this stream subscription is included by default, regardless of + /// whether the stream has explicitly been subscribed to or not. + /// + /// It's possible for both [isDefault] and [hasExplicitSubscription] to be + /// true at the same time - this happens when a default stream was subscribed + /// explicitly. + bool get isDefault; + + /// Whether this stream has been subscribed to explicitly. + /// + /// It's possible for both [isDefault] and [hasExplicitSubscription] to be + /// true at the same time - this happens when a default stream was subscribed + /// explicitly. + bool get hasExplicitSubscription; + + /// For sync streams that have a time-to-live, the current time at which the + /// stream would expire if not subscribed to again. DateTime? get expiresAt; + + /// Whether this stream subscription has been synced at least once. bool get hasSynced; + + /// If [hasSynced] is true, the last time data from this stream has been + /// synced. DateTime? get lastSyncedAt; } +/// A handle to a [SyncStreamDescription] that allows subscribing to the stream. +/// +/// To obtain an instance of [SyncStream], call [PowerSyncDatabase.syncStream]. abstract interface class SyncStream extends SyncStreamDescription { + /// Adds a subscription to this stream, requesting it to be included when + /// connecting to the sync service. + /// + /// The [priority] can be used to override the priority of this stream. Future subscribe({ Duration? ttl, BucketPriority? priority, - Map? parameters, }); + /// Resolves the current subscription state of this stream. Future get current; } +/// A [SyncStream] that has been subscribed to. abstract interface class SyncStreamSubscription implements SyncStreamDescription, SyncSubscriptionDefinition { + /// A variant of [PowerSyncDatabase.waitForFirstSync] that is specific to + /// this stream subscription. Future waitForFirstSync(); + + /// Removes this stream subscription from the database, if it has been + /// subscribed to explicitly. + /// + /// The subscription may still be included for a while, until the client + /// reconnects and receives new snapshots from the sync service. Future unsubscribe({bool immediately = false}); } @@ -42,8 +97,11 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { final List associatedBuckets; @override final bool active; + @override final bool isDefault; @override + final bool hasExplicitSubscription; + @override final DateTime? expiresAt; @override final DateTime? lastSyncedAt; @@ -58,6 +116,7 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { required this.associatedBuckets, required this.active, required this.isDefault, + required this.hasExplicitSubscription, required this.expiresAt, required this.lastSyncedAt, }); @@ -70,6 +129,7 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { associatedBuckets: (json['associated_buckets'] as List).cast(), active: json['active'] as bool, isDefault: json['is_default'] as bool, + hasExplicitSubscription: json['has_explicit_subscription'] as bool, expiresAt: switch (json['expires_at']) { null => null, final timestamp as int => From 19edf167707fd79fad9d0ab2946bb5cc1178551a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 11 Aug 2025 16:04:06 +0200 Subject: [PATCH 06/33] Remove immediate unsubscribe options --- .../powersync_core/lib/src/sync/connection_manager.dart | 7 ++----- packages/powersync_core/lib/src/sync/stream.dart | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 0b662a70..06e31562 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -217,13 +217,11 @@ final class ConnectionManager { Future unsubscribe({ required String stream, required Object? parameters, - required bool immediate, }) async { await _subscriptionsCommand({ 'unsubscribe': { 'stream': stream, 'params': parameters, - 'immediate': immediate, }, }); } @@ -334,9 +332,8 @@ final class _SyncStreamSubscription implements SyncStreamSubscription { }); @override - Future unsubscribe({bool immediately = false}) async { - await _connections.unsubscribe( - stream: name, parameters: parameters, immediate: immediately); + Future unsubscribe() async { + await _connections.unsubscribe(stream: name, parameters: parameters); } @override diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index 7f279256..840d0fa3 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -83,7 +83,7 @@ abstract interface class SyncStreamSubscription /// /// The subscription may still be included for a while, until the client /// reconnects and receives new snapshots from the sync service. - Future unsubscribe({bool immediately = false}); + Future unsubscribe(); } /// An `ActiveStreamSubscription` as part of the sync status in Rust. From de4050b5cd06386c4a26966fe4c3c72dda6805c9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 12 Aug 2025 12:50:51 +0200 Subject: [PATCH 07/33] Register stream subscriptions --- .../native/native_powersync_database.dart | 2 + .../powersync_database_impl_stub.dart | 3 + .../lib/src/database/powersync_db_mixin.dart | 5 +- .../database/web/web_powersync_database.dart | 2 + .../lib/src/sync/connection_manager.dart | 205 ++++++++++-------- .../powersync_core/lib/src/sync/options.dart | 5 + .../powersync_core/lib/src/sync/stream.dart | 13 +- .../lib/src/sync/streaming_sync.dart | 8 + .../lib/src/sync/sync_status.dart | 31 +-- 9 files changed, 157 insertions(+), 117 deletions(-) diff --git a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart index 3ebb95f9..427bc9d9 100644 --- a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart @@ -133,6 +133,8 @@ class PowerSyncDatabaseImpl Future connectInternal({ required PowerSyncBackendConnector connector, required ResolvedSyncOptions options, + required List initiallyActiveStreams, + required Stream> activeStreams, required AbortController abort, required Zone asyncWorkZone, }) async { diff --git a/packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart b/packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart index a4f0b419..ae891cb7 100644 --- a/packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart +++ b/packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart @@ -7,6 +7,7 @@ import 'package:powersync_core/src/abort_controller.dart'; import 'package:powersync_core/src/database/powersync_db_mixin.dart'; import 'package:powersync_core/src/open_factory/abstract_powersync_open_factory.dart'; import '../sync/options.dart'; +import '../sync/streaming_sync.dart'; import 'powersync_database.dart'; import '../connector.dart'; @@ -115,6 +116,8 @@ class PowerSyncDatabaseImpl Future connectInternal({ required PowerSyncBackendConnector connector, required AbortController abort, + required List initiallyActiveStreams, + required Stream> activeStreams, required Zone asyncWorkZone, required ResolvedSyncOptions options, }) { diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 784a93bb..a46d76f9 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -19,6 +19,7 @@ import 'package:powersync_core/src/sync/options.dart'; import 'package:powersync_core/src/sync/sync_status.dart'; import '../sync/stream.dart'; +import '../sync/streaming_sync.dart'; mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Schema used for the local database. @@ -131,7 +132,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { return isInitialized; } - Future> get activeSubscriptions { + Future> get subscribedStreams { throw UnimplementedError(); } @@ -273,6 +274,8 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { Future connectInternal({ required PowerSyncBackendConnector connector, required ResolvedSyncOptions options, + required List initiallyActiveStreams, + required Stream> activeStreams, required AbortController abort, required Zone asyncWorkZone, }); diff --git a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart index 4af2821e..6b4f8292 100644 --- a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart @@ -128,6 +128,8 @@ class PowerSyncDatabaseImpl Future connectInternal({ required PowerSyncBackendConnector connector, required AbortController abort, + required List initiallyActiveStreams, + required Stream> activeStreams, required Zone asyncWorkZone, required ResolvedSyncOptions options, }) async { diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 06e31562..06d189a4 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -9,12 +9,28 @@ import 'package:powersync_core/src/database/powersync_db_mixin.dart'; import 'package:powersync_core/src/sync/options.dart'; import 'package:powersync_core/src/sync/stream.dart'; +import 'streaming_sync.dart'; + +/// A (stream name, JSON parameters) pair that uniquely identifies a stream +/// instantiation to subscribe to. +typedef _RawStreamKey = (String, String); + @internal final class ConnectionManager { final PowerSyncDatabaseMixin db; final ActiveDatabaseGroup _activeGroup; + /// All streams (with parameters) for which a subscription has been requested + /// explicitly. + final Map<_RawStreamKey, _ActiveSubscription> _locallyActiveSubscriptions = + {}; + final StreamController _statusController = StreamController(); + + /// Fires when an entry is added or removed from [_locallyActiveSubscriptions] + /// while we're connected. + StreamController? _subscriptionsChanged; + SyncStatus _currentStatus = const SyncStatus(connected: false, lastSyncedAt: null); @@ -29,9 +45,6 @@ final class ConnectionManager { /// sync mutex. AbortController? _abortActiveSync; - /// Only to be called in the sync mutex. - Future Function()? _connectWithLastOptions; - ConnectionManager(this.db) : _activeGroup = db.group; void checkNotConnected() { @@ -60,7 +73,8 @@ final class ConnectionManager { // connecting and disconnecting. await _activeGroup.syncConnectMutex.lock(() async { await _abortCurrentSync(); - _connectWithLastOptions = null; + _subscriptionsChanged?.close(); + _subscriptionsChanged = null; }); manuallyChangeSyncStatus( @@ -78,18 +92,10 @@ final class ConnectionManager { } } - Future reconnect() async { - // Also wrap this in the sync mutex to ensure there's no race between us - // connecting and disconnecting. - await _activeGroup.syncConnectMutex.lock(() async { - if (_connectWithLastOptions case final activeSync?) { - await _abortCurrentSync(); - assert(_abortActiveSync == null); - - await activeSync(); - } - }); - } + List get _subscribedStreams => [ + for (final active in _locallyActiveSubscriptions.values) + (name: active.name, parameters: active.encodedParameters) + ]; Future connect({ required PowerSyncBackendConnector connector, @@ -106,6 +112,8 @@ final class ConnectionManager { late void Function() retryHandler; + final subscriptionsChanged = StreamController(); + Future connectWithSyncLock() async { // Ensure there has not been a subsequent connect() call installing a new // sync client. @@ -117,6 +125,10 @@ final class ConnectionManager { connector: connector, options: options, abort: thisConnectAborter, + initiallyActiveStreams: _subscribedStreams, + activeStreams: subscriptionsChanged.stream.map((_) { + return _subscribedStreams; + }), // Run follow-up async tasks in the parent zone, a new one is introduced // while we hold the lock (and async tasks won't hold the sync lock). asyncWorkZone: zone, @@ -148,7 +160,7 @@ final class ConnectionManager { // Disconnect a previous sync client, if one is active. await _abortCurrentSync(); assert(_abortActiveSync == null); - _connectWithLastOptions = connectWithSyncLock; + _subscriptionsChanged = subscriptionsChanged; // Install the abort controller for this particular connect call, allowing // it to be disconnected. @@ -187,6 +199,30 @@ final class ConnectionManager { } } + _SyncStreamSubscriptionHandle _referenceStreamSubscription( + String stream, Map? parameters) { + final key = (stream, json.encode(parameters)); + _ActiveSubscription active; + + if (_locallyActiveSubscriptions[key] case final current?) { + active = current; + } else { + active = _ActiveSubscription(this, + name: stream, parameters: parameters, encodedParameters: key.$2); + _locallyActiveSubscriptions[key] = active; + _subscriptionsChanged?.add(null); + } + + return _SyncStreamSubscriptionHandle(active); + } + + void _clearSubscription(_ActiveSubscription subscription) { + assert(subscription.refcount == 0); + _locallyActiveSubscriptions + .remove((subscription.name, subscription.encodedParameters)); + _subscriptionsChanged?.add(null); + } + Future _subscriptionsCommand(Object? command) async { await db.writeTransaction((tx) { return db.execute( @@ -194,71 +230,39 @@ final class ConnectionManager { ['subscriptions', json.encode(command)], ); }); - - await reconnect(); + _subscriptionsChanged?.add(null); } Future subscribe({ required String stream, - required Object? parameters, + required Map? parameters, Duration? ttl, - BucketPriority? priority, + StreamPriority? priority, }) async { await _subscriptionsCommand({ 'subscribe': { - 'stream': stream, - 'params': parameters, + 'stream': { + 'name': stream, + 'params': parameters, + }, 'ttl': ttl?.inSeconds, 'priority': priority, }, }); } - Future unsubscribe({ + Future unsubscribeAll({ required String stream, required Object? parameters, }) async { await _subscriptionsCommand({ 'unsubscribe': { - 'stream': stream, + 'name': stream, 'params': parameters, }, }); } - Future resolveCurrent( - String name, Map? parameters) async { - final row = await db.getOptional( - 'SELECT stream_name, active, is_default, local_priority, local_params, expires_at, last_synced_at, ttl FROM ps_stream_subscriptions WHERE stream_name = ? AND local_params = ?', - [name, json.encode(parameters)], - ); - - if (row == null) { - return null; - } - - return _SyncStreamSubscription( - this, - name: name, - parameters: - json.decode(row['local_params'] as String) as Map?, - active: row['active'] != 0, - isDefault: row['is_default'] != 0, - hasExplicitSubscription: row['ttl'] != null, - expiresAt: switch (row['expires_at']) { - null => null, - final expiresAt as int => - DateTime.fromMicrosecondsSinceEpoch(expiresAt * 1000), - }, - hasSynced: row['has_synced'] != 0, - lastSyncedAt: switch (row['last_synced_at']) { - null => null, - final lastSyncedAt as int => - DateTime.fromMicrosecondsSinceEpoch(lastSyncedAt * 1000), - }, - ); - } - SyncStream syncStream(String name, Map? parameters) { return _SyncStreamImplementation(this, name, parameters); } @@ -280,14 +284,9 @@ final class _SyncStreamImplementation implements SyncStream { _SyncStreamImplementation(this._connections, this.name, this.parameters); @override - Future get current { - return _connections.resolveCurrent(name, parameters); - } - - @override - Future subscribe({ + Future subscribe({ Duration? ttl, - BucketPriority? priority, + StreamPriority? priority, }) async { await _connections.subscribe( stream: name, @@ -295,55 +294,71 @@ final class _SyncStreamImplementation implements SyncStream { ttl: ttl, priority: priority, ); + + return _connections._referenceStreamSubscription(name, parameters); + } + + @override + Future unsubscribeAll() async { + await _connections.unsubscribeAll(stream: name, parameters: parameters); } } -final class _SyncStreamSubscription implements SyncStreamSubscription { - final ConnectionManager _connections; +final class _ActiveSubscription { + final ConnectionManager connections; + var refcount = 0; - @override final String name; - @override + final String encodedParameters; final Map? parameters; - @override - final bool active; - @override - final bool isDefault; - @override - final bool hasExplicitSubscription; - @override - final DateTime? expiresAt; - @override - final bool hasSynced; - @override - final DateTime? lastSyncedAt; - - _SyncStreamSubscription( - this._connections, { + _ActiveSubscription( + this.connections, { required this.name, + required this.encodedParameters, required this.parameters, - required this.active, - required this.isDefault, - required this.hasExplicitSubscription, - required this.expiresAt, - required this.hasSynced, - required this.lastSyncedAt, }); + void decrementRefCount() { + refcount--; + if (refcount == 0) { + connections._clearSubscription(this); + } + } +} + +final class _SyncStreamSubscriptionHandle implements SyncStreamSubscription { + final _ActiveSubscription _source; + + _SyncStreamSubscriptionHandle(this._source) { + _source.refcount++; + + // This is not unreliable, but can help decrementing refcounts on the inner + // subscription when this handle is deallocated without [unsubscribe] being + // called. + _finalizer.attach(this, _source, detach: this); + } + + @override + String get name => _source.name; + + @override + Map? get parameters => _source.parameters; + @override Future unsubscribe() async { - await _connections.unsubscribe(stream: name, parameters: parameters); + _finalizer.detach(this); + _source.decrementRefCount(); } @override Future waitForFirstSync() async { - if (hasSynced) { - return; - } - return _connections.firstStatusMatching((status) { + return _source.connections.firstStatusMatching((status) { final currentProgress = status.statusFor(this); return currentProgress?.subscription.hasSynced ?? false; }); } + + static final Finalizer<_ActiveSubscription> _finalizer = + Finalizer((sub) => sub.decrementRefCount()); } diff --git a/packages/powersync_core/lib/src/sync/options.dart b/packages/powersync_core/lib/src/sync/options.dart index 6ae94b25..f67dd144 100644 --- a/packages/powersync_core/lib/src/sync/options.dart +++ b/packages/powersync_core/lib/src/sync/options.dart @@ -27,11 +27,14 @@ final class SyncOptions { /// The [SyncClientImplementation] to use. final SyncClientImplementation syncImplementation; + final bool? includeDefaultStreams; + const SyncOptions({ this.crudThrottleTime, this.retryDelay, this.params, this.syncImplementation = SyncClientImplementation.defaultClient, + this.includeDefaultStreams, }); SyncOptions _copyWith({ @@ -96,6 +99,8 @@ extension type ResolvedSyncOptions(SyncOptions source) { Map get params => source.params ?? const {}; + bool get includeDefaultStreams => source.includeDefaultStreams ?? true; + (ResolvedSyncOptions, bool) applyFrom(SyncOptions other) { final newOptions = SyncOptions( crudThrottleTime: other.crudThrottleTime ?? crudThrottleTime, diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index 840d0fa3..a23e4816 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -62,18 +62,17 @@ abstract interface class SyncStream extends SyncStreamDescription { /// connecting to the sync service. /// /// The [priority] can be used to override the priority of this stream. - Future subscribe({ + Future subscribe({ Duration? ttl, - BucketPriority? priority, + StreamPriority? priority, }); - /// Resolves the current subscription state of this stream. - Future get current; + Future unsubscribeAll(); } /// A [SyncStream] that has been subscribed to. abstract interface class SyncStreamSubscription - implements SyncStreamDescription, SyncSubscriptionDefinition { + implements SyncStreamDescription { /// A variant of [PowerSyncDatabase.waitForFirstSync] that is specific to /// this stream subscription. Future waitForFirstSync(); @@ -93,7 +92,7 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { final String name; @override final Map? parameters; - final BucketPriority priority; + final StreamPriority priority; final List associatedBuckets; @override final bool active; @@ -125,7 +124,7 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { return CoreActiveStreamSubscription._( name: json['name'] as String, parameters: json['parameters'] as Map, - priority: BucketPriority(json['priority'] as int), + priority: StreamPriority(json['priority'] as int), associatedBuckets: (json['associated_buckets'] as List).cast(), active: json['active'] as bool, isDefault: json['is_default'] as bool, diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index ad0886a1..89929759 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -21,6 +21,8 @@ import 'stream_utils.dart'; import 'sync_status.dart'; import 'protocol.dart'; +typedef SubscribedStream = ({String name, String parameters}); + abstract interface class StreamingSync { Stream get statusStream; @@ -36,6 +38,7 @@ class StreamingSyncImplementation implements StreamingSync { final BucketStorage adapter; final InternalConnector connector; final ResolvedSyncOptions options; + final List activeSubscriptions; final Logger logger; @@ -69,6 +72,7 @@ class StreamingSyncImplementation implements StreamingSync { required this.crudUpdateTriggerStream, required this.options, required http.Client client, + this.activeSubscriptions = const [], Mutex? syncMutex, Mutex? crudMutex, Logger? logger, @@ -589,6 +593,7 @@ typedef BucketDescription = ({ final class _ActiveRustStreamingIteration { final StreamingSyncImplementation sync; + var _isActive = true; var _hadSyncLine = false; @@ -604,6 +609,9 @@ final class _ActiveRustStreamingIteration { convert.json.encode({ 'parameters': sync.options.params, 'schema': convert.json.decode(sync.schemaJson), + 'include_defaults': sync.options.includeDefaultStreams, + 'active_streams': sync.activeSubscriptions + .map((s) => {'name': s.name, 'params': s.parameters}) }), ); assert(_completedStream.isCompleted, 'Should have started streaming'); diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index df616988..d46e3850 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -121,7 +121,7 @@ final class SyncStatus { /// /// This returns null when the sync stream is currently being opened and we /// don't have reliable information about all included streams yet (in that - /// state, [PowerSyncDatabase.activeSubscriptions] can still be used to + /// state, [PowerSyncDatabase.subscribedStreams] can still be used to /// resolve known subscriptions locally). Iterable? get activeSubscriptions { return _internalSubscriptions?.map((subscription) { @@ -213,39 +213,42 @@ final class SyncStreamStatus { final CoreActiveStreamSubscription _internal; SyncSubscriptionDefinition get subscription => _internal; - BucketPriority get priority => _internal.priority; + StreamPriority get priority => _internal.priority; bool get isDefault => _internal.isDefault; SyncStreamStatus._(this._internal, SyncDownloadProgress? progress) : progress = progress?._internal._forStream(_internal); } -/// The priority of a PowerSync bucket. -extension type const BucketPriority._(int priorityNumber) { +@Deprecated('Use StreamPriority instead') +typedef BucketPriority = StreamPriority; + +/// The priority of a PowerSync stream. +extension type const StreamPriority._(int priorityNumber) { static const _highest = 0; - factory BucketPriority(int i) { + factory StreamPriority(int i) { assert(i >= _highest); - return BucketPriority._(i); + return StreamPriority._(i); } - bool operator >(BucketPriority other) => comparator(this, other) > 0; - bool operator >=(BucketPriority other) => comparator(this, other) >= 0; - bool operator <(BucketPriority other) => comparator(this, other) < 0; - bool operator <=(BucketPriority other) => comparator(this, other) <= 0; + bool operator >(StreamPriority other) => comparator(this, other) > 0; + bool operator >=(StreamPriority other) => comparator(this, other) >= 0; + bool operator <(StreamPriority other) => comparator(this, other) < 0; + bool operator <=(StreamPriority other) => comparator(this, other) <= 0; - /// A [Comparator] instance suitable for comparing [BucketPriority] values. - static int comparator(BucketPriority a, BucketPriority b) => + /// A [Comparator] instance suitable for comparing [StreamPriority] values. + static int comparator(StreamPriority a, StreamPriority b) => -a.priorityNumber.compareTo(b.priorityNumber); /// The priority used by PowerSync to indicate that a full sync was completed. - static const fullSyncPriority = BucketPriority._(2147483647); + static const fullSyncPriority = StreamPriority._(2147483647); } /// Partial information about the synchronization status for buckets within a /// priority. typedef SyncPriorityStatus = ({ - BucketPriority priority, + StreamPriority priority, DateTime? lastSyncedAt, bool? hasSynced, }); From c1ed1e659f0e5d33ae5a2743005773263b179ffe Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 12 Aug 2025 13:37:17 +0200 Subject: [PATCH 08/33] Native: Propagate stream subscriptions --- .../native/native_powersync_database.dart | 9 +++++++++ .../lib/src/sync/streaming_sync.dart | 16 ++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart index 427bc9d9..6c44ad51 100644 --- a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart @@ -142,6 +142,7 @@ class PowerSyncDatabaseImpl bool triedSpawningIsolate = false; StreamSubscription? crudUpdateSubscription; + StreamSubscription? activeStreamsSubscription; final receiveMessages = ReceivePort(); final receiveUnhandledErrors = ReceivePort(); final receiveExit = ReceivePort(); @@ -159,6 +160,7 @@ class PowerSyncDatabaseImpl // Cleanup crudUpdateSubscription?.cancel(); + activeStreamsSubscription?.cancel(); receiveMessages.close(); receiveUnhandledErrors.close(); receiveExit.close(); @@ -200,6 +202,10 @@ class PowerSyncDatabaseImpl crudUpdateSubscription = crudStream.listen((event) { port.send(['update']); }); + + activeStreamsSubscription = activeStreams.listen((streams) { + port.send(['changed_subscriptions', streams]); + }); } else if (action == 'uploadCrud') { await (data[1] as PortCompleter).handle(() async { await connector.uploadData(this); @@ -368,6 +374,9 @@ Future _syncIsolate(_PowerSyncDatabaseIsolateArgs args) async { } } else if (action == 'close') { await shutdown(); + } else if (action == 'changed_subscriptions') { + openedStreamingSync + ?.updateSubscriptions(action[1] as List); } } }); diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index 89929759..0bd4577a 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -38,7 +38,7 @@ class StreamingSyncImplementation implements StreamingSync { final BucketStorage adapter; final InternalConnector connector; final ResolvedSyncOptions options; - final List activeSubscriptions; + List _activeSubscriptions; final Logger logger; @@ -72,7 +72,7 @@ class StreamingSyncImplementation implements StreamingSync { required this.crudUpdateTriggerStream, required this.options, required http.Client client, - this.activeSubscriptions = const [], + List activeSubscriptions = const [], Mutex? syncMutex, Mutex? crudMutex, Logger? logger, @@ -84,7 +84,8 @@ class StreamingSyncImplementation implements StreamingSync { syncMutex = syncMutex ?? Mutex(identifier: "sync-$identifier"), crudMutex = crudMutex ?? Mutex(identifier: "crud-$identifier"), _userAgentHeaders = userAgentHeaders(), - logger = logger ?? isolateLogger; + logger = logger ?? isolateLogger, + _activeSubscriptions = activeSubscriptions; Duration get _retryDelay => options.retryDelay; @@ -128,6 +129,13 @@ class StreamingSyncImplementation implements StreamingSync { return _abort?.aborted ?? false; } + void updateSubscriptions(List streams) { + _activeSubscriptions = streams; + if (_nonLineSyncEvents.hasListener) { + _nonLineSyncEvents.add(const AbortCurrentIteration()); + } + } + @override Future streamingSync() async { try { @@ -610,7 +618,7 @@ final class _ActiveRustStreamingIteration { 'parameters': sync.options.params, 'schema': convert.json.decode(sync.schemaJson), 'include_defaults': sync.options.includeDefaultStreams, - 'active_streams': sync.activeSubscriptions + 'active_streams': sync._activeSubscriptions .map((s) => {'name': s.name, 'params': s.parameters}) }), ); From a3fb07cc13b7fcb0bca248719836bf89f4855f39 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 12 Aug 2025 14:27:05 +0200 Subject: [PATCH 09/33] Track subscriptions across requests --- .../database/web/web_powersync_database.dart | 5 ++ .../lib/src/sync/streaming_sync.dart | 3 + .../lib/src/web/sync_controller.dart | 11 +++ .../lib/src/web/sync_worker.dart | 75 +++++++++++++++---- .../lib/src/web/sync_worker_protocol.dart | 54 ++++++++++++- 5 files changed, 134 insertions(+), 14 deletions(-) diff --git a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart index 6b4f8292..15a83c7d 100644 --- a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart @@ -143,6 +143,7 @@ class PowerSyncDatabaseImpl connector: connector, options: options.source, workerUri: Uri.base.resolve('/powersync_sync.worker.js'), + subscriptions: initiallyActiveStreams, ); } catch (e) { logger.warning( @@ -159,6 +160,7 @@ class PowerSyncDatabaseImpl crudUpdateTriggerStream: crudStream, options: options, client: BrowserClient(), + activeSubscriptions: initiallyActiveStreams, // Only allows 1 sync implementation to run at a time per database // This should be global (across tabs) when using Navigator locks. identifier: database.openFactory.path, @@ -170,7 +172,10 @@ class PowerSyncDatabaseImpl }); sync.streamingSync(); + final subscriptions = activeStreams.listen(sync.updateSubscriptions); + abort.onAbort.then((_) async { + subscriptions.cancel(); await sync.abort(); abort.completeAbort(); }).ignore(); diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index 0bd4577a..4512a858 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -30,6 +30,8 @@ abstract interface class StreamingSync { /// Close any active streams. Future abort(); + + void updateSubscriptions(List streams); } @internal @@ -129,6 +131,7 @@ class StreamingSyncImplementation implements StreamingSync { return _abort?.aborted ?? false; } + @override void updateSubscriptions(List streams) { _activeSubscriptions = streams; if (_nonLineSyncEvents.hasListener) { diff --git a/packages/powersync_core/lib/src/web/sync_controller.dart b/packages/powersync_core/lib/src/web/sync_controller.dart index 7f05cff3..b3f0ef18 100644 --- a/packages/powersync_core/lib/src/web/sync_controller.dart +++ b/packages/powersync_core/lib/src/web/sync_controller.dart @@ -15,6 +15,7 @@ class SyncWorkerHandle implements StreamingSync { final PowerSyncBackendConnector connector; final SyncOptions options; late final WorkerCommunicationChannel _channel; + List subscriptions; final StreamController _status = StreamController.broadcast(); @@ -24,6 +25,7 @@ class SyncWorkerHandle implements StreamingSync { required this.options, required MessagePort sendToWorker, required SharedWorker worker, + required this.subscriptions, }) { _channel = WorkerCommunicationChannel( port: sendToWorker, @@ -81,6 +83,7 @@ class SyncWorkerHandle implements StreamingSync { required PowerSyncBackendConnector connector, required Uri workerUri, required SyncOptions options, + required List subscriptions, }) async { final worker = SharedWorker(workerUri.toString().toJS); final handle = SyncWorkerHandle._( @@ -89,6 +92,7 @@ class SyncWorkerHandle implements StreamingSync { connector: connector, sendToWorker: worker.port, worker: worker, + subscriptions: subscriptions, ); // Make sure that the worker is working, or throw immediately. @@ -116,6 +120,13 @@ class SyncWorkerHandle implements StreamingSync { database.database.openFactory.path, ResolvedSyncOptions(options), database.schema, + subscriptions, ); } + + @override + void updateSubscriptions(List streams) { + subscriptions = streams; + _channel.updateSubscriptions(streams); + } } diff --git a/packages/powersync_core/lib/src/web/sync_worker.dart b/packages/powersync_core/lib/src/web/sync_worker.dart index ddc4eaf0..1aaa5cfa 100644 --- a/packages/powersync_core/lib/src/web/sync_worker.dart +++ b/packages/powersync_core/lib/src/web/sync_worker.dart @@ -8,6 +8,7 @@ import 'dart:convert'; import 'dart:js_interop'; import 'package:async/async.dart'; +import 'package:collection/collection.dart'; import 'package:http/browser_client.dart'; import 'package:logging/logging.dart'; import 'package:powersync_core/powersync_core.dart'; @@ -45,8 +46,12 @@ class _SyncWorker { }); } - _SyncRunner referenceSyncTask(String databaseIdentifier, SyncOptions options, - String schemaJson, _ConnectedClient client) { + _SyncRunner referenceSyncTask( + String databaseIdentifier, + SyncOptions options, + String schemaJson, + List subscriptions, + _ConnectedClient client) { return _requestedSyncTasks.putIfAbsent(databaseIdentifier, () { return _SyncRunner(databaseIdentifier); }) @@ -54,6 +59,7 @@ class _SyncWorker { client, options, schemaJson, + subscriptions, ); } } @@ -90,13 +96,20 @@ class _ConnectedClient { }, ); - _runner = _worker.referenceSyncTask(request.databaseName, - recoveredOptions, request.schemaJson, this); + _runner = _worker.referenceSyncTask( + request.databaseName, + recoveredOptions, + request.schemaJson, + request.subscriptions?.toDart ?? const [], + this, + ); return (JSObject(), null); case SyncWorkerMessageType.abortSynchronization: _runner?.disconnectClient(this); _runner = null; return (JSObject(), null); + case SyncWorkerMessageType.updateSubscriptions: + return (JSObject(), null); default: throw StateError('Unexpected message type $type'); } @@ -137,9 +150,10 @@ class _SyncRunner { final StreamGroup<_RunnerEvent> _group = StreamGroup(); final StreamController<_RunnerEvent> _mainEvents = StreamController(); - StreamingSync? sync; + StreamingSyncImplementation? sync; _ConnectedClient? databaseHost; - final connections = <_ConnectedClient>[]; + final connections = <_ConnectedClient, List>{}; + List currentStreams = []; _SyncRunner(this.identifier) { _group.add(_mainEvents.stream); @@ -152,8 +166,9 @@ class _SyncRunner { :final client, :final options, :final schemaJson, + :final subscriptions, ): - connections.add(client); + connections[client] = subscriptions; final (newOptions, reconnect) = this.options.applyFrom(options); this.options = newOptions; this.schemaJson = schemaJson; @@ -165,6 +180,8 @@ class _SyncRunner { sync?.abort(); sync = null; await _requestDatabase(client); + } else { + reindexSubscriptions(); } case _RemoveConnection(:final client): connections.remove(client); @@ -191,6 +208,12 @@ class _SyncRunner { } else { await _requestDatabase(newHost); } + case _ClientSubscriptionsChanged( + :final client, + :final subscriptions + ): + connections[client] = subscriptions; + reindexSubscriptions(); } } catch (e, s) { _logger.warning('Error handling $event', e, s); @@ -199,12 +222,22 @@ class _SyncRunner { }); } + /// Updates [currentStreams] to the union of values in [connections]. + void reindexSubscriptions() { + final before = currentStreams.toSet(); + final after = connections.values.flattenedToSet; + if (!const SetEquality().equals(before, after)) { + currentStreams = after.toList(); + sync?.updateSubscriptions(currentStreams); + } + } + /// Pings all current [connections], removing those that don't answer in 5s /// (as they are likely closed tabs as well). /// /// Returns the first client that responds (without waiting for others). Future<_ConnectedClient?> _collectActiveClients() async { - final candidates = connections.toList(); + final candidates = connections.keys.toList(); if (candidates.isEmpty) { return null; } @@ -269,6 +302,7 @@ class _SyncRunner { ); } + currentStreams = connections.values.flattenedToSet.toList(); sync = StreamingSyncImplementation( adapter: WebBucketStorage(database), schemaJson: client._runner!.schemaJson, @@ -283,10 +317,11 @@ class _SyncRunner { options: options, client: BrowserClient(), identifier: identifier, + activeSubscriptions: currentStreams, ); sync!.statusStream.listen((event) { _logger.fine('Broadcasting sync event: $event'); - for (final client in connections) { + for (final client in connections.keys) { client.channel.notify(SyncWorkerMessageType.notifySyncStatus, SerializedSyncStatus.from(event)); } @@ -294,9 +329,9 @@ class _SyncRunner { sync!.streamingSync(); } - void registerClient( - _ConnectedClient client, SyncOptions options, String schemaJson) { - _mainEvents.add(_AddConnection(client, options, schemaJson)); + void registerClient(_ConnectedClient client, SyncOptions options, + String schemaJson, List subscriptions) { + _mainEvents.add(_AddConnection(client, options, schemaJson, subscriptions)); } /// Remove a client, disconnecting if no clients remain.. @@ -308,6 +343,11 @@ class _SyncRunner { void disconnectClient(_ConnectedClient client) { _mainEvents.add(_DisconnectClient(client)); } + + void updateClientSubscriptions( + _ConnectedClient client, List subscriptions) { + _mainEvents.add(_ClientSubscriptionsChanged(client, subscriptions)); + } } sealed class _RunnerEvent {} @@ -316,8 +356,10 @@ final class _AddConnection implements _RunnerEvent { final _ConnectedClient client; final SyncOptions options; final String schemaJson; + final List subscriptions; - _AddConnection(this.client, this.options, this.schemaJson); + _AddConnection( + this.client, this.options, this.schemaJson, this.subscriptions); } final class _RemoveConnection implements _RunnerEvent { @@ -332,6 +374,13 @@ final class _DisconnectClient implements _RunnerEvent { _DisconnectClient(this.client); } +final class _ClientSubscriptionsChanged implements _RunnerEvent { + final _ConnectedClient client; + final List subscriptions; + + _ClientSubscriptionsChanged(this.client, this.subscriptions); +} + final class _ActiveDatabaseClosed implements _RunnerEvent { const _ActiveDatabaseClosed(); } diff --git a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart index 3c64d90f..00674420 100644 --- a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart +++ b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart @@ -9,6 +9,7 @@ import 'package:web/web.dart'; import '../connector.dart'; import '../log.dart'; +import '../sync/streaming_sync.dart'; import '../sync/sync_status.dart'; /// Names used in [SyncWorkerMessage] @@ -20,6 +21,9 @@ enum SyncWorkerMessageType { /// If parameters change, the sync worker reconnects. startSynchronization, + /// Update the active subscriptions that this client is interested in. + updateSubscriptions, + /// The [SyncWorkerMessage.payload] for the request is a numeric id, the /// response can be anything (void). /// This disconnects immediately, even if other clients are still open. @@ -74,6 +78,7 @@ extension type StartSynchronization._(JSObject _) implements JSObject { required String implementationName, required String schemaJson, String? syncParamsEncoded, + UpdateSubscriptions? subscriptions, }); external String get databaseName; @@ -83,6 +88,36 @@ extension type StartSynchronization._(JSObject _) implements JSObject { external String? get implementationName; external String get schemaJson; external String? get syncParamsEncoded; + external UpdateSubscriptions? get subscriptions; +} + +@anonymous +extension type UpdateSubscriptions.__(JSObject _inner) implements JSObject { + external factory UpdateSubscriptions._({ + required int requestId, + required JSArray content, + }); + + factory UpdateSubscriptions(int requestId, List streams) { + return UpdateSubscriptions._( + requestId: requestId, + content: streams + .map((e) => [e.name.toJS, e.parameters.toJS].toJS) + .toList() + .toJS, + ); + } + + external int get requestId; + external JSArray get content; + + List get toDart { + return content.toDart.map((e) { + final [name, parameters] = (e as JSArray).toDart; + + return (name: name.toDart, parameters: parameters.toDart); + }).toList(); + } } @anonymous @@ -339,6 +374,8 @@ final class WorkerCommunicationChannel { return; case SyncWorkerMessageType.startSynchronization: requestId = (message.payload as StartSynchronization).requestId; + case SyncWorkerMessageType.updateSubscriptions: + requestId = (message.payload as UpdateSubscriptions).requestId; case SyncWorkerMessageType.requestEndpoint: case SyncWorkerMessageType.abortSynchronization: case SyncWorkerMessageType.credentialsCallback: @@ -413,7 +450,11 @@ final class WorkerCommunicationChannel { } Future startSynchronization( - String databaseName, ResolvedSyncOptions options, Schema schema) async { + String databaseName, + ResolvedSyncOptions options, + Schema schema, + List streams, + ) async { final (id, completion) = _newRequest(); port.postMessage(SyncWorkerMessage( type: SyncWorkerMessageType.startSynchronization.name, @@ -428,11 +469,22 @@ final class WorkerCommunicationChannel { null => null, final params => jsonEncode(params), }, + subscriptions: UpdateSubscriptions(-1, streams), ), )); await completion; } + Future updateSubscriptions(List streams) async { + final (id, completion) = _newRequest(); + port.postMessage(SyncWorkerMessage( + type: SyncWorkerMessageType.updateSubscriptions.name, + payload: UpdateSubscriptions(id, streams), + )); + + await completion; + } + Future abortSynchronization() async { await _numericRequest(SyncWorkerMessageType.abortSynchronization); } From 42da494bcfe780cb67ebcbf5d3c0c4fa658caa7e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 12 Aug 2025 15:08:49 +0200 Subject: [PATCH 10/33] Use connection manager for tests --- .../lib/widgets/guard_by_sync.dart | 4 +- .../xcshareddata/swiftpm/Package.resolved | 6 +- .../lib/src/sync/connection_manager.dart | 3 +- .../lib/src/sync/streaming_sync.dart | 1 + .../test/in_memory_sync_test.dart | 83 ++++++++---------- .../test/utils/abstract_test_utils.dart | 85 +++++++++++++++++-- 6 files changed, 123 insertions(+), 59 deletions(-) diff --git a/demos/django-todolist/lib/widgets/guard_by_sync.dart b/demos/django-todolist/lib/widgets/guard_by_sync.dart index 000b5c8d..b65986b0 100644 --- a/demos/django-todolist/lib/widgets/guard_by_sync.dart +++ b/demos/django-todolist/lib/widgets/guard_by_sync.dart @@ -7,9 +7,9 @@ import 'package:powersync_django_todolist_demo/powersync.dart'; class GuardBySync extends StatelessWidget { final Widget child; - /// When set, wait only for a complete sync within the [BucketPriority] + /// When set, wait only for a complete sync within the [StreamPriority] /// instead of a full sync. - final BucketPriority? priority; + final StreamPriority? priority; const GuardBySync({ super.key, diff --git a/demos/supabase-todolist-drift/ios/Runner.xcworkspace/xcshareddata/swiftpm/Package.resolved b/demos/supabase-todolist-drift/ios/Runner.xcworkspace/xcshareddata/swiftpm/Package.resolved index 8e5eb05f..0c12c1e5 100644 --- a/demos/supabase-todolist-drift/ios/Runner.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/demos/supabase-todolist-drift/ios/Runner.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -5,7 +5,7 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/simolus3/CSQLite.git", "state" : { - "revision" : "a8d28afef08ad8faa4ee9ef7845f61c2e8ac5810" + "revision" : "a268235ae86718e66d6a29feef3bd22c772eb82b" } }, { @@ -13,8 +13,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/powersync-ja/powersync-sqlite-core-swift.git", "state" : { - "revision" : "00776db5157c8648671b00e6673603144fafbfeb", - "version" : "0.4.5" + "revision" : "b2a81af14e9ad83393eb187bb02e62e6db8b5ad6", + "version" : "0.4.6" } } ], diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 06d189a4..d1992972 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -25,7 +25,8 @@ final class ConnectionManager { final Map<_RawStreamKey, _ActiveSubscription> _locallyActiveSubscriptions = {}; - final StreamController _statusController = StreamController(); + final StreamController _statusController = + StreamController.broadcast(); /// Fires when an entry is added or removed from [_locallyActiveSubscriptions] /// while we're connected. diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index 4512a858..927f5978 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -623,6 +623,7 @@ final class _ActiveRustStreamingIteration { 'include_defaults': sync.options.includeDefaultStreams, 'active_streams': sync._activeSubscriptions .map((s) => {'name': s.name, 'params': s.parameters}) + .toList(), }), ); assert(_completedStream.isCompleted, 'Should have started streaming'); diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/in_memory_sync_test.dart index e4ab531b..9d44cb5c 100644 --- a/packages/powersync_core/test/in_memory_sync_test.dart +++ b/packages/powersync_core/test/in_memory_sync_test.dart @@ -5,7 +5,6 @@ import 'package:async/async.dart'; import 'package:logging/logging.dart'; import 'package:powersync_core/powersync_core.dart'; import 'package:powersync_core/sqlite3_common.dart'; -import 'package:powersync_core/src/sync/streaming_sync.dart'; import 'package:powersync_core/src/sync/protocol.dart'; import 'package:shelf/shelf.dart'; import 'package:shelf_router/shelf_router.dart'; @@ -55,35 +54,32 @@ void _declareTests(String name, SyncOptions options, bool bson) { late TestPowerSyncFactory factory; late CommonDatabase raw; - late PowerSyncDatabase database; + late TestDatabase database; late MockSyncService syncService; late Logger logger; - late StreamingSync syncClient; var credentialsCallbackCount = 0; Future Function(PowerSyncDatabase) uploadData = (db) async {}; - void createSyncClient({Schema? schema}) { + Future connect() async { final (client, server) = inMemoryServer(); server.mount((req) => syncService.router(req)); - final thisSyncClient = syncClient = database.connectWithMockService( - client, - TestConnector(() async { - credentialsCallbackCount++; - return PowerSyncCredentials( - endpoint: server.url.toString(), - token: 'token$credentialsCallbackCount', - expiresAt: DateTime.now(), - ); - }, uploadData: (db) => uploadData(db)), + database.httpClient = client; + await database.connect( + connector: TestConnector( + () async { + credentialsCallbackCount++; + return PowerSyncCredentials( + endpoint: server.url.toString(), + token: 'token$credentialsCallbackCount', + expiresAt: DateTime.now(), + ); + }, + uploadData: (db) => uploadData(db), + ), options: options, - customSchema: schema, ); - - addTearDown(() async { - await thisSyncClient.abort(); - }); } setUp(() async { @@ -94,7 +90,6 @@ void _declareTests(String name, SyncOptions options, bool bson) { factory = await testUtils.testFactory(); (raw, database) = await factory.openInMemoryDatabase(); await database.initialize(); - createSyncClient(); }); tearDown(() async { @@ -111,7 +106,7 @@ void _declareTests(String name, SyncOptions options, bool bson) { } }); } - syncClient.streamingSync(); + await connect(); await syncService.waitForListener; expect(database.currentStatus.lastSyncedAt, isNull); @@ -146,7 +141,7 @@ void _declareTests(String name, SyncOptions options, bool bson) { }); await expectLater( status, emits(isSyncStatus(downloading: false, hasSynced: true))); - await syncClient.abort(); + await database.disconnect(); final independentDb = factory.wrapRaw(raw, logger: ignoredLogger); addTearDown(independentDb.close); @@ -157,7 +152,7 @@ void _declareTests(String name, SyncOptions options, bool bson) { // A complete sync also means that all partial syncs have completed expect( independentDb.currentStatus - .statusForPriority(BucketPriority(3)) + .statusForPriority(StreamPriority(3)) .hasSynced, isTrue); }); @@ -251,7 +246,7 @@ void _declareTests(String name, SyncOptions options, bool bson) { database.watch('SELECT * FROM lists', throttle: Duration.zero)); await expectLater(query, emits(isEmpty)); - createSyncClient(schema: schema); + await database.updateSchema(schema); await waitForConnection(); syncService @@ -376,13 +371,13 @@ void _declareTests(String name, SyncOptions options, bool bson) { status, emitsThrough( isSyncStatus(downloading: true, hasSynced: false).having( - (e) => e.statusForPriority(BucketPriority(0)).hasSynced, + (e) => e.statusForPriority(StreamPriority(0)).hasSynced, 'status for $prio', isTrue, )), ); - await database.waitForFirstSync(priority: BucketPriority(prio)); + await database.waitForFirstSync(priority: StreamPriority(prio)); expect(await database.getAll('SELECT * FROM customers'), hasLength(prio + 1)); } @@ -419,9 +414,9 @@ void _declareTests(String name, SyncOptions options, bool bson) { 'priority': 1, } }); - await database.waitForFirstSync(priority: BucketPriority(1)); + await database.waitForFirstSync(priority: StreamPriority(1)); expect(database.currentStatus.hasSynced, isFalse); - await syncClient.abort(); + await database.disconnect(); final independentDb = factory.wrapRaw(raw, logger: ignoredLogger); addTearDown(independentDb.close); @@ -430,12 +425,12 @@ void _declareTests(String name, SyncOptions options, bool bson) { // Completing a sync for prio 1 implies a completed sync for prio 0 expect( independentDb.currentStatus - .statusForPriority(BucketPriority(0)) + .statusForPriority(StreamPriority(0)) .hasSynced, isTrue); expect( independentDb.currentStatus - .statusForPriority(BucketPriority(3)) + .statusForPriority(StreamPriority(3)) .hasSynced, isFalse); }); @@ -683,10 +678,9 @@ void _declareTests(String name, SyncOptions options, bool bson) { await expectProgress(status, total: progress(5, 10)); // Emulate the app closing - create a new independent sync client. - await syncClient.abort(); + await database.disconnect(); syncService.endCurrentListener(); - createSyncClient(); status = await waitForConnection(); // Send same checkpoint again @@ -717,10 +711,9 @@ void _declareTests(String name, SyncOptions options, bool bson) { await expectProgress(status, total: progress(5, 10)); // Emulate the app closing - create a new independent sync client. - await syncClient.abort(); + await database.disconnect(); syncService.endCurrentListener(); - createSyncClient(); status = await waitForConnection(); // Send checkpoint with additional data @@ -751,9 +744,9 @@ void _declareTests(String name, SyncOptions options, bool bson) { // A sync rule deploy could reset buckets, making the new bucket smaller // than the existing one. - await syncClient.abort(); + await database.disconnect(); syncService.endCurrentListener(); - createSyncClient(); + status = await waitForConnection(); syncService.addLine({ 'checkpoint': Checkpoint( @@ -772,8 +765,8 @@ void _declareTests(String name, SyncOptions options, bool bson) { await expectProgress( status, priorities: { - BucketPriority(0): prio0, - BucketPriority(2): prio2, + StreamPriority(0): prio0, + StreamPriority(2): prio2, }, total: prio2, ); @@ -837,7 +830,7 @@ void _declareTests(String name, SyncOptions options, bool bson) { }); await expectLater(status, emits(isSyncStatus(downloading: true))); - await syncClient.abort(); + await database.disconnect(); expect(syncService.controller.hasListener, isFalse); }); @@ -856,9 +849,6 @@ void _declareTests(String name, SyncOptions options, bool bson) { syncService.addLine({ 'checkpoint_complete': {'last_op_id': '10'} }); - - await pumpEventQueue(); - expect(syncService.controller.hasListener, isFalse); syncService.endCurrentListener(); // Should reconnect after delay. @@ -878,9 +868,6 @@ void _declareTests(String name, SyncOptions options, bool bson) { await expectLater(status, emits(isSyncStatus(downloading: true))); syncService.addKeepAlive(0); - - await pumpEventQueue(); - expect(syncService.controller.hasListener, isFalse); syncService.endCurrentListener(); // Should reconnect after delay. @@ -952,11 +939,11 @@ void _declareTests(String name, SyncOptions options, bool bson) { await Completer().future; })); - syncClient.streamingSync(); + await connect(); await requestStarted.future; expect(database.currentStatus, isSyncStatus(connecting: true)); - await syncClient.abort(); + await database.disconnect(); expect(database.currentStatus.anyError, isNull); }); @@ -975,7 +962,7 @@ void _declareTests(String name, SyncOptions options, bool bson) { }); await expectLater(status, emits(isSyncStatus(downloading: true))); - await syncClient.abort(); + await database.disconnect(); expect(database.currentStatus.anyError, isNull); }); }); diff --git a/packages/powersync_core/test/utils/abstract_test_utils.dart b/packages/powersync_core/test/utils/abstract_test_utils.dart index a95d2604..4a97c0a9 100644 --- a/packages/powersync_core/test/utils/abstract_test_utils.dart +++ b/packages/powersync_core/test/utils/abstract_test_utils.dart @@ -1,8 +1,11 @@ +import 'dart:async'; import 'dart:convert'; import 'package:http/http.dart'; import 'package:logging/logging.dart'; import 'package:powersync_core/powersync_core.dart'; +import 'package:powersync_core/src/abort_controller.dart'; +import 'package:powersync_core/src/database/powersync_db_mixin.dart'; import 'package:powersync_core/src/sync/bucket_storage.dart'; import 'package:powersync_core/src/sync/internal_connector.dart'; import 'package:powersync_core/src/sync/options.dart'; @@ -63,7 +66,7 @@ Logger _makeTestLogger({Level level = Level.ALL, String? name}) { abstract mixin class TestPowerSyncFactory implements PowerSyncOpenFactory { Future openRawInMemoryDatabase(); - Future<(CommonDatabase, PowerSyncDatabase)> openInMemoryDatabase({ + Future<(CommonDatabase, TestDatabase)> openInMemoryDatabase({ Schema? schema, Logger? logger, }) async { @@ -71,16 +74,16 @@ abstract mixin class TestPowerSyncFactory implements PowerSyncOpenFactory { return (raw, wrapRaw(raw, customSchema: schema, logger: logger)); } - PowerSyncDatabase wrapRaw( + TestDatabase wrapRaw( CommonDatabase raw, { Logger? logger, Schema? customSchema, }) { - return PowerSyncDatabase.withDatabase( - schema: customSchema ?? schema, + return TestDatabase( database: SqliteDatabase.singleConnection( SqliteConnection.synchronousWrapper(raw)), - logger: logger, + logger: logger ?? Logger.detached('PowerSync.test'), + schema: schema, ); } } @@ -151,6 +154,78 @@ class TestConnector extends PowerSyncBackendConnector { } } +final class TestDatabase + with SqliteQueries, PowerSyncDatabaseMixin + implements PowerSyncDatabase { + @override + final SqliteDatabase database; + @override + final Logger logger; + @override + Schema schema; + + @override + late final Future isInitialized; + + Client? httpClient; + + TestDatabase({ + required this.database, + required this.logger, + required this.schema, + }) { + isInitialized = baseInit(); + } + + @override + Future connectInternal({ + required PowerSyncBackendConnector connector, + required ResolvedSyncOptions options, + required List initiallyActiveStreams, + required Stream> activeStreams, + required AbortController abort, + required Zone asyncWorkZone, + }) async { + final impl = StreamingSyncImplementation( + adapter: BucketStorage(this), + schemaJson: jsonEncode(schema), + client: httpClient!, + options: options, + connector: InternalConnector.wrap(connector, this), + logger: logger, + crudUpdateTriggerStream: database + .onChange(['ps_crud'], throttle: const Duration(milliseconds: 10)), + activeSubscriptions: initiallyActiveStreams, + ); + impl.statusStream.listen(setStatus); + + asyncWorkZone.run(impl.streamingSync); + final subscriptions = activeStreams.listen(impl.updateSubscriptions); + + abort.onAbort.then((_) async { + subscriptions.cancel(); + await impl.abort(); + abort.completeAbort(); + }).ignore(); + } + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {String? debugContext, Duration? lockTimeout}) async { + await isInitialized; + return database.readLock(callback, + debugContext: debugContext, lockTimeout: lockTimeout); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {String? debugContext, Duration? lockTimeout}) async { + await isInitialized; + return database.writeLock(callback, + debugContext: debugContext, lockTimeout: lockTimeout); + } +} + extension MockSync on PowerSyncDatabase { StreamingSyncImplementation connectWithMockService( Client client, From 4a512f8a6ab3c71ecf67a4a99c7e6a8ae8d2e1bc Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 12 Aug 2025 15:09:33 +0200 Subject: [PATCH 11/33] Move sync test --- .../test/{ => sync}/in_memory_sync_test.dart | 10 +++++----- .../test/{ => sync}/streaming_sync_test.dart | 8 ++++---- .../test/{ => sync}/sync_types_test.dart | 0 3 files changed, 9 insertions(+), 9 deletions(-) rename packages/powersync_core/test/{ => sync}/in_memory_sync_test.dart (99%) rename packages/powersync_core/test/{ => sync}/streaming_sync_test.dart (97%) rename packages/powersync_core/test/{ => sync}/sync_types_test.dart (100%) diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/sync/in_memory_sync_test.dart similarity index 99% rename from packages/powersync_core/test/in_memory_sync_test.dart rename to packages/powersync_core/test/sync/in_memory_sync_test.dart index 9d44cb5c..044914e0 100644 --- a/packages/powersync_core/test/in_memory_sync_test.dart +++ b/packages/powersync_core/test/sync/in_memory_sync_test.dart @@ -10,11 +10,11 @@ import 'package:shelf/shelf.dart'; import 'package:shelf_router/shelf_router.dart'; import 'package:test/test.dart'; -import 'bucket_storage_test.dart'; -import 'server/sync_server/in_memory_sync_server.dart'; -import 'utils/abstract_test_utils.dart'; -import 'utils/in_memory_http.dart'; -import 'utils/test_utils_impl.dart'; +import '../bucket_storage_test.dart'; +import '../server/sync_server/in_memory_sync_server.dart'; +import '../utils/abstract_test_utils.dart'; +import '../utils/in_memory_http.dart'; +import '../utils/test_utils_impl.dart'; void main() { _declareTests( diff --git a/packages/powersync_core/test/streaming_sync_test.dart b/packages/powersync_core/test/sync/streaming_sync_test.dart similarity index 97% rename from packages/powersync_core/test/streaming_sync_test.dart rename to packages/powersync_core/test/sync/streaming_sync_test.dart index 40becd16..5017993f 100644 --- a/packages/powersync_core/test/streaming_sync_test.dart +++ b/packages/powersync_core/test/sync/streaming_sync_test.dart @@ -9,10 +9,10 @@ import 'package:logging/logging.dart'; import 'package:powersync_core/powersync_core.dart'; import 'package:test/test.dart'; -import 'server/sync_server/in_memory_sync_server.dart'; -import 'test_server.dart'; -import 'utils/abstract_test_utils.dart'; -import 'utils/test_utils_impl.dart'; +import '../server/sync_server/in_memory_sync_server.dart'; +import '../test_server.dart'; +import '../utils/abstract_test_utils.dart'; +import '../utils/test_utils_impl.dart'; final testUtils = TestUtils(); diff --git a/packages/powersync_core/test/sync_types_test.dart b/packages/powersync_core/test/sync/sync_types_test.dart similarity index 100% rename from packages/powersync_core/test/sync_types_test.dart rename to packages/powersync_core/test/sync/sync_types_test.dart From 96296a95db989351fc64390e41979593c39da92a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 12 Aug 2025 16:33:16 +0200 Subject: [PATCH 12/33] Start with some unit tests --- .../powersync_core/lib/powersync_core.dart | 1 + .../lib/src/sync/connection_manager.dart | 4 +- .../lib/src/sync/instruction.dart | 2 +- .../lib/src/sync/mutable_sync_status.dart | 10 +- .../powersync_core/lib/src/sync/options.dart | 6 +- .../powersync_core/lib/src/sync/stream.dart | 7 +- .../lib/src/sync/sync_status.dart | 8 +- .../lib/src/web/sync_worker_protocol.dart | 2 +- .../test/{ => sync}/bucket_storage_test.dart | 10 +- .../test/sync/in_memory_sync_test.dart | 50 +---- .../powersync_core/test/sync/stream_test.dart | 186 ++++++++++++++++++ packages/powersync_core/test/sync/utils.dart | 134 +++++++++++++ 12 files changed, 352 insertions(+), 68 deletions(-) rename packages/powersync_core/test/{ => sync}/bucket_storage_test.dart (99%) create mode 100644 packages/powersync_core/test/sync/stream_test.dart create mode 100644 packages/powersync_core/test/sync/utils.dart diff --git a/packages/powersync_core/lib/powersync_core.dart b/packages/powersync_core/lib/powersync_core.dart index b4dfe35d..62946382 100644 --- a/packages/powersync_core/lib/powersync_core.dart +++ b/packages/powersync_core/lib/powersync_core.dart @@ -11,6 +11,7 @@ export 'src/log.dart'; export 'src/open_factory.dart'; export 'src/schema.dart'; export 'src/sync/options.dart' hide ResolvedSyncOptions; +export 'src/sync/stream.dart' hide CoreActiveStreamSubscription; export 'src/sync/sync_status.dart' hide BucketProgress, InternalSyncDownloadProgress; export 'src/uuid.dart'; diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index d1992972..8a92143c 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -7,7 +7,6 @@ import 'package:powersync_core/src/abort_controller.dart'; import 'package:powersync_core/src/database/active_instances.dart'; import 'package:powersync_core/src/database/powersync_db_mixin.dart'; import 'package:powersync_core/src/sync/options.dart'; -import 'package:powersync_core/src/sync/stream.dart'; import 'streaming_sync.dart'; @@ -189,6 +188,7 @@ final class ConnectionManager { hasSynced: status.lastSyncedAt != null ? true : status.hasSynced ?? currentStatus.hasSynced, + streamSubscriptions: status.internalSubscriptions, ); // If the absence of hasSynced was the only difference, the new states @@ -226,7 +226,7 @@ final class ConnectionManager { Future _subscriptionsCommand(Object? command) async { await db.writeTransaction((tx) { - return db.execute( + return tx.execute( 'SELECT powersync_control(?, ?)', ['subscriptions', json.encode(command)], ); diff --git a/packages/powersync_core/lib/src/sync/instruction.dart b/packages/powersync_core/lib/src/sync/instruction.dart index 6344e017..a6eedcd2 100644 --- a/packages/powersync_core/lib/src/sync/instruction.dart +++ b/packages/powersync_core/lib/src/sync/instruction.dart @@ -85,7 +85,7 @@ final class CoreSyncStatus { null => null, final raw as Map => DownloadProgress.fromJson(raw), }, - streams: (json['stream'] as List?) + streams: (json['streams'] as List?) ?.map((e) => CoreActiveStreamSubscription.fromJson(e as Map)) .toList(), diff --git a/packages/powersync_core/lib/src/sync/mutable_sync_status.dart b/packages/powersync_core/lib/src/sync/mutable_sync_status.dart index 2707cb58..02e5ff48 100644 --- a/packages/powersync_core/lib/src/sync/mutable_sync_status.dart +++ b/packages/powersync_core/lib/src/sync/mutable_sync_status.dart @@ -16,7 +16,7 @@ final class MutableSyncStatus { InternalSyncDownloadProgress? downloadProgress; List priorityStatusEntries = const []; - List streams = const []; + List? streams; DateTime? lastSyncedAt; @@ -53,9 +53,9 @@ final class MutableSyncStatus { hasSynced: true, lastSyncedAt: now, priority: maxBy( - applied.checksums.map((cs) => BucketPriority(cs.priority)), + applied.checksums.map((cs) => StreamPriority(cs.priority)), (priority) => priority, - compare: BucketPriority.comparator, + compare: StreamPriority.comparator, )!, ) ]; @@ -92,8 +92,9 @@ final class MutableSyncStatus { final downloading => InternalSyncDownloadProgress(downloading.buckets), }; lastSyncedAt = status.priorityStatus - .firstWhereOrNull((s) => s.priority == BucketPriority.fullSyncPriority) + .firstWhereOrNull((s) => s.priority == StreamPriority.fullSyncPriority) ?.lastSyncedAt; + streams = status.streams; } SyncStatus immutableSnapshot() { @@ -108,6 +109,7 @@ final class MutableSyncStatus { hasSynced: null, // Stream client is not supposed to set this value. uploadError: uploadError, downloadError: downloadError, + streamSubscriptions: streams, ); } } diff --git a/packages/powersync_core/lib/src/sync/options.dart b/packages/powersync_core/lib/src/sync/options.dart index f67dd144..e83bb436 100644 --- a/packages/powersync_core/lib/src/sync/options.dart +++ b/packages/powersync_core/lib/src/sync/options.dart @@ -47,6 +47,7 @@ final class SyncOptions { retryDelay: retryDelay, params: params ?? this.params, syncImplementation: syncImplementation, + includeDefaultStreams: includeDefaultStreams, ); } } @@ -106,11 +107,14 @@ extension type ResolvedSyncOptions(SyncOptions source) { crudThrottleTime: other.crudThrottleTime ?? crudThrottleTime, retryDelay: other.retryDelay ?? retryDelay, params: other.params ?? params, + includeDefaultStreams: + other.includeDefaultStreams ?? includeDefaultStreams, ); final didChange = !_mapEquality.equals(newOptions.params, params) || newOptions.crudThrottleTime != crudThrottleTime || - newOptions.retryDelay != retryDelay; + newOptions.retryDelay != retryDelay || + newOptions.includeDefaultStreams != includeDefaultStreams; return (ResolvedSyncOptions(newOptions), didChange); } diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index a23e4816..a57bfb54 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -123,8 +123,11 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { factory CoreActiveStreamSubscription.fromJson(Map json) { return CoreActiveStreamSubscription._( name: json['name'] as String, - parameters: json['parameters'] as Map, - priority: StreamPriority(json['priority'] as int), + parameters: json['parameters'] as Map?, + priority: switch (json['priority'] as int?) { + final prio? => StreamPriority(prio), + null => StreamPriority.fullSyncPriority, + }, associatedBuckets: (json['associated_buckets'] as List).cast(), active: json['active'] as bool, isDefault: json['is_default'] as bool, diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index d46e3850..2013f39a 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -208,6 +208,12 @@ final class SyncStatus { static const _mapEquality = MapEquality(); } +@internal +extension InternalSyncStatusAccess on SyncStatus { + List? get internalSubscriptions => + _internalSubscriptions; +} + final class SyncStreamStatus { final ProgressWithOperations? progress; final CoreActiveStreamSubscription _internal; @@ -276,7 +282,7 @@ class UploadQueueStats { /// Per-bucket download progress information. @internal typedef BucketProgress = ({ - BucketPriority priority, + StreamPriority priority, int atLast, int sinceLast, int targetCount, diff --git a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart index 00674420..88367210 100644 --- a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart +++ b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart @@ -92,7 +92,7 @@ extension type StartSynchronization._(JSObject _) implements JSObject { } @anonymous -extension type UpdateSubscriptions.__(JSObject _inner) implements JSObject { +extension type UpdateSubscriptions._raw(JSObject _inner) implements JSObject { external factory UpdateSubscriptions._({ required int requestId, required JSArray content, diff --git a/packages/powersync_core/test/bucket_storage_test.dart b/packages/powersync_core/test/sync/bucket_storage_test.dart similarity index 99% rename from packages/powersync_core/test/bucket_storage_test.dart rename to packages/powersync_core/test/sync/bucket_storage_test.dart index 94338791..496e5a49 100644 --- a/packages/powersync_core/test/bucket_storage_test.dart +++ b/packages/powersync_core/test/sync/bucket_storage_test.dart @@ -4,8 +4,9 @@ import 'package:powersync_core/src/sync/protocol.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:test/test.dart'; -import 'utils/abstract_test_utils.dart'; -import 'utils/test_utils_impl.dart'; +import '../utils/abstract_test_utils.dart'; +import '../utils/test_utils_impl.dart'; +import 'utils.dart'; final testUtils = TestUtils(); @@ -39,11 +40,6 @@ const removeAsset1_4 = OplogEntry( const removeAsset1_5 = OplogEntry( opId: '5', op: OpType.remove, rowType: 'assets', rowId: 'O1', checksum: 5); -BucketChecksum checksum( - {required String bucket, required int checksum, int priority = 1}) { - return BucketChecksum(bucket: bucket, priority: priority, checksum: checksum); -} - SyncDataBatch syncDataBatch(List data) { return SyncDataBatch(data); } diff --git a/packages/powersync_core/test/sync/in_memory_sync_test.dart b/packages/powersync_core/test/sync/in_memory_sync_test.dart index 044914e0..0b478e3c 100644 --- a/packages/powersync_core/test/sync/in_memory_sync_test.dart +++ b/packages/powersync_core/test/sync/in_memory_sync_test.dart @@ -10,11 +10,11 @@ import 'package:shelf/shelf.dart'; import 'package:shelf_router/shelf_router.dart'; import 'package:test/test.dart'; -import '../bucket_storage_test.dart'; import '../server/sync_server/in_memory_sync_server.dart'; import '../utils/abstract_test_utils.dart'; import '../utils/in_memory_http.dart'; import '../utils/test_utils_impl.dart'; +import 'utils.dart'; void main() { _declareTests( @@ -968,51 +968,3 @@ void _declareTests(String name, SyncOptions options, bool bson) { }); }); } - -TypeMatcher isSyncStatus({ - Object? downloading, - Object? connected, - Object? connecting, - Object? hasSynced, - Object? downloadProgress, -}) { - var matcher = isA(); - if (downloading != null) { - matcher = matcher.having((e) => e.downloading, 'downloading', downloading); - } - if (connected != null) { - matcher = matcher.having((e) => e.connected, 'connected', connected); - } - if (connecting != null) { - matcher = matcher.having((e) => e.connecting, 'connecting', connecting); - } - if (hasSynced != null) { - matcher = matcher.having((e) => e.hasSynced, 'hasSynced', hasSynced); - } - if (downloadProgress != null) { - matcher = matcher.having( - (e) => e.downloadProgress, 'downloadProgress', downloadProgress); - } - - return matcher; -} - -TypeMatcher isSyncDownloadProgress({ - required Object progress, - Map priorities = const {}, -}) { - var matcher = - isA().having((e) => e, 'untilCompletion', progress); - priorities.forEach((priority, expected) { - matcher = matcher.having( - (e) => e.untilPriority(priority), 'untilPriority($priority)', expected); - }); - - return matcher; -} - -TypeMatcher progress(int completed, int total) { - return isA() - .having((e) => e.downloadedOperations, 'completed', completed) - .having((e) => e.totalOperations, 'total', total); -} diff --git a/packages/powersync_core/test/sync/stream_test.dart b/packages/powersync_core/test/sync/stream_test.dart new file mode 100644 index 00000000..9d85769f --- /dev/null +++ b/packages/powersync_core/test/sync/stream_test.dart @@ -0,0 +1,186 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:async/async.dart'; +import 'package:logging/logging.dart'; +import 'package:powersync_core/powersync_core.dart'; + +import 'package:test/test.dart'; + +import '../server/sync_server/in_memory_sync_server.dart'; +import '../utils/abstract_test_utils.dart'; +import '../utils/in_memory_http.dart'; +import '../utils/test_utils_impl.dart'; +import 'utils.dart'; + +void main() { + late final testUtils = TestUtils(); + + late TestPowerSyncFactory factory; + + late TestDatabase database; + late MockSyncService syncService; + late Logger logger; + late SyncOptions options; + + var credentialsCallbackCount = 0; + + Future connect() async { + final (client, server) = inMemoryServer(); + server.mount(syncService.router.call); + + database.httpClient = client; + await database.connect( + connector: TestConnector( + () async { + credentialsCallbackCount++; + return PowerSyncCredentials( + endpoint: server.url.toString(), + token: 'token$credentialsCallbackCount', + expiresAt: DateTime.now(), + ); + }, + uploadData: (db) async {}, + ), + options: options, + ); + } + + setUp(() async { + options = SyncOptions(syncImplementation: SyncClientImplementation.rust); + logger = Logger.detached('powersync.active')..level = Level.ALL; + credentialsCallbackCount = 0; + syncService = MockSyncService(); + + factory = await testUtils.testFactory(); + (_, database) = await factory.openInMemoryDatabase(); + await database.initialize(); + }); + + tearDown(() async { + await database.close(); + await syncService.stop(); + }); + + Future> waitForConnection( + {bool expectNoWarnings = true}) async { + if (expectNoWarnings) { + logger.onRecord.listen((e) { + if (e.level >= Level.WARNING) { + fail('Unexpected log: $e, ${e.stackTrace}'); + } + }); + } + await connect(); + await syncService.waitForListener; + + expect(database.currentStatus.lastSyncedAt, isNull); + expect(database.currentStatus.downloading, isFalse); + final status = StreamQueue(database.statusStream); + addTearDown(status.cancel); + + syncService.addKeepAlive(); + await expectLater( + status, emitsThrough(isSyncStatus(connected: true, hasSynced: false))); + return status; + } + + test('can disable default streams', () async { + options = SyncOptions( + syncImplementation: SyncClientImplementation.rust, + includeDefaultStreams: false, + ); + + await waitForConnection(); + final request = await syncService.waitForListener; + expect(json.decode(await request.readAsString()), + containsPair('streams', containsPair('include_defaults', false))); + }); + + test('subscribes with streams', () async { + final a = await database.syncStream('stream', {'foo': 'a'}).subscribe(); + final b = await database.syncStream('stream', {'foo': 'b'}).subscribe( + priority: StreamPriority(1)); + + final statusStream = await waitForConnection(); + final request = await syncService.waitForListener; + expect( + json.decode(await request.readAsString()), + containsPair( + 'streams', + containsPair('subscriptions', [ + { + 'stream': 'stream', + 'parameters': {'foo': 'a'}, + 'override_priority': null, + }, + { + 'stream': 'stream', + 'parameters': {'foo': 'b'}, + 'override_priority': 1, + }, + ]), + ), + ); + + syncService.addLine( + checkpoint( + lastOpId: 0, + buckets: [ + bucketDescription('a', subscriptions: [ + {'sub': 0} + ]), + bucketDescription('b', priority: 1, subscriptions: [ + {'sub': 1} + ]) + ], + streams: [ + stream('stream', false), + ], + ), + ); + + var status = await statusStream.next; + for (final subscription in [a, b]) { + expect(status.statusFor(subscription)!.subscription.active, true); + expect(status.statusFor(subscription)!.subscription.lastSyncedAt, isNull); + expect( + status.statusFor(subscription)!.subscription.hasExplicitSubscription, + true, + ); + } + + syncService.addLine(checkpointComplete(priority: 1)); + status = await statusStream.next; + expect(status.statusFor(a)!.subscription.lastSyncedAt, isNull); + expect(status.statusFor(b)!.subscription.lastSyncedAt, isNotNull); + await b.waitForFirstSync(); + + syncService.addLine(checkpointComplete()); + await a.waitForFirstSync(); + }); + + test('reports default streams', () async { + final status = await waitForConnection(); + syncService.addLine( + checkpoint(lastOpId: 0, streams: [stream('default_stream', true)]), + ); + + await expectLater( + status, + emits( + isSyncStatus( + activeSubscriptions: [ + isStreamStatus( + subscription: isSyncSubscription( + name: 'default_stream', + parameters: null, + ), + isDefault: true, + ), + ], + ), + ), + ); + }); +} diff --git a/packages/powersync_core/test/sync/utils.dart b/packages/powersync_core/test/sync/utils.dart new file mode 100644 index 00000000..0a619514 --- /dev/null +++ b/packages/powersync_core/test/sync/utils.dart @@ -0,0 +1,134 @@ +import 'package:powersync_core/powersync_core.dart'; +import 'package:powersync_core/src/sync/protocol.dart'; +import 'package:test/test.dart'; + +TypeMatcher isSyncStatus({ + Object? downloading, + Object? connected, + Object? connecting, + Object? hasSynced, + Object? downloadProgress, + Object? activeSubscriptions, +}) { + var matcher = isA(); + if (downloading != null) { + matcher = matcher.having((e) => e.downloading, 'downloading', downloading); + } + if (connected != null) { + matcher = matcher.having((e) => e.connected, 'connected', connected); + } + if (connecting != null) { + matcher = matcher.having((e) => e.connecting, 'connecting', connecting); + } + if (hasSynced != null) { + matcher = matcher.having((e) => e.hasSynced, 'hasSynced', hasSynced); + } + if (downloadProgress != null) { + matcher = matcher.having( + (e) => e.downloadProgress, 'downloadProgress', downloadProgress); + } + if (activeSubscriptions != null) { + matcher = matcher.having((e) => e.activeSubscriptions, + 'activeSubscriptions', activeSubscriptions); + } + + return matcher; +} + +TypeMatcher isSyncDownloadProgress({ + required Object progress, + Map priorities = const {}, +}) { + var matcher = + isA().having((e) => e, 'untilCompletion', progress); + priorities.forEach((priority, expected) { + matcher = matcher.having( + (e) => e.untilPriority(priority), 'untilPriority($priority)', expected); + }); + + return matcher; +} + +TypeMatcher progress(int completed, int total) { + return isA() + .having((e) => e.downloadedOperations, 'completed', completed) + .having((e) => e.totalOperations, 'total', total); +} + +TypeMatcher isStreamStatus({ + required Object? subscription, + Object? progress, + Object? isDefault, +}) { + var matcher = isA() + .having((e) => e.subscription, 'subscription', subscription); + if (progress case final progress?) { + matcher = matcher.having((e) => e.progress, 'progress', progress); + } + if (isDefault case final isDefault?) { + matcher = matcher.having((e) => e.isDefault, 'isDefault', isDefault); + } + + return matcher; +} + +TypeMatcher isSyncSubscription({ + required Object name, + required Object? parameters, +}) { + return isA() + .having((e) => e.name, 'name', name) + .having((e) => e.parameters, 'parameters', parameters); +} + +BucketChecksum checksum( + {required String bucket, required int checksum, int priority = 1}) { + return BucketChecksum(bucket: bucket, priority: priority, checksum: checksum); +} + +/// Creates a `checkpoint` line. +Object checkpoint({ + required int lastOpId, + List buckets = const [], + String? writeCheckpoint, + List streams = const [], +}) { + return { + 'checkpoint': { + 'last_op_id': '$lastOpId', + 'write_checkpoint': null, + 'buckets': buckets, + 'streams': streams, + } + }; +} + +Object stream(String name, bool isDefault, {List errors = const []}) { + return {'name': name, 'is_default': isDefault, 'errors': errors}; +} + +/// Creates a `checkpoint_complete` or `partial_checkpoint_complete` line. +Object checkpointComplete({int? priority, String lastOpId = '1'}) { + return { + priority == null ? 'checkpoint_complete' : 'partial_checkpoint_complete': { + 'last_op_id': lastOpId, + if (priority != null) 'priority': priority, + }, + }; +} + +Object bucketDescription( + String name, { + int checksum = 0, + int priority = 3, + int count = 1, + Object? subscriptions, +}) { + return { + 'bucket': name, + 'checksum': checksum, + 'priority': priority, + 'count': count, + if (subscriptions != null) 'subscriptions': subscriptions, + }; +} From 5ff844b6fe06f78b69883fe89f0bbac4d11fb97b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 12 Aug 2025 17:14:44 +0200 Subject: [PATCH 13/33] Fix a few more warnings --- packages/powersync_core/lib/powersync_core.dart | 2 +- packages/powersync_core/lib/src/sync/connection_manager.dart | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/powersync_core/lib/powersync_core.dart b/packages/powersync_core/lib/powersync_core.dart index 62946382..ef2e97c7 100644 --- a/packages/powersync_core/lib/powersync_core.dart +++ b/packages/powersync_core/lib/powersync_core.dart @@ -13,5 +13,5 @@ export 'src/schema.dart'; export 'src/sync/options.dart' hide ResolvedSyncOptions; export 'src/sync/stream.dart' hide CoreActiveStreamSubscription; export 'src/sync/sync_status.dart' - hide BucketProgress, InternalSyncDownloadProgress; + hide BucketProgress, InternalSyncDownloadProgress, InternalSyncStatusAccess; export 'src/uuid.dart'; diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 8a92143c..bf797518 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -2,11 +2,13 @@ import 'dart:async'; import 'dart:convert'; import 'package:meta/meta.dart'; -import 'package:powersync_core/powersync_core.dart'; import 'package:powersync_core/src/abort_controller.dart'; +import 'package:powersync_core/src/connector.dart'; import 'package:powersync_core/src/database/active_instances.dart'; import 'package:powersync_core/src/database/powersync_db_mixin.dart'; import 'package:powersync_core/src/sync/options.dart'; +import 'package:powersync_core/src/sync/stream.dart'; +import 'package:powersync_core/src/sync/sync_status.dart'; import 'streaming_sync.dart'; From cf201e025f7108ecb00b17bf242ad05b832defa9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 13 Aug 2025 11:08:44 +0200 Subject: [PATCH 14/33] Properly update subscriptions --- .../lib/src/database/powersync_db_mixin.dart | 46 +++++-------------- .../lib/src/sync/connection_manager.dart | 2 +- .../lib/src/sync/instruction.dart | 4 +- .../lib/src/sync/mutable_sync_status.dart | 4 +- .../lib/src/sync/streaming_sync.dart | 22 +++++++-- .../powersync_core/test/sync/stream_test.dart | 28 +++++++++++ 6 files changed, 62 insertions(+), 44 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index a46d76f9..93a04a58 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'package:async/async.dart'; import 'package:logging/logging.dart'; @@ -15,6 +16,8 @@ import 'package:powersync_core/src/schema.dart'; import 'package:powersync_core/src/schema_logic.dart'; import 'package:powersync_core/src/schema_logic.dart' as schema_logic; import 'package:powersync_core/src/sync/connection_manager.dart'; +import 'package:powersync_core/src/sync/instruction.dart'; +import 'package:powersync_core/src/sync/mutable_sync_status.dart'; import 'package:powersync_core/src/sync/options.dart'; import 'package:powersync_core/src/sync/sync_status.dart'; @@ -138,42 +141,15 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { Future _updateHasSynced() async { // Query the database to see if any data has been synced. - final result = await database.getAll( - 'SELECT priority, last_synced_at FROM ps_sync_state ORDER BY priority;', + final row = await database.get( + 'SELECT powersync_offline_sync_status() AS r;', ); - const prioritySentinel = 2147483647; - var hasSynced = false; - DateTime? lastCompleteSync; - final priorityStatusEntries = []; - DateTime parseDateTime(String sql) { - return DateTime.parse('${sql}Z').toLocal(); - } - - for (final row in result) { - final priority = row.columnAt(0) as int; - final lastSyncedAt = parseDateTime(row.columnAt(1) as String); - - if (priority == prioritySentinel) { - hasSynced = true; - lastCompleteSync = lastSyncedAt; - } else { - priorityStatusEntries.add(( - hasSynced: true, - lastSyncedAt: lastSyncedAt, - priority: BucketPriority(priority) - )); - } - } + final status = CoreSyncStatus.fromJson( + json.decode(row['r'] as String) as Map); - if (hasSynced != currentStatus.hasSynced) { - final status = SyncStatus( - hasSynced: hasSynced, - lastSyncedAt: lastCompleteSync, - priorityStatusEntries: priorityStatusEntries, - ); - setStatus(status); - } + setStatus((MutableSyncStatus()..applyFromCore(status)) + .immutableSnapshot(setLastSynced: true)); } /// Returns a [Future] which will resolve once at least one full sync cycle @@ -181,10 +157,10 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// reached across all buckets). /// /// When [priority] is null (the default), this method waits for the first - /// full sync checkpoint to complete. When set to a [BucketPriority] however, + /// full sync checkpoint to complete. When set to a [StreamPriority] however, /// it completes once all buckets within that priority (as well as those in /// higher priorities) have been synchronized at least once. - Future waitForFirstSync({BucketPriority? priority}) async { + Future waitForFirstSync({StreamPriority? priority}) async { bool matches(SyncStatus status) { if (priority == null) { return status.hasSynced == true; diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index bf797518..aeb57691 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -197,7 +197,7 @@ final class ConnectionManager { // would be equal and don't require an event. So, check again. if (newStatus != currentStatus) { _currentStatus = newStatus; - _statusController.add(currentStatus); + _statusController.add(_currentStatus); } } } diff --git a/packages/powersync_core/lib/src/sync/instruction.dart b/packages/powersync_core/lib/src/sync/instruction.dart index a6eedcd2..cde81303 100644 --- a/packages/powersync_core/lib/src/sync/instruction.dart +++ b/packages/powersync_core/lib/src/sync/instruction.dart @@ -85,8 +85,8 @@ final class CoreSyncStatus { null => null, final raw as Map => DownloadProgress.fromJson(raw), }, - streams: (json['streams'] as List?) - ?.map((e) => + streams: (json['streams'] as List) + .map((e) => CoreActiveStreamSubscription.fromJson(e as Map)) .toList(), ); diff --git a/packages/powersync_core/lib/src/sync/mutable_sync_status.dart b/packages/powersync_core/lib/src/sync/mutable_sync_status.dart index 02e5ff48..273cd597 100644 --- a/packages/powersync_core/lib/src/sync/mutable_sync_status.dart +++ b/packages/powersync_core/lib/src/sync/mutable_sync_status.dart @@ -97,7 +97,7 @@ final class MutableSyncStatus { streams = status.streams; } - SyncStatus immutableSnapshot() { + SyncStatus immutableSnapshot({bool setLastSynced = false}) { return SyncStatus( connected: connected, connecting: connecting, @@ -106,7 +106,7 @@ final class MutableSyncStatus { downloadProgress: downloadProgress?.asSyncDownloadProgress, priorityStatusEntries: UnmodifiableListView(priorityStatusEntries), lastSyncedAt: lastSyncedAt, - hasSynced: null, // Stream client is not supposed to set this value. + hasSynced: setLastSynced ? lastSyncedAt != null : null, uploadError: uploadError, downloadError: downloadError, streamSubscriptions: streams, diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index 927f5978..6ca48bb6 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -135,7 +135,7 @@ class StreamingSyncImplementation implements StreamingSync { void updateSubscriptions(List streams) { _activeSubscriptions = streams; if (_nonLineSyncEvents.hasListener) { - _nonLineSyncEvents.add(const AbortCurrentIteration()); + _nonLineSyncEvents.add(HandleChangedSubscriptions(streams)); } } @@ -464,6 +464,7 @@ class StreamingSyncImplementation implements StreamingSync { _state.updateStatus((s) => s.setConnected()); await handleLine(line as StreamingSyncLine); case UploadCompleted(): + case HandleChangedSubscriptions(): // Only relevant for the Rust sync implementation. break; case AbortCurrentIteration(): @@ -613,6 +614,12 @@ final class _ActiveRustStreamingIteration { _ActiveRustStreamingIteration(this.sync); + List _encodeSubscriptions(List subscriptions) { + return sync._activeSubscriptions + .map((s) => {'name': s.name, 'params': s.parameters}) + .toList(); + } + Future syncIteration() async { try { await _control( @@ -621,9 +628,7 @@ final class _ActiveRustStreamingIteration { 'parameters': sync.options.params, 'schema': convert.json.decode(sync.schemaJson), 'include_defaults': sync.options.includeDefaultStreams, - 'active_streams': sync._activeSubscriptions - .map((s) => {'name': s.name, 'params': s.parameters}) - .toList(), + 'active_streams': _encodeSubscriptions(sync._activeSubscriptions), }), ); assert(_completedStream.isCompleted, 'Should have started streaming'); @@ -673,6 +678,9 @@ final class _ActiveRustStreamingIteration { break loop; case TokenRefreshComplete(): await _control('refreshed_token'); + case HandleChangedSubscriptions(:final currentSubscriptions): + await _control('update_subscriptions', + convert.json.encode(_encodeSubscriptions(currentSubscriptions))); } } } @@ -762,3 +770,9 @@ final class TokenRefreshComplete implements SyncEvent { final class AbortCurrentIteration implements SyncEvent { const AbortCurrentIteration(); } + +final class HandleChangedSubscriptions implements SyncEvent { + final List currentSubscriptions; + + HandleChangedSubscriptions(this.currentSubscriptions); +} diff --git a/packages/powersync_core/test/sync/stream_test.dart b/packages/powersync_core/test/sync/stream_test.dart index 9d85769f..04d344ea 100644 --- a/packages/powersync_core/test/sync/stream_test.dart +++ b/packages/powersync_core/test/sync/stream_test.dart @@ -183,4 +183,32 @@ void main() { ), ); }); + + test('changes subscriptions dynamically', () async { + await waitForConnection(); + syncService.addKeepAlive(); + + final subscription = await database.syncStream('a').subscribe(); + syncService.endCurrentListener(); + final request = await syncService.waitForListener; + expect( + json.decode(await request.readAsString()), + containsPair( + 'streams', + containsPair('subscriptions', [ + { + 'stream': 'a', + 'parameters': null, + 'override_priority': null, + }, + ]), + ), + ); + + // Given that the subscription has a TTL, dropping the handle should not + // re-subscribe. + await subscription.unsubscribe(); + await pumpEventQueue(); + expect(syncService.controller.hasListener, isTrue); + }); } From 15b5d195e9a5bd8b3af1a96506b03062652180a8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 13 Aug 2025 12:47:02 +0200 Subject: [PATCH 15/33] Adopt in example --- .../lib/widgets/todo_list_page.dart | 85 +++++++++++++++---- .../native/native_powersync_database.dart | 2 +- .../powersync_core/lib/src/sync/options.dart | 2 + .../powersync_core/lib/src/sync/stream.dart | 20 +++++ .../lib/src/sync/sync_status.dart | 1 + .../lib/src/web/sync_worker.dart | 2 + .../lib/src/web/sync_worker_protocol.dart | 15 +++- 7 files changed, 109 insertions(+), 18 deletions(-) diff --git a/demos/supabase-todolist/lib/widgets/todo_list_page.dart b/demos/supabase-todolist/lib/widgets/todo_list_page.dart index 7e28238e..83c5c286 100644 --- a/demos/supabase-todolist/lib/widgets/todo_list_page.dart +++ b/demos/supabase-todolist/lib/widgets/todo_list_page.dart @@ -1,4 +1,5 @@ import 'package:flutter/material.dart'; +import 'package:powersync/powersync.dart'; import '../powersync.dart'; import './status_app_bar.dart'; @@ -38,30 +39,82 @@ class TodoListPage extends StatelessWidget { } } -class TodoListWidget extends StatelessWidget { +class TodoListWidget extends StatefulWidget { final TodoList list; const TodoListWidget({super.key, required this.list}); + @override + State createState() => _TodoListWidgetState(); +} + +class _TodoListWidgetState extends State { + SyncStreamSubscription? _listSubscription; + + void _subscribe(String listId) { + db + .syncStream('todos', {'list': listId}) + .subscribe(ttl: const Duration(hours: 1)) + .then((sub) { + if (mounted && widget.list.id == listId) { + setState(() { + _listSubscription = sub; + }); + } else { + sub.unsubscribe(); + } + }); + } + + @override + void initState() { + super.initState(); + _subscribe(widget.list.id); + } + + @override + void didUpdateWidget(covariant TodoListWidget oldWidget) { + super.didUpdateWidget(oldWidget); + _subscribe(widget.list.id); + } + + @override + void dispose() { + super.dispose(); + _listSubscription?.unsubscribe(); + } + @override Widget build(BuildContext context) { return StreamBuilder( - stream: TodoList.watchSyncStatus().map((e) => e.hasSynced), - initialData: db.currentStatus.hasSynced, + stream: db.statusStream, + initialData: db.currentStatus, builder: (context, snapshot) { - return StreamBuilder( - stream: list.watchItems(), - builder: (context, snapshot) { - final items = snapshot.data ?? const []; - - return ListView( - padding: const EdgeInsets.symmetric(vertical: 8.0), - children: items.map((todo) { - return TodoItemWidget(todo: todo); - }).toList(), - ); - }, - ); + final hasSynced = switch (_listSubscription) { + null => null, + final sub => snapshot.requireData.statusFor(sub), + } + ?.subscription + .hasSynced ?? + false; + + if (!hasSynced) { + return const CircularProgressIndicator(); + } else { + return StreamBuilder( + stream: widget.list.watchItems(), + builder: (context, snapshot) { + final items = snapshot.data ?? const []; + + return ListView( + padding: const EdgeInsets.symmetric(vertical: 8.0), + children: items.map((todo) { + return TodoItemWidget(todo: todo); + }).toList(), + ); + }, + ); + } }, ); } diff --git a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart index 6c44ad51..40489365 100644 --- a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart @@ -376,7 +376,7 @@ Future _syncIsolate(_PowerSyncDatabaseIsolateArgs args) async { await shutdown(); } else if (action == 'changed_subscriptions') { openedStreamingSync - ?.updateSubscriptions(action[1] as List); + ?.updateSubscriptions(message[1] as List); } } }); diff --git a/packages/powersync_core/lib/src/sync/options.dart b/packages/powersync_core/lib/src/sync/options.dart index e83bb436..48ed293e 100644 --- a/packages/powersync_core/lib/src/sync/options.dart +++ b/packages/powersync_core/lib/src/sync/options.dart @@ -107,6 +107,7 @@ extension type ResolvedSyncOptions(SyncOptions source) { crudThrottleTime: other.crudThrottleTime ?? crudThrottleTime, retryDelay: other.retryDelay ?? retryDelay, params: other.params ?? params, + syncImplementation: other.syncImplementation, includeDefaultStreams: other.includeDefaultStreams ?? includeDefaultStreams, ); @@ -114,6 +115,7 @@ extension type ResolvedSyncOptions(SyncOptions source) { final didChange = !_mapEquality.equals(newOptions.params, params) || newOptions.crudThrottleTime != crudThrottleTime || newOptions.retryDelay != retryDelay || + newOptions.syncImplementation != source.syncImplementation || newOptions.includeDefaultStreams != includeDefaultStreams; return (ResolvedSyncOptions(newOptions), didChange); } diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index a57bfb54..0dd8d9a7 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -144,4 +144,24 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { }, ); } + + Map toJson() { + return { + 'name': name, + 'parameters': parameters, + 'priority': priority.priorityNumber, + 'associated_buckets': associatedBuckets, + 'active': active, + 'is_default': isDefault, + 'has_explicit_subscription': hasExplicitSubscription, + 'expires_at': switch (expiresAt) { + null => null, + final expiresAt => expiresAt.millisecondsSinceEpoch / 1000, + }, + 'last_synced_at': switch (lastSyncedAt) { + null => null, + final lastSyncedAt => lastSyncedAt.millisecondsSinceEpoch / 1000, + } + }; + } } diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index 2013f39a..3665d0f8 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -58,6 +58,7 @@ final class SyncStatus { final List? _internalSubscriptions; + @internal const SyncStatus({ this.connected = false, this.connecting = false, diff --git a/packages/powersync_core/lib/src/web/sync_worker.dart b/packages/powersync_core/lib/src/web/sync_worker.dart index 1aaa5cfa..25dfddaf 100644 --- a/packages/powersync_core/lib/src/web/sync_worker.dart +++ b/packages/powersync_core/lib/src/web/sync_worker.dart @@ -109,6 +109,8 @@ class _ConnectedClient { _runner = null; return (JSObject(), null); case SyncWorkerMessageType.updateSubscriptions: + _runner?.updateClientSubscriptions( + this, (payload as UpdateSubscriptions).toDart); return (JSObject(), null); default: throw StateError('Unexpected message type $type'); diff --git a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart index 88367210..99ca78b8 100644 --- a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart +++ b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart @@ -5,6 +5,7 @@ import 'dart:js_interop'; import 'package:logging/logging.dart'; import 'package:powersync_core/src/schema.dart'; import 'package:powersync_core/src/sync/options.dart'; +import 'package:powersync_core/src/sync/stream.dart'; import 'package:web/web.dart'; import '../connector.dart'; @@ -247,6 +248,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { required String? downloadError, required JSArray? priorityStatusEntries, required JSArray? syncProgress, + required JSString streamSubscriptions, }); factory SerializedSyncStatus.from(SyncStatus status) { @@ -272,6 +274,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { var other => SerializedBucketProgress.serialize( InternalSyncDownloadProgress.ofPublic(other).buckets), }, + streamSubscriptions: json.encode(status.internalSubscriptions).toJS, ); } @@ -285,8 +288,11 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { external String? downloadError; external JSArray? priorityStatusEntries; external JSArray? syncProgress; + external JSString? streamSubscriptions; SyncStatus asSyncStatus() { + final streamSubscriptions = this.streamSubscriptions?.toDart; + return SyncStatus( connected: connected, connecting: connecting, @@ -306,7 +312,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { final syncedMillis = (rawSynced as JSNumber?)?.toDartInt; return ( - priority: BucketPriority((rawPriority as JSNumber).toDartInt), + priority: StreamPriority((rawPriority as JSNumber).toDartInt), lastSyncedAt: syncedMillis != null ? DateTime.fromMicrosecondsSinceEpoch(syncedMillis) : null, @@ -320,6 +326,13 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { SerializedBucketProgress.deserialize(serializedProgress)) .asSyncDownloadProgress, }, + streamSubscriptions: switch (streamSubscriptions) { + null => null, + final serialized => (json.decode(serialized) as List) + .map((e) => CoreActiveStreamSubscription.fromJson( + e as Map)) + .toList(), + }, ); } } From 14ba7da478ca3702b7c5dec17fd35141f5ab3106 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 13 Aug 2025 14:46:37 +0200 Subject: [PATCH 16/33] Update subscription state while offline --- .../lib/src/database/powersync_db_mixin.dart | 22 +----------------- .../lib/src/sync/connection_manager.dart | 23 +++++++++++++++++++ .../powersync_core/test/sync/stream_test.dart | 8 +++++++ 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 93a04a58..21aeb874 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:convert'; import 'package:async/async.dart'; import 'package:logging/logging.dart'; @@ -16,8 +15,6 @@ import 'package:powersync_core/src/schema.dart'; import 'package:powersync_core/src/schema_logic.dart'; import 'package:powersync_core/src/schema_logic.dart' as schema_logic; import 'package:powersync_core/src/sync/connection_manager.dart'; -import 'package:powersync_core/src/sync/instruction.dart'; -import 'package:powersync_core/src/sync/mutable_sync_status.dart'; import 'package:powersync_core/src/sync/options.dart'; import 'package:powersync_core/src/sync/sync_status.dart'; @@ -109,7 +106,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { await _checkVersion(); await database.execute('SELECT powersync_init()'); await updateSchema(schema); - await _updateHasSynced(); + await _connections.resolveOfflineSyncStatus(); } /// Check that a supported version of the powersync extension is loaded. @@ -135,23 +132,6 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { return isInitialized; } - Future> get subscribedStreams { - throw UnimplementedError(); - } - - Future _updateHasSynced() async { - // Query the database to see if any data has been synced. - final row = await database.get( - 'SELECT powersync_offline_sync_status() AS r;', - ); - - final status = CoreSyncStatus.fromJson( - json.decode(row['r'] as String) as Map); - - setStatus((MutableSyncStatus()..applyFromCore(status)) - .immutableSnapshot(setLastSynced: true)); - } - /// Returns a [Future] which will resolve once at least one full sync cycle /// has completed (meaninng that the first consistent checkpoint has been /// reached across all buckets). diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index aeb57691..480e02e2 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -10,6 +10,8 @@ import 'package:powersync_core/src/sync/options.dart'; import 'package:powersync_core/src/sync/stream.dart'; import 'package:powersync_core/src/sync/sync_status.dart'; +import 'instruction.dart'; +import 'mutable_sync_status.dart'; import 'streaming_sync.dart'; /// A (stream name, JSON parameters) pair that uniquely identifies a stream @@ -252,6 +254,15 @@ final class ConnectionManager { 'priority': priority, }, }); + + await _activeGroup.syncMutex.lock(() async { + if (_abortActiveSync == null) { + // Since we're not connected, update the offline sync status to reflect + // the new subscription. + // With a connection, the sync client would include it in its state. + await resolveOfflineSyncStatus(); + } + }); } Future unsubscribeAll({ @@ -266,6 +277,18 @@ final class ConnectionManager { }); } + Future resolveOfflineSyncStatus() async { + final row = await db.database.get( + 'SELECT powersync_offline_sync_status() AS r;', + ); + + final status = CoreSyncStatus.fromJson( + json.decode(row['r'] as String) as Map); + + manuallyChangeSyncStatus((MutableSyncStatus()..applyFromCore(status)) + .immutableSnapshot(setLastSynced: true)); + } + SyncStream syncStream(String name, Map? parameters) { return _SyncStreamImplementation(this, name, parameters); } diff --git a/packages/powersync_core/test/sync/stream_test.dart b/packages/powersync_core/test/sync/stream_test.dart index 04d344ea..0d6d8f59 100644 --- a/packages/powersync_core/test/sync/stream_test.dart +++ b/packages/powersync_core/test/sync/stream_test.dart @@ -211,4 +211,12 @@ void main() { await pumpEventQueue(); expect(syncService.controller.hasListener, isTrue); }); + + test('subscriptions update while offline', () async { + final stream = StreamQueue(database.statusStream); + + final subscription = await database.syncStream('foo').subscribe(); + var status = await stream.next; + expect(status.statusFor(subscription), isNotNull); + }); } From 5b7da6438d75c33089e28e95cb9d0cd34d176612 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 27 Aug 2025 10:44:16 +0200 Subject: [PATCH 17/33] Use new progress information --- .../powersync_core/lib/src/sync/stream.dart | 16 +++++++++--- .../lib/src/sync/sync_status.dart | 25 ++++++------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index 0dd8d9a7..a97f8d29 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -93,7 +93,7 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { @override final Map? parameters; final StreamPriority priority; - final List associatedBuckets; + final ({int total, int downloaded}) progress; @override final bool active; @override @@ -112,7 +112,7 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { required this.name, required this.parameters, required this.priority, - required this.associatedBuckets, + required this.progress, required this.active, required this.isDefault, required this.hasExplicitSubscription, @@ -128,7 +128,7 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { final prio? => StreamPriority(prio), null => StreamPriority.fullSyncPriority, }, - associatedBuckets: (json['associated_buckets'] as List).cast(), + progress: _progressFromJson(json['progress'] as Map), active: json['active'] as bool, isDefault: json['is_default'] as bool, hasExplicitSubscription: json['has_explicit_subscription'] as bool, @@ -150,7 +150,10 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { 'name': name, 'parameters': parameters, 'priority': priority.priorityNumber, - 'associated_buckets': associatedBuckets, + 'progress': { + 'total': progress.total, + 'downloaded': progress.downloaded, + }, 'active': active, 'is_default': isDefault, 'has_explicit_subscription': hasExplicitSubscription, @@ -164,4 +167,9 @@ final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { } }; } + + static ({int total, int downloaded}) _progressFromJson( + Map json) { + return (total: json['total'] as int, downloaded: json['downloaded'] as int); + } } diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index 3665d0f8..b2d2d3ec 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -148,9 +148,9 @@ final class SyncStatus { /// information extracted from the lower priority `2` since each partial sync /// in priority `2` necessarily includes a consistent view over data in /// priority `1`. - SyncPriorityStatus statusForPriority(BucketPriority priority) { + SyncPriorityStatus statusForPriority(StreamPriority priority) { assert(priorityStatusEntries.isSortedByCompare( - (e) => e.priority, BucketPriority.comparator)); + (e) => e.priority, StreamPriority.comparator)); for (final known in priorityStatusEntries) { // Lower-priority buckets are synchronized after higher-priority buckets, @@ -309,7 +309,7 @@ final class InternalSyncDownloadProgress extends ProgressWithOperations { final sinceLast = savedProgress?.sinceLast ?? 0; buckets[bucket.bucket] = ( - priority: BucketPriority._(bucket.priority), + priority: StreamPriority._(bucket.priority), atLast: atLast, sinceLast: sinceLast, targetCount: bucket.count ?? 0, @@ -324,7 +324,7 @@ final class InternalSyncDownloadProgress extends ProgressWithOperations { return InternalSyncDownloadProgress({ for (final bucket in target.checksums) bucket.bucket: ( - priority: BucketPriority(bucket.priority), + priority: StreamPriority(bucket.priority), atLast: 0, sinceLast: 0, targetCount: knownCount, @@ -343,7 +343,7 @@ final class InternalSyncDownloadProgress extends ProgressWithOperations { /// Sums the total target and completed operations for all buckets up until /// the given [priority] (inclusive). - ProgressWithOperations untilPriority(BucketPriority priority) { + ProgressWithOperations untilPriority(StreamPriority priority) { final (total, downloaded) = buckets.values .where((e) => e.priority >= priority) .fold((0, 0), _addProgress); @@ -352,18 +352,7 @@ final class InternalSyncDownloadProgress extends ProgressWithOperations { } ProgressWithOperations _forStream(CoreActiveStreamSubscription subscription) { - final (total, downloaded) = subscription.associatedBuckets.fold( - (0, 0), - (prev, bucket) { - final foundProgress = buckets[bucket]; - if (foundProgress == null) { - return prev; - } - - return _addProgress(prev, foundProgress); - }, - ); - + final (:total, :downloaded) = subscription.progress; return ProgressWithOperations._(total, downloaded); } @@ -475,7 +464,7 @@ extension type SyncDownloadProgress._(InternalSyncDownloadProgress _internal) /// The returned [ProgressWithOperations] tracks the target amount of /// operations that need to be downloaded in total and how many of them have /// already been received. - ProgressWithOperations untilPriority(BucketPriority priority) { + ProgressWithOperations untilPriority(StreamPriority priority) { return _internal.untilPriority(priority); } } From eeb6a471fbea80d2ae739b8349da270d629cb3f9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 1 Sep 2025 15:39:48 +0200 Subject: [PATCH 18/33] Only use sync streams after opt-in --- .../lib/app_config_template.dart | 3 ++ .../lib/widgets/todo_list_page.dart | 47 ++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/demos/supabase-todolist/lib/app_config_template.dart b/demos/supabase-todolist/lib/app_config_template.dart index ccfdfa21..3b39b950 100644 --- a/demos/supabase-todolist/lib/app_config_template.dart +++ b/demos/supabase-todolist/lib/app_config_template.dart @@ -6,4 +6,7 @@ class AppConfig { static const String powersyncUrl = 'https://foo.powersync.journeyapps.com'; static const String supabaseStorageBucket = ''; // Optional. Only required when syncing attachments and using Supabase Storage. See packages/powersync_attachments_helper. + // Whether the PowerSync instance uses sync streams to make fetching todo + // items optional. + static const bool hasSyncStreams = true; } diff --git a/demos/supabase-todolist/lib/widgets/todo_list_page.dart b/demos/supabase-todolist/lib/widgets/todo_list_page.dart index 83c5c286..79af5191 100644 --- a/demos/supabase-todolist/lib/widgets/todo_list_page.dart +++ b/demos/supabase-todolist/lib/widgets/todo_list_page.dart @@ -1,6 +1,7 @@ import 'package:flutter/material.dart'; import 'package:powersync/powersync.dart'; +import '../app_config.dart'; import '../powersync.dart'; import './status_app_bar.dart'; import './todo_item_dialog.dart'; @@ -33,22 +34,54 @@ class TodoListPage extends StatelessWidget { ); return Scaffold( - appBar: StatusAppBar(title: Text(list.name)), - floatingActionButton: button, - body: TodoListWidget(list: list)); + appBar: StatusAppBar(title: Text(list.name)), + floatingActionButton: button, + body: AppConfig.hasSyncStreams + ? _SyncStreamTodoListWidget(list: list) + : TodoListWidget(list: list), + ); } } -class TodoListWidget extends StatefulWidget { +class TodoListWidget extends StatelessWidget { final TodoList list; const TodoListWidget({super.key, required this.list}); @override - State createState() => _TodoListWidgetState(); + Widget build(BuildContext context) { + return StreamBuilder( + stream: TodoList.watchSyncStatus().map((e) => e.hasSynced), + initialData: db.currentStatus.hasSynced, + builder: (context, snapshot) { + return StreamBuilder( + stream: list.watchItems(), + builder: (context, snapshot) { + final items = snapshot.data ?? const []; + + return ListView( + padding: const EdgeInsets.symmetric(vertical: 8.0), + children: items.map((todo) { + return TodoItemWidget(todo: todo); + }).toList(), + ); + }, + ); + }, + ); + } +} + +class _SyncStreamTodoListWidget extends StatefulWidget { + final TodoList list; + + const _SyncStreamTodoListWidget({required this.list}); + + @override + State<_SyncStreamTodoListWidget> createState() => _SyncStreamTodosState(); } -class _TodoListWidgetState extends State { +class _SyncStreamTodosState extends State<_SyncStreamTodoListWidget> { SyncStreamSubscription? _listSubscription; void _subscribe(String listId) { @@ -73,7 +106,7 @@ class _TodoListWidgetState extends State { } @override - void didUpdateWidget(covariant TodoListWidget oldWidget) { + void didUpdateWidget(covariant _SyncStreamTodoListWidget oldWidget) { super.didUpdateWidget(oldWidget); _subscribe(widget.list.id); } From 118de7736898e532123189bb90cf4fe645b9785b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 1 Sep 2025 15:52:09 +0200 Subject: [PATCH 19/33] Fix tests --- .../lib/powersync/powersync.dart | 7 ++++--- demos/supabase-todolist-drift/lib/screens/lists.dart | 2 +- demos/supabase-todolist/lib/widgets/guard_by_sync.dart | 4 ++-- demos/supabase-todolist/lib/widgets/lists_page.dart | 2 +- packages/powersync_core/lib/src/sync/instruction.dart | 4 ++-- .../powersync_core/lib/src/sync/streaming_sync.dart | 2 +- .../lib/src/web/sync_worker_protocol.dart | 2 +- .../powersync_core/test/sync/in_memory_sync_test.dart | 2 +- packages/powersync_core/test/sync/sync_types_test.dart | 10 +++++----- 9 files changed, 18 insertions(+), 17 deletions(-) diff --git a/demos/supabase-todolist-drift/lib/powersync/powersync.dart b/demos/supabase-todolist-drift/lib/powersync/powersync.dart index cfd73336..cb327bfd 100644 --- a/demos/supabase-todolist-drift/lib/powersync/powersync.dart +++ b/demos/supabase-todolist-drift/lib/powersync/powersync.dart @@ -47,18 +47,19 @@ Future powerSyncInstance(Ref ref) async { return db; } -final _syncStatusInternal = StreamProvider((ref) { +final _syncStatusInternal = StreamProvider((ref) { return Stream.fromFuture( ref.watch(powerSyncInstanceProvider.future), - ).asyncExpand((db) => db.statusStream).startWith(const SyncStatus()); + ).asyncExpand((db) => db.statusStream).startWith(null); }); final syncStatus = Provider((ref) { + // ignore: invalid_use_of_internal_member return ref.watch(_syncStatusInternal).value ?? const SyncStatus(); }); @riverpod -bool didCompleteSync(Ref ref, [BucketPriority? priority]) { +bool didCompleteSync(Ref ref, [StreamPriority? priority]) { final status = ref.watch(syncStatus); if (priority != null) { return status.statusForPriority(priority).hasSynced ?? false; diff --git a/demos/supabase-todolist-drift/lib/screens/lists.dart b/demos/supabase-todolist-drift/lib/screens/lists.dart index ebaa0857..c995a275 100644 --- a/demos/supabase-todolist-drift/lib/screens/lists.dart +++ b/demos/supabase-todolist-drift/lib/screens/lists.dart @@ -35,7 +35,7 @@ final class _ListsWidget extends ConsumerWidget { @override Widget build(BuildContext context, WidgetRef ref) { final lists = ref.watch(listsNotifierProvider); - final didSync = ref.watch(didCompleteSyncProvider(BucketPriority(1))); + final didSync = ref.watch(didCompleteSyncProvider(StreamPriority(1))); if (!didSync) { return const Text('Busy with sync...'); diff --git a/demos/supabase-todolist/lib/widgets/guard_by_sync.dart b/demos/supabase-todolist/lib/widgets/guard_by_sync.dart index d55ed4e3..6d4d12b9 100644 --- a/demos/supabase-todolist/lib/widgets/guard_by_sync.dart +++ b/demos/supabase-todolist/lib/widgets/guard_by_sync.dart @@ -7,9 +7,9 @@ import 'package:powersync_flutter_demo/powersync.dart'; class GuardBySync extends StatelessWidget { final Widget child; - /// When set, wait only for a complete sync within the [BucketPriority] + /// When set, wait only for a complete sync within the [StreamPriority] /// instead of a full sync. - final BucketPriority? priority; + final StreamPriority? priority; const GuardBySync({ super.key, diff --git a/demos/supabase-todolist/lib/widgets/lists_page.dart b/demos/supabase-todolist/lib/widgets/lists_page.dart index c41aabbe..e883a445 100644 --- a/demos/supabase-todolist/lib/widgets/lists_page.dart +++ b/demos/supabase-todolist/lib/widgets/lists_page.dart @@ -66,5 +66,5 @@ class ListsWidget extends StatelessWidget { ); } - static final _listsPriority = BucketPriority(1); + static final _listsPriority = StreamPriority(1); } diff --git a/packages/powersync_core/lib/src/sync/instruction.dart b/packages/powersync_core/lib/src/sync/instruction.dart index cde81303..462f784e 100644 --- a/packages/powersync_core/lib/src/sync/instruction.dart +++ b/packages/powersync_core/lib/src/sync/instruction.dart @@ -94,7 +94,7 @@ final class CoreSyncStatus { static SyncPriorityStatus _priorityStatusFromJson(Map json) { return ( - priority: BucketPriority(json['priority'] as int), + priority: StreamPriority(json['priority'] as int), hasSynced: json['has_synced'] as bool?, lastSyncedAt: switch (json['last_synced_at']) { null => null, @@ -123,7 +123,7 @@ final class DownloadProgress { static BucketProgress _bucketProgressFromJson(Map json) { return ( - priority: BucketPriority(json['priority'] as int), + priority: StreamPriority(json['priority'] as int), atLast: json['at_last'] as int, sinceLast: json['since_last'] as int, targetCount: json['target_count'] as int, diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index 6ca48bb6..933fb0a1 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -385,7 +385,7 @@ class StreamingSyncImplementation implements StreamingSync { // checkpoint later. } else { _updateStatusForPriority(( - priority: BucketPriority(bucketPriority), + priority: StreamPriority(bucketPriority), lastSyncedAt: DateTime.now(), hasSynced: true, )); diff --git a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart index 99ca78b8..950cd1d6 100644 --- a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart +++ b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart @@ -226,7 +226,7 @@ extension type SerializedBucketProgress._(JSObject _) implements JSObject { return { for (final entry in array.toDart) entry.name: ( - priority: BucketPriority(entry.priority), + priority: StreamPriority(entry.priority), atLast: entry.atLast, sinceLast: entry.sinceLast, targetCount: entry.targetCount, diff --git a/packages/powersync_core/test/sync/in_memory_sync_test.dart b/packages/powersync_core/test/sync/in_memory_sync_test.dart index 0b478e3c..87c18d82 100644 --- a/packages/powersync_core/test/sync/in_memory_sync_test.dart +++ b/packages/powersync_core/test/sync/in_memory_sync_test.dart @@ -618,7 +618,7 @@ void _declareTests(String name, SyncOptions options, bool bson) { Future expectProgress( StreamQueue status, { required Object total, - Map priorities = const {}, + Map priorities = const {}, }) async { await expectLater( status, diff --git a/packages/powersync_core/test/sync/sync_types_test.dart b/packages/powersync_core/test/sync/sync_types_test.dart index 261152b2..5cd24c9d 100644 --- a/packages/powersync_core/test/sync/sync_types_test.dart +++ b/packages/powersync_core/test/sync/sync_types_test.dart @@ -216,11 +216,11 @@ void main() { } }); - test('bucket priority comparisons', () { - expect(BucketPriority(0) < BucketPriority(3), isFalse); - expect(BucketPriority(0) > BucketPriority(3), isTrue); - expect(BucketPriority(0) >= BucketPriority(3), isTrue); - expect(BucketPriority(0) >= BucketPriority(0), isTrue); + test('stream priority comparisons', () { + expect(StreamPriority(0) < StreamPriority(3), isFalse); + expect(StreamPriority(0) > StreamPriority(3), isTrue); + expect(StreamPriority(0) >= StreamPriority(3), isTrue); + expect(StreamPriority(0) >= StreamPriority(0), isTrue); }); }); } From 3486cb6875b109039325d525f6ad03cae805da79 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 1 Sep 2025 16:01:26 +0200 Subject: [PATCH 20/33] Raise min collection dependency --- packages/powersync_core/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/powersync_core/pubspec.yaml b/packages/powersync_core/pubspec.yaml index fa3dd02a..929b2d5e 100644 --- a/packages/powersync_core/pubspec.yaml +++ b/packages/powersync_core/pubspec.yaml @@ -19,7 +19,7 @@ dependencies: uuid: ^4.2.0 async: ^2.10.0 logging: ^1.1.1 - collection: ^1.17.0 + collection: ^1.19.0 web: ^1.0.0 # Only used internally to download WASM / worker files. From eec0a0447a648cfbadeca926d4df652ebfcadb74 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 1 Sep 2025 18:15:40 +0200 Subject: [PATCH 21/33] More worker debugging --- demos/benchmarks/pubspec.lock | 44 +++++++++---------- .../lib/app_config_template.dart | 2 +- .../lib/widgets/todo_list_page.dart | 2 +- .../lib/src/sync/connection_manager.dart | 2 +- .../lib/src/sync/instruction.dart | 7 ++- .../lib/src/sync/streaming_sync.dart | 38 ++++++++++++---- .../lib/src/web/sync_worker.dart | 3 ++ 7 files changed, 62 insertions(+), 36 deletions(-) diff --git a/demos/benchmarks/pubspec.lock b/demos/benchmarks/pubspec.lock index 026996f9..b45942df 100644 --- a/demos/benchmarks/pubspec.lock +++ b/demos/benchmarks/pubspec.lock @@ -111,10 +111,10 @@ packages: dependency: "direct main" description: name: http - sha256: "2c11f3f94c687ee9bad77c171151672986360b2b001d109814ee7140b2cf261b" + sha256: bb2ce4590bc2667c96f318d68cac1b5a7987ec819351d32b1c987239a815e007 url: "https://pub.dev" source: hosted - version: "1.4.0" + version: "1.5.0" http_parser: dependency: transitive description: @@ -135,26 +135,26 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "6bb818ecbdffe216e81182c2f0714a2e62b593f4a4f13098713ff1685dfb6ab0" + sha256: "8dcda04c3fc16c14f48a7bb586d4be1f0d1572731b6d81d51772ef47c02081e0" url: "https://pub.dev" source: hosted - version: "10.0.9" + version: "11.0.1" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: f8b613e7e6a13ec79cfdc0e97638fddb3ab848452eff057653abd3edba760573 + sha256: "1dbc140bb5a23c75ea9c4811222756104fbcd1a27173f0c34ca01e16bea473c1" url: "https://pub.dev" source: hosted - version: "3.0.9" + version: "3.0.10" leak_tracker_testing: dependency: transitive description: name: leak_tracker_testing - sha256: "6ba465d5d76e67ddf503e1161d1f4a6bc42306f9d66ca1e8f079a47290fb06d3" + sha256: "8d5a2d49f4a66b49744b23b018848400d23e54caf9463f4eb20df3eb8acb2eb1" url: "https://pub.dev" source: hosted - version: "3.0.1" + version: "3.0.2" lints: dependency: transitive description: @@ -281,21 +281,21 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.15.0" + version: "1.15.2" powersync_core: dependency: "direct overridden" description: path: "../../packages/powersync_core" relative: true source: path - version: "1.5.0" + version: "1.5.2" powersync_flutter_libs: dependency: "direct overridden" description: path: "../../packages/powersync_flutter_libs" relative: true source: path - version: "0.4.10" + version: "0.4.11" pub_semver: dependency: transitive description: @@ -337,10 +337,10 @@ packages: dependency: transitive description: name: sqlite3 - sha256: "310af39c40dd0bb2058538333c9d9840a2725ae0b9f77e4fd09ad6696aa8f66e" + sha256: f393d92c71bdcc118d6203d07c991b9be0f84b1a6f89dd4f7eed348131329924 url: "https://pub.dev" source: hosted - version: "2.7.5" + version: "2.9.0" sqlite3_flutter_libs: dependency: transitive description: @@ -353,18 +353,18 @@ packages: dependency: transitive description: name: sqlite3_web - sha256: "967e076442f7e1233bd7241ca61f3efe4c7fc168dac0f38411bdb3bdf471eb3c" + sha256: "0f6ebcb4992d1892ac5c8b5ecd22a458ab9c5eb6428b11ae5ecb5d63545844da" url: "https://pub.dev" source: hosted - version: "0.3.1" + version: "0.3.2" sqlite_async: dependency: "direct main" description: name: sqlite_async - sha256: a60e8d5c8df8e694933bd5a312c38393e79ad77d784bb91c6f38ba627bfb7aec + sha256: "6116bfc6aef6ce77730b478385ba4a58873df45721f6a9bc6ffabf39b6576e36" url: "https://pub.dev" source: hosted - version: "0.11.4" + version: "0.12.1" stack_trace: dependency: transitive description: @@ -401,10 +401,10 @@ packages: dependency: transitive description: name: test_api - sha256: fb31f383e2ee25fbbfe06b40fe21e1e458d14080e3c67e7ba0acfde4df4e0bbd + sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00" url: "https://pub.dev" source: hosted - version: "0.7.4" + version: "0.7.6" typed_data: dependency: transitive description: @@ -433,10 +433,10 @@ packages: dependency: transitive description: name: vector_math - sha256: "80b3257d1492ce4d091729e3a67a60407d227c27241d6927be0130c98e741803" + sha256: d530bd74fea330e6e364cda7a85019c434070188383e1cd8d9777ee586914c5b url: "https://pub.dev" source: hosted - version: "2.1.4" + version: "2.2.0" vm_service: dependency: transitive description: @@ -470,5 +470,5 @@ packages: source: hosted version: "3.1.3" sdks: - dart: ">=3.7.0 <4.0.0" + dart: ">=3.8.0-0 <4.0.0" flutter: ">=3.27.0" diff --git a/demos/supabase-todolist/lib/app_config_template.dart b/demos/supabase-todolist/lib/app_config_template.dart index 3b39b950..05ea5164 100644 --- a/demos/supabase-todolist/lib/app_config_template.dart +++ b/demos/supabase-todolist/lib/app_config_template.dart @@ -8,5 +8,5 @@ class AppConfig { ''; // Optional. Only required when syncing attachments and using Supabase Storage. See packages/powersync_attachments_helper. // Whether the PowerSync instance uses sync streams to make fetching todo // items optional. - static const bool hasSyncStreams = true; + static const bool hasSyncStreams = false; } diff --git a/demos/supabase-todolist/lib/widgets/todo_list_page.dart b/demos/supabase-todolist/lib/widgets/todo_list_page.dart index 79af5191..04349245 100644 --- a/demos/supabase-todolist/lib/widgets/todo_list_page.dart +++ b/demos/supabase-todolist/lib/widgets/todo_list_page.dart @@ -132,7 +132,7 @@ class _SyncStreamTodosState extends State<_SyncStreamTodoListWidget> { false; if (!hasSynced) { - return const CircularProgressIndicator(); + return const Center(child: CircularProgressIndicator()); } else { return StreamBuilder( stream: widget.list.watchItems(), diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 480e02e2..77249c02 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -255,7 +255,7 @@ final class ConnectionManager { }, }); - await _activeGroup.syncMutex.lock(() async { + await _activeGroup.syncConnectMutex.lock(() async { if (_abortActiveSync == null) { // Since we're not connected, update the offline sync status to reflect // the new subscription. diff --git a/packages/powersync_core/lib/src/sync/instruction.dart b/packages/powersync_core/lib/src/sync/instruction.dart index 462f784e..3479e281 100644 --- a/packages/powersync_core/lib/src/sync/instruction.dart +++ b/packages/powersync_core/lib/src/sync/instruction.dart @@ -14,7 +14,8 @@ sealed class Instruction { EstablishSyncStream.fromJson(establish as Map), {'FetchCredentials': final creds} => FetchCredentials.fromJson(creds as Map), - {'CloseSyncStream': _} => const CloseSyncStream(), + {'CloseSyncStream': final closeOptions as Map} => + CloseSyncStream(closeOptions['hide_disconnect'] as bool), {'FlushFileSystem': _} => const FlushFileSystem(), {'DidCompleteSync': _} => const DidCompleteSync(), _ => UnknownSyncInstruction(json) @@ -142,7 +143,9 @@ final class FetchCredentials implements Instruction { } final class CloseSyncStream implements Instruction { - const CloseSyncStream(); + final bool hideDisconnect; + + const CloseSyncStream(this.hideDisconnect); } final class FlushFileSystem implements Instruction { diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index 933fb0a1..e2880437 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -309,7 +309,12 @@ class StreamingSyncImplementation implements StreamingSync { } Future _rustStreamingSyncIteration() async { - await _ActiveRustStreamingIteration(this).syncIteration(); + logger.info('Starting Rust sync iteration'); + final response = await _ActiveRustStreamingIteration(this).syncIteration(); + logger.info( + 'Ending Rust sync iteration. Immediate restart: ${response.immediateRestart}'); + // Note: With the current loop in streamingSync(), any return value that + // isn't an exception triggers an immediate restart. } Future<(List, Map)> @@ -610,7 +615,7 @@ final class _ActiveRustStreamingIteration { var _hadSyncLine = false; StreamSubscription? _completedUploads; - final Completer _completedStream = Completer(); + final Completer _completedStream = Completer(); _ActiveRustStreamingIteration(this.sync); @@ -620,7 +625,7 @@ final class _ActiveRustStreamingIteration { .toList(); } - Future syncIteration() async { + Future syncIteration() async { try { await _control( 'start', @@ -632,7 +637,7 @@ final class _ActiveRustStreamingIteration { }), ); assert(_completedStream.isCompleted, 'Should have started streaming'); - await _completedStream.future; + return await _completedStream.future; } finally { _isActive = false; _completedUploads?.cancel(); @@ -655,10 +660,12 @@ final class _ActiveRustStreamingIteration { }).map(ReceivedLine.new); } - Future _handleLines(EstablishSyncStream request) async { + Future _handleLines( + EstablishSyncStream request) async { final events = addBroadcast( _receiveLines(request.request), sync._nonLineSyncEvents.stream); + var needsImmediateRestart = false; loop: await for (final event in events) { if (!_isActive || sync.aborted) { @@ -674,7 +681,8 @@ final class _ActiveRustStreamingIteration { await _control('line_text', line); case UploadCompleted(): await _control('completed_upload'); - case AbortCurrentIteration(): + case AbortCurrentIteration(:final hideDisconnectState): + needsImmediateRestart = hideDisconnectState; break loop; case TokenRefreshComplete(): await _control('refreshed_token'); @@ -683,6 +691,8 @@ final class _ActiveRustStreamingIteration { convert.json.encode(_encodeSubscriptions(currentSubscriptions))); } } + + return (immediateRestart: needsImmediateRestart); } /// Triggers a local CRUD upload when the first sync line has been received. @@ -736,10 +746,11 @@ final class _ActiveRustStreamingIteration { sync.logger.warning('Could not prefetch credentials', e, s); }); } - case CloseSyncStream(): + case CloseSyncStream(:final hideDisconnect): if (!sync.aborted) { _isActive = false; - sync._nonLineSyncEvents.add(const AbortCurrentIteration()); + sync._nonLineSyncEvents + .add(AbortCurrentIteration(hideDisconnectState: hideDisconnect)); } case FlushFileSystem(): await sync.adapter.flushFileSystem(); @@ -751,6 +762,8 @@ final class _ActiveRustStreamingIteration { } } +typedef RustSyncIterationResult = ({bool immediateRestart}); + sealed class SyncEvent {} final class ReceivedLine implements SyncEvent { @@ -768,7 +781,14 @@ final class TokenRefreshComplete implements SyncEvent { } final class AbortCurrentIteration implements SyncEvent { - const AbortCurrentIteration(); + /// Whether we should immediately disconnect and hide the `disconnected` + /// state. + /// + /// This is used when we're changing subscription, to hide the brief downtime + /// we have while reconnecting. + final bool hideDisconnectState; + + const AbortCurrentIteration({this.hideDisconnectState = false}); } final class HandleChangedSubscriptions implements SyncEvent { diff --git a/packages/powersync_core/lib/src/web/sync_worker.dart b/packages/powersync_core/lib/src/web/sync_worker.dart index 25dfddaf..1c92808f 100644 --- a/packages/powersync_core/lib/src/web/sync_worker.dart +++ b/packages/powersync_core/lib/src/web/sync_worker.dart @@ -229,6 +229,8 @@ class _SyncRunner { final before = currentStreams.toSet(); final after = connections.values.flattenedToSet; if (!const SetEquality().equals(before, after)) { + _logger.info( + 'Subscriptions across tabs have changed, checking whether a reconnect is necessary'); currentStreams = after.toList(); sync?.updateSubscriptions(currentStreams); } @@ -320,6 +322,7 @@ class _SyncRunner { client: BrowserClient(), identifier: identifier, activeSubscriptions: currentStreams, + logger: _logger, ); sync!.statusStream.listen((event) { _logger.fine('Broadcasting sync event: $event'); From d00f7815ae5d02f4f877e71a85d135f9c695ffc9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 2 Sep 2025 11:56:44 +0200 Subject: [PATCH 22/33] Web: Fix reconnect --- .../lib/src/sync/streaming_sync.dart | 89 +++++++++++++------ 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index e2880437..25426abf 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -528,7 +528,8 @@ class StreamingSyncImplementation implements StreamingSync { } Future _postStreamRequest( - Object? data, bool acceptBson) async { + Object? data, bool acceptBson, + {Future? onAbort}) async { const ndJson = 'application/x-ndjson'; const bson = 'application/vnd.powersync.bson-stream'; @@ -538,8 +539,8 @@ class StreamingSyncImplementation implements StreamingSync { } final uri = credentials.endpointUri('sync/stream'); - final request = - http.AbortableRequest('POST', uri, abortTrigger: _abort!.onAbort); + final request = http.AbortableRequest('POST', uri, + abortTrigger: onAbort ?? _abort!.onAbort); request.headers['Content-Type'] = 'application/json'; request.headers['Authorization'] = "Token ${credentials.token}"; request.headers['Accept'] = @@ -645,9 +646,10 @@ final class _ActiveRustStreamingIteration { } } - Stream _receiveLines(Object? data) { + Stream _receiveLines(Object? data, + {required Future onAbort}) { return streamFromFutureAwaitInCancellation( - sync._postStreamRequest(data, true)) + sync._postStreamRequest(data, true, onAbort: onAbort)) .asyncExpand((response) { if (response == null) { return null; @@ -662,33 +664,66 @@ final class _ActiveRustStreamingIteration { Future _handleLines( EstablishSyncStream request) async { + // This is a workaround for https://github.com/dart-lang/http/issues/1820: + // When cancelling the stream subscription of an HTTP response with the + // fetch-based client implementation, cancelling the subscription is delayed + // until the next chunk (typically a token_expires_in message in our case). + // So, before cancelling, we complete an abort controller for the request to + // speed things up. This is not an issue in most cases because the abort + // controller on this stream would be completed when disconnecting. But + // when switching sync streams, that's not the case and we need a second + // abort controller for the inner iteration. + final innerAbort = Completer.sync(); final events = addBroadcast( - _receiveLines(request.request), sync._nonLineSyncEvents.stream); + _receiveLines( + request.request, + onAbort: Future.any([ + sync._abort!.onAbort, + innerAbort.future, + ]), + ), + sync._nonLineSyncEvents.stream, + ); var needsImmediateRestart = false; loop: - await for (final event in events) { - if (!_isActive || sync.aborted) { - break; - } + try { + await for (final event in events) { + if (!_isActive || sync.aborted) { + innerAbort.complete(); + break; + } - switch (event) { - case ReceivedLine(line: final Uint8List line): - _triggerCrudUploadOnFirstLine(); - await _control('line_binary', line); - case ReceivedLine(line: final line as String): - _triggerCrudUploadOnFirstLine(); - await _control('line_text', line); - case UploadCompleted(): - await _control('completed_upload'); - case AbortCurrentIteration(:final hideDisconnectState): - needsImmediateRestart = hideDisconnectState; - break loop; - case TokenRefreshComplete(): - await _control('refreshed_token'); - case HandleChangedSubscriptions(:final currentSubscriptions): - await _control('update_subscriptions', - convert.json.encode(_encodeSubscriptions(currentSubscriptions))); + switch (event) { + case ReceivedLine(line: final Uint8List line): + _triggerCrudUploadOnFirstLine(); + await _control('line_binary', line); + case ReceivedLine(line: final line as String): + _triggerCrudUploadOnFirstLine(); + await _control('line_text', line); + case UploadCompleted(): + await _control('completed_upload'); + case AbortCurrentIteration(:final hideDisconnectState): + innerAbort.complete(); + needsImmediateRestart = hideDisconnectState; + break loop; + case TokenRefreshComplete(): + await _control('refreshed_token'); + case HandleChangedSubscriptions(:final currentSubscriptions): + await _control( + 'update_subscriptions', + convert.json + .encode(_encodeSubscriptions(currentSubscriptions))); + } + } + } on http.RequestAbortedException { + // Unlike a regular cancellation, cancelling via the abort controller + // emits an error. We did mean to just cancel the stream, so we can + // safely ignore that. + if (innerAbort.isCompleted) { + // ignore + } else { + rethrow; } } From aef99141050f2c89882b5e6ab0bb16db155c3abf Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 2 Sep 2025 12:19:59 +0200 Subject: [PATCH 23/33] Document TestDatabase --- packages/powersync_core/test/utils/abstract_test_utils.dart | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/powersync_core/test/utils/abstract_test_utils.dart b/packages/powersync_core/test/utils/abstract_test_utils.dart index 4a97c0a9..be5b420a 100644 --- a/packages/powersync_core/test/utils/abstract_test_utils.dart +++ b/packages/powersync_core/test/utils/abstract_test_utils.dart @@ -154,6 +154,11 @@ class TestConnector extends PowerSyncBackendConnector { } } +/// A [PowerSyncDatabase] implemented by a single in-memory database connection +/// and a mock-HTTP sync client. +/// +/// This ensures tests for sync cover the `ConnectionManager` and other methods +/// exposed by the mixin. final class TestDatabase with SqliteQueries, PowerSyncDatabaseMixin implements PowerSyncDatabase { From 1f1dbdbe12cac3d88fa493db65a01621ed5295d2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 3 Sep 2025 12:07:04 +0200 Subject: [PATCH 24/33] Rename to SyncSubscriptionDescription --- packages/powersync_core/lib/src/sync/stream.dart | 5 +++-- packages/powersync_core/lib/src/sync/sync_status.dart | 2 +- packages/powersync_core/test/sync/utils.dart | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index a97f8d29..4eeb2ca5 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -21,7 +21,7 @@ abstract interface class SyncStreamDescription { /// /// This includes the [SyncStreamDescription] along with information about the /// current sync status. -abstract interface class SyncSubscriptionDefinition +abstract interface class SyncSubscriptionDescription extends SyncStreamDescription { /// Whether this stream is active, meaning that the subscription has been /// acknownledged by the sync serivce. @@ -87,7 +87,8 @@ abstract interface class SyncStreamSubscription /// An `ActiveStreamSubscription` as part of the sync status in Rust. @internal -final class CoreActiveStreamSubscription implements SyncSubscriptionDefinition { +final class CoreActiveStreamSubscription + implements SyncSubscriptionDescription { @override final String name; @override diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index b2d2d3ec..be38c5fe 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -219,7 +219,7 @@ final class SyncStreamStatus { final ProgressWithOperations? progress; final CoreActiveStreamSubscription _internal; - SyncSubscriptionDefinition get subscription => _internal; + SyncSubscriptionDescription get subscription => _internal; StreamPriority get priority => _internal.priority; bool get isDefault => _internal.isDefault; diff --git a/packages/powersync_core/test/sync/utils.dart b/packages/powersync_core/test/sync/utils.dart index 0a619514..19570022 100644 --- a/packages/powersync_core/test/sync/utils.dart +++ b/packages/powersync_core/test/sync/utils.dart @@ -72,11 +72,11 @@ TypeMatcher isStreamStatus({ return matcher; } -TypeMatcher isSyncSubscription({ +TypeMatcher isSyncSubscription({ required Object name, required Object? parameters, }) { - return isA() + return isA() .having((e) => e.name, 'name', name) .having((e) => e.parameters, 'parameters', parameters); } From f92239186242d30f2cf2f3b2ddf306c564c4fd2b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 3 Sep 2025 14:05:27 +0200 Subject: [PATCH 25/33] Misc API improvements --- .../lib/src/sync/connection_manager.dart | 2 +- packages/powersync_core/lib/src/sync/stream.dart | 10 +++++----- .../powersync_core/lib/src/sync/sync_status.dart | 8 ++------ packages/powersync_core/test/sync/stream_test.dart | 4 ++-- packages/powersync_core/test/sync/utils.dart | 13 ++++++++----- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 77249c02..104be32a 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -372,7 +372,7 @@ final class _SyncStreamSubscriptionHandle implements SyncStreamSubscription { Map? get parameters => _source.parameters; @override - Future unsubscribe() async { + void unsubscribe() async { _finalizer.detach(this); _source.decrementRefCount(); } diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index 4eeb2ca5..c1fccb26 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -77,12 +77,12 @@ abstract interface class SyncStreamSubscription /// this stream subscription. Future waitForFirstSync(); - /// Removes this stream subscription from the database, if it has been - /// subscribed to explicitly. + /// Removes this subscription. /// - /// The subscription may still be included for a while, until the client - /// reconnects and receives new snapshots from the sync service. - Future unsubscribe(); + /// Once all [SyncStreamSubscription]s for a [SyncStream] have been + /// unsubscribed, the `ttl` for that stream starts running. When it expires + /// without subscribing again, the stream will be evicted. + void unsubscribe(); } /// An `ActiveStreamSubscription` as part of the sync status in Rust. diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index be38c5fe..f3badb92 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -2,7 +2,6 @@ import 'dart:math'; import 'package:collection/collection.dart'; import 'package:meta/meta.dart'; -import '../database/powersync_database.dart'; import 'bucket_storage.dart'; import 'protocol.dart'; @@ -120,10 +119,8 @@ final class SyncStatus { /// All sync streams currently being tracked in this subscription. /// - /// This returns null when the sync stream is currently being opened and we - /// don't have reliable information about all included streams yet (in that - /// state, [PowerSyncDatabase.subscribedStreams] can still be used to - /// resolve known subscriptions locally). + /// This returns null when the database is currently being opened and we + /// don't have reliable information about all included streams yet. Iterable? get activeSubscriptions { return _internalSubscriptions?.map((subscription) { return SyncStreamStatus._(subscription, downloadProgress); @@ -221,7 +218,6 @@ final class SyncStreamStatus { SyncSubscriptionDescription get subscription => _internal; StreamPriority get priority => _internal.priority; - bool get isDefault => _internal.isDefault; SyncStreamStatus._(this._internal, SyncDownloadProgress? progress) : progress = progress?._internal._forStream(_internal); diff --git a/packages/powersync_core/test/sync/stream_test.dart b/packages/powersync_core/test/sync/stream_test.dart index 0d6d8f59..41786441 100644 --- a/packages/powersync_core/test/sync/stream_test.dart +++ b/packages/powersync_core/test/sync/stream_test.dart @@ -175,8 +175,8 @@ void main() { subscription: isSyncSubscription( name: 'default_stream', parameters: null, + isDefault: true, ), - isDefault: true, ), ], ), @@ -207,7 +207,7 @@ void main() { // Given that the subscription has a TTL, dropping the handle should not // re-subscribe. - await subscription.unsubscribe(); + subscription.unsubscribe(); await pumpEventQueue(); expect(syncService.controller.hasListener, isTrue); }); diff --git a/packages/powersync_core/test/sync/utils.dart b/packages/powersync_core/test/sync/utils.dart index 19570022..2de3660e 100644 --- a/packages/powersync_core/test/sync/utils.dart +++ b/packages/powersync_core/test/sync/utils.dart @@ -58,16 +58,12 @@ TypeMatcher progress(int completed, int total) { TypeMatcher isStreamStatus({ required Object? subscription, Object? progress, - Object? isDefault, }) { var matcher = isA() .having((e) => e.subscription, 'subscription', subscription); if (progress case final progress?) { matcher = matcher.having((e) => e.progress, 'progress', progress); } - if (isDefault case final isDefault?) { - matcher = matcher.having((e) => e.isDefault, 'isDefault', isDefault); - } return matcher; } @@ -75,10 +71,17 @@ TypeMatcher isStreamStatus({ TypeMatcher isSyncSubscription({ required Object name, required Object? parameters, + bool? isDefault, }) { - return isA() + var matcher = isA() .having((e) => e.name, 'name', name) .having((e) => e.parameters, 'parameters', parameters); + + if (isDefault != null) { + matcher = matcher.having((e) => e.isDefault, 'isDefault', isDefault); + } + + return matcher; } BucketChecksum checksum( From 2b6ba006a60a0a9849d9338323fe6924ac30ba09 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 3 Sep 2025 14:06:20 +0200 Subject: [PATCH 26/33] activeSubscriptions -> subscriptions --- packages/powersync_core/lib/src/sync/sync_status.dart | 2 +- packages/powersync_core/test/sync/stream_test.dart | 2 +- packages/powersync_core/test/sync/utils.dart | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index f3badb92..57177392 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -121,7 +121,7 @@ final class SyncStatus { /// /// This returns null when the database is currently being opened and we /// don't have reliable information about all included streams yet. - Iterable? get activeSubscriptions { + Iterable? get subscriptions { return _internalSubscriptions?.map((subscription) { return SyncStreamStatus._(subscription, downloadProgress); }); diff --git a/packages/powersync_core/test/sync/stream_test.dart b/packages/powersync_core/test/sync/stream_test.dart index 41786441..7aa88fb4 100644 --- a/packages/powersync_core/test/sync/stream_test.dart +++ b/packages/powersync_core/test/sync/stream_test.dart @@ -170,7 +170,7 @@ void main() { status, emits( isSyncStatus( - activeSubscriptions: [ + subscriptions: [ isStreamStatus( subscription: isSyncSubscription( name: 'default_stream', diff --git a/packages/powersync_core/test/sync/utils.dart b/packages/powersync_core/test/sync/utils.dart index 2de3660e..f4767fb2 100644 --- a/packages/powersync_core/test/sync/utils.dart +++ b/packages/powersync_core/test/sync/utils.dart @@ -8,7 +8,7 @@ TypeMatcher isSyncStatus({ Object? connecting, Object? hasSynced, Object? downloadProgress, - Object? activeSubscriptions, + Object? subscriptions, }) { var matcher = isA(); if (downloading != null) { @@ -27,9 +27,9 @@ TypeMatcher isSyncStatus({ matcher = matcher.having( (e) => e.downloadProgress, 'downloadProgress', downloadProgress); } - if (activeSubscriptions != null) { - matcher = matcher.having((e) => e.activeSubscriptions, - 'activeSubscriptions', activeSubscriptions); + if (subscriptions != null) { + matcher = + matcher.having((e) => e.subscriptions, 'subscriptions', subscriptions); } return matcher; From 612748ecc1ac6f3aac839f5b01b88e972f43ea5d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 4 Sep 2025 14:32:53 +0200 Subject: [PATCH 27/33] Improve some API names --- .../lib/src/sync/connection_manager.dart | 2 +- .../powersync_core/lib/src/sync/sync_status.dart | 10 +++++----- packages/powersync_core/test/sync/stream_test.dart | 14 +++++++------- packages/powersync_core/test/sync/utils.dart | 7 +++---- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 104be32a..263671f7 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -380,7 +380,7 @@ final class _SyncStreamSubscriptionHandle implements SyncStreamSubscription { @override Future waitForFirstSync() async { return _source.connections.firstStatusMatching((status) { - final currentProgress = status.statusFor(this); + final currentProgress = status.forStream(this); return currentProgress?.subscription.hasSynced ?? false; }); } diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index 57177392..13ff02c5 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -117,11 +117,11 @@ final class SyncStatus { ); } - /// All sync streams currently being tracked in this subscription. + /// All sync streams currently being tracked in the database. /// /// This returns null when the database is currently being opened and we /// don't have reliable information about all included streams yet. - Iterable? get subscriptions { + Iterable? get syncStreams { return _internalSubscriptions?.map((subscription) { return SyncStreamStatus._(subscription, downloadProgress); }); @@ -166,9 +166,9 @@ final class SyncStatus { ); } - /// If the [stream] appears in [activeSubscriptions], returns the current - /// status for that stream. - SyncStreamStatus? statusFor(SyncStreamDescription stream) { + /// If the [stream] appears in [syncStreams], returns the current status for + /// that stream. + SyncStreamStatus? forStream(SyncStreamDescription stream) { final raw = _internalSubscriptions?.firstWhereOrNull( (e) => e.name == stream.name && diff --git a/packages/powersync_core/test/sync/stream_test.dart b/packages/powersync_core/test/sync/stream_test.dart index 7aa88fb4..6d27456e 100644 --- a/packages/powersync_core/test/sync/stream_test.dart +++ b/packages/powersync_core/test/sync/stream_test.dart @@ -142,18 +142,18 @@ void main() { var status = await statusStream.next; for (final subscription in [a, b]) { - expect(status.statusFor(subscription)!.subscription.active, true); - expect(status.statusFor(subscription)!.subscription.lastSyncedAt, isNull); + expect(status.forStream(subscription)!.subscription.active, true); + expect(status.forStream(subscription)!.subscription.lastSyncedAt, isNull); expect( - status.statusFor(subscription)!.subscription.hasExplicitSubscription, + status.forStream(subscription)!.subscription.hasExplicitSubscription, true, ); } syncService.addLine(checkpointComplete(priority: 1)); status = await statusStream.next; - expect(status.statusFor(a)!.subscription.lastSyncedAt, isNull); - expect(status.statusFor(b)!.subscription.lastSyncedAt, isNotNull); + expect(status.forStream(a)!.subscription.lastSyncedAt, isNull); + expect(status.forStream(b)!.subscription.lastSyncedAt, isNotNull); await b.waitForFirstSync(); syncService.addLine(checkpointComplete()); @@ -170,7 +170,7 @@ void main() { status, emits( isSyncStatus( - subscriptions: [ + syncStreams: [ isStreamStatus( subscription: isSyncSubscription( name: 'default_stream', @@ -217,6 +217,6 @@ void main() { final subscription = await database.syncStream('foo').subscribe(); var status = await stream.next; - expect(status.statusFor(subscription), isNotNull); + expect(status.forStream(subscription), isNotNull); }); } diff --git a/packages/powersync_core/test/sync/utils.dart b/packages/powersync_core/test/sync/utils.dart index f4767fb2..53654f12 100644 --- a/packages/powersync_core/test/sync/utils.dart +++ b/packages/powersync_core/test/sync/utils.dart @@ -8,7 +8,7 @@ TypeMatcher isSyncStatus({ Object? connecting, Object? hasSynced, Object? downloadProgress, - Object? subscriptions, + Object? syncStreams, }) { var matcher = isA(); if (downloading != null) { @@ -27,9 +27,8 @@ TypeMatcher isSyncStatus({ matcher = matcher.having( (e) => e.downloadProgress, 'downloadProgress', downloadProgress); } - if (subscriptions != null) { - matcher = - matcher.having((e) => e.subscriptions, 'subscriptions', subscriptions); + if (syncStreams != null) { + matcher = matcher.having((e) => e.syncStreams, 'syncStreams', syncStreams); } return matcher; From cabae1dbdf40ed66d740e16ad8f38884cdfbd63a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 4 Sep 2025 17:56:26 +0200 Subject: [PATCH 28/33] Fix demo --- demos/supabase-todolist/lib/widgets/todo_list_page.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/supabase-todolist/lib/widgets/todo_list_page.dart b/demos/supabase-todolist/lib/widgets/todo_list_page.dart index 04349245..b1245321 100644 --- a/demos/supabase-todolist/lib/widgets/todo_list_page.dart +++ b/demos/supabase-todolist/lib/widgets/todo_list_page.dart @@ -125,7 +125,7 @@ class _SyncStreamTodosState extends State<_SyncStreamTodoListWidget> { builder: (context, snapshot) { final hasSynced = switch (_listSubscription) { null => null, - final sub => snapshot.requireData.statusFor(sub), + final sub => snapshot.requireData.forStream(sub), } ?.subscription .hasSynced ?? From f47c22b3ab5e7d0f3092523ff6b660de254fe69e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 8 Sep 2025 09:44:51 +0200 Subject: [PATCH 29/33] Finalizer: Just print warning --- .../lib/src/sync/connection_manager.dart | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index 263671f7..d29821fb 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -358,10 +358,6 @@ final class _SyncStreamSubscriptionHandle implements SyncStreamSubscription { _SyncStreamSubscriptionHandle(this._source) { _source.refcount++; - - // This is not unreliable, but can help decrementing refcounts on the inner - // subscription when this handle is deallocated without [unsubscribe] being - // called. _finalizer.attach(this, _source, detach: this); } @@ -385,6 +381,12 @@ final class _SyncStreamSubscriptionHandle implements SyncStreamSubscription { }); } - static final Finalizer<_ActiveSubscription> _finalizer = - Finalizer((sub) => sub.decrementRefCount()); + static final Finalizer<_ActiveSubscription> _finalizer = Finalizer((sub) { + sub.connections.db.logger.warning( + 'A subscription to ${sub.name} (with parameters ${sub.parameters}) ' + 'leaked! Please ensure calling SyncStreamSubscription.unsubscribe() ' + "when you don't need a subscription anymore. For global " + 'subscriptions, consider storing them in global fields to avoid this ' + 'warning.'); + }); } From 53c0d7156282ad323de9f207bd551982cca1dde2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 9 Sep 2025 11:36:27 +0200 Subject: [PATCH 30/33] Two more tests --- .../lib/src/database/powersync_db_mixin.dart | 3 ++ .../lib/src/sync/connection_manager.dart | 10 +++-- .../powersync_core/lib/src/sync/options.dart | 4 ++ .../powersync_core/lib/src/sync/stream.dart | 2 +- .../lib/src/sync/streaming_sync.dart | 3 +- .../lib/src/sync/sync_status.dart | 10 +++++ .../powersync_core/test/sync/stream_test.dart | 41 +++++++++++++++++++ 7 files changed, 68 insertions(+), 5 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 21aeb874..98f13783 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -504,6 +504,9 @@ SELECT * FROM crud_entries; await database.refreshSchema(); } + /// Create a [SyncStream] instance for the given [name] and [parameters]. + /// + /// Use [SyncStream.subscribe] to subscribe to the returned stream. SyncStream syncStream(String name, [Map? parameters]) { return _connections.syncStream(name, parameters); } diff --git a/packages/powersync_core/lib/src/sync/connection_manager.dart b/packages/powersync_core/lib/src/sync/connection_manager.dart index d29821fb..8a326642 100644 --- a/packages/powersync_core/lib/src/sync/connection_manager.dart +++ b/packages/powersync_core/lib/src/sync/connection_manager.dart @@ -355,6 +355,7 @@ final class _ActiveSubscription { final class _SyncStreamSubscriptionHandle implements SyncStreamSubscription { final _ActiveSubscription _source; + var _active = true; _SyncStreamSubscriptionHandle(this._source) { _source.refcount++; @@ -368,9 +369,12 @@ final class _SyncStreamSubscriptionHandle implements SyncStreamSubscription { Map? get parameters => _source.parameters; @override - void unsubscribe() async { - _finalizer.detach(this); - _source.decrementRefCount(); + void unsubscribe() { + if (_active) { + _active = false; + _finalizer.detach(this); + _source.decrementRefCount(); + } } @override diff --git a/packages/powersync_core/lib/src/sync/options.dart b/packages/powersync_core/lib/src/sync/options.dart index 48ed293e..ee8b3c63 100644 --- a/packages/powersync_core/lib/src/sync/options.dart +++ b/packages/powersync_core/lib/src/sync/options.dart @@ -27,6 +27,10 @@ final class SyncOptions { /// The [SyncClientImplementation] to use. final SyncClientImplementation syncImplementation; + /// Whether streams that have been defined with `auto_subscribe: true` should + /// be synced when they don't have an explicit subscription. + /// + /// This is enabled by default. final bool? includeDefaultStreams; const SyncOptions({ diff --git a/packages/powersync_core/lib/src/sync/stream.dart b/packages/powersync_core/lib/src/sync/stream.dart index c1fccb26..80c447be 100644 --- a/packages/powersync_core/lib/src/sync/stream.dart +++ b/packages/powersync_core/lib/src/sync/stream.dart @@ -24,7 +24,7 @@ abstract interface class SyncStreamDescription { abstract interface class SyncSubscriptionDescription extends SyncStreamDescription { /// Whether this stream is active, meaning that the subscription has been - /// acknownledged by the sync serivce. + /// acknowledged by the sync serivce. bool get active; /// Whether this stream subscription is included by default, regardless of diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index 25426abf..60deef12 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -622,7 +622,8 @@ final class _ActiveRustStreamingIteration { List _encodeSubscriptions(List subscriptions) { return sync._activeSubscriptions - .map((s) => {'name': s.name, 'params': s.parameters}) + .map((s) => + {'name': s.name, 'params': convert.json.decode(s.parameters)}) .toList(); } diff --git a/packages/powersync_core/lib/src/sync/sync_status.dart b/packages/powersync_core/lib/src/sync/sync_status.dart index 13ff02c5..61ae7c5f 100644 --- a/packages/powersync_core/lib/src/sync/sync_status.dart +++ b/packages/powersync_core/lib/src/sync/sync_status.dart @@ -212,11 +212,21 @@ extension InternalSyncStatusAccess on SyncStatus { _internalSubscriptions; } +/// Current information about a [SyncStream] that the sync client is subscribed +/// to. final class SyncStreamStatus { + /// If the [SyncStatus] is currently [SyncStatus.downloading], download + /// progress for this stream. final ProgressWithOperations? progress; final CoreActiveStreamSubscription _internal; + /// The [SyncSubscriptionDescription] providing information about the current + /// stream state. SyncSubscriptionDescription get subscription => _internal; + + /// The [StreamPriority] of the current stream. + /// + /// New data on higher-priority streams can interrupt lower-priority streams. StreamPriority get priority => _internal.priority; SyncStreamStatus._(this._internal, SyncDownloadProgress? progress) diff --git a/packages/powersync_core/test/sync/stream_test.dart b/packages/powersync_core/test/sync/stream_test.dart index 6d27456e..1625656c 100644 --- a/packages/powersync_core/test/sync/stream_test.dart +++ b/packages/powersync_core/test/sync/stream_test.dart @@ -219,4 +219,45 @@ void main() { var status = await stream.next; expect(status.forStream(subscription), isNotNull); }); + + test('unsubscribing multiple times has no effect', () async { + final a = await database.syncStream('a').subscribe(); + final aAgain = await database.syncStream('a').subscribe(); + a.unsubscribe(); + a.unsubscribe(); // Should not decrement the refcount again + + // Pretend the streams are expired - they should still be requested because + // the core extension extends the lifetime of streams currently referenced + // before connecting. + await database.execute( + 'UPDATE ps_stream_subscriptions SET expires_at = unixepoch() - 1000'); + + await waitForConnection(); + final request = await syncService.waitForListener; + expect( + json.decode(await request.readAsString()), + containsPair( + 'streams', + containsPair('subscriptions', isNotEmpty), + ), + ); + aAgain.unsubscribe(); + }); + + test('unsubscribeAll', () async { + final a = await database.syncStream('a').subscribe(); + await database.syncStream('a').unsubscribeAll(); + + // Despite a being active, it should not be requested. + await waitForConnection(); + final request = await syncService.waitForListener; + expect( + json.decode(await request.readAsString()), + containsPair( + 'streams', + containsPair('subscriptions', isEmpty), + ), + ); + a.unsubscribe(); + }); } From 4c3049c9121bd4ba46c207380564b62c2bc7254d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 23 Sep 2025 17:20:04 +0200 Subject: [PATCH 31/33] Make lists page stream-aware --- .../lib/widgets/list_item_sync_stream.dart | 78 +++++++++++++++++++ .../lib/widgets/lists_page.dart | 6 +- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 demos/supabase-todolist/lib/widgets/list_item_sync_stream.dart diff --git a/demos/supabase-todolist/lib/widgets/list_item_sync_stream.dart b/demos/supabase-todolist/lib/widgets/list_item_sync_stream.dart new file mode 100644 index 00000000..e400f8aa --- /dev/null +++ b/demos/supabase-todolist/lib/widgets/list_item_sync_stream.dart @@ -0,0 +1,78 @@ +import 'package:flutter/material.dart'; + +import '../powersync.dart'; +import './todo_list_page.dart'; +import '../models/todo_list.dart'; + +/// A variant of the `ListItem` that only shows a summary of completed and +/// pending items when the respective list has an active sync stream. +class SyncStreamsAwareListItem extends StatelessWidget { + SyncStreamsAwareListItem({ + required this.list, + }) : super(key: ObjectKey(list)); + + final TodoList list; + + Future delete() async { + // Server will take care of deleting related todos + await list.delete(); + } + + @override + Widget build(BuildContext context) { + viewList() { + var navigator = Navigator.of(context); + + navigator.push( + MaterialPageRoute(builder: (context) => TodoListPage(list: list))); + } + + return StreamBuilder( + stream: db.statusStream, + initialData: db.currentStatus, + builder: (context, asyncSnapshot) { + final status = asyncSnapshot.requireData; + final stream = + status.forStream(db.syncStream('todos', {'list': list.id})); + + String subtext; + if (stream == null || !stream.subscription.active) { + subtext = 'Items not loaded - click to fetch.'; + } else { + subtext = + '${list.pendingCount} pending, ${list.completedCount} completed'; + } + + return Card( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + ListTile( + onTap: viewList, + leading: const Icon(Icons.list), + title: Text(list.name), + subtitle: Text(subtext), + ), + Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + IconButton( + iconSize: 30, + icon: const Icon( + Icons.delete, + color: Colors.red, + ), + tooltip: 'Delete List', + alignment: Alignment.centerRight, + onPressed: delete, + ), + const SizedBox(width: 8), + ], + ), + ], + ), + ); + }, + ); + } +} diff --git a/demos/supabase-todolist/lib/widgets/lists_page.dart b/demos/supabase-todolist/lib/widgets/lists_page.dart index e883a445..564e03ee 100644 --- a/demos/supabase-todolist/lib/widgets/lists_page.dart +++ b/demos/supabase-todolist/lib/widgets/lists_page.dart @@ -1,11 +1,13 @@ import 'package:flutter/material.dart'; import 'package:powersync/powersync.dart'; +import '../app_config.dart'; import './list_item.dart'; import './list_item_dialog.dart'; import '../main.dart'; import '../models/todo_list.dart'; import 'guard_by_sync.dart'; +import 'list_item_sync_stream.dart'; void _showAddDialog(BuildContext context) async { return showDialog( @@ -55,7 +57,9 @@ class ListsWidget extends StatelessWidget { return ListView( padding: const EdgeInsets.symmetric(vertical: 8.0), children: todoLists.map((list) { - return ListItemWidget(list: list); + return AppConfig.hasSyncStreams + ? SyncStreamsAwareListItem(list: list) + : ListItemWidget(list: list); }).toList(), ); } else { From ab0ffec16fe565c02a1469c4f5a15710e60ea133 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 2 Oct 2025 14:06:49 +0200 Subject: [PATCH 32/33] Typo --- .../powersync_core/lib/src/database/powersync_db_mixin.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 98f13783..fd722a2a 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -180,7 +180,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { // Now we can close the database await database.close(); - // If there are paused subscriptionso n the status stream, don't delay + // If there are paused subscriptions on the status stream, don't delay // closing the database because of that. _connections.close(); await _activeGroup.close(); From 88f23b91f4cfaaa79bb0acf0b7705bb8373b6c49 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 2 Oct 2025 15:02:28 +0200 Subject: [PATCH 33/33] Fix bad merge --- packages/powersync_core/test/utils/abstract_test_utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/powersync_core/test/utils/abstract_test_utils.dart b/packages/powersync_core/test/utils/abstract_test_utils.dart index be5b420a..96469c5a 100644 --- a/packages/powersync_core/test/utils/abstract_test_utils.dart +++ b/packages/powersync_core/test/utils/abstract_test_utils.dart @@ -83,7 +83,7 @@ abstract mixin class TestPowerSyncFactory implements PowerSyncOpenFactory { database: SqliteDatabase.singleConnection( SqliteConnection.synchronousWrapper(raw)), logger: logger ?? Logger.detached('PowerSync.test'), - schema: schema, + schema: customSchema ?? schema, ); } }