Merge branch 'master' into dev/versionstamps

Author: KrzysFR

FdbShell/Commands/BasicCommands.cs

@@ -220,9 +220,9 @@ public static async Task Count(string[] path, ITuple extras, IFdbDatabase db, Te
 			var copy = KeySubspace.Copy(folder);
 			log.WriteLine("# Counting keys under {0} ...", FdbKey.Dump(copy.GetPrefix()));
 
-			var progress = new Progress<STuple<long, Slice>>((state) =>
+			var progress = new Progress<(long Count, Slice Current)>((state) =>
 			{
-				log.Write("\r# Found {0:N0} keys...", state.Item1);
+				log.Write("\r# Found {0:N0} keys...", state.Count);
 			});
 
 			long count = await Fdb.System.EstimateCountAsync(db, copy.ToRange(), progress, ct);

FdbTop/Program.cs

@@ -819,7 +819,7 @@ private static void ShowProcessesScreen(FdbSystemStatus status, HistoryMetric cu
 				{
 					foreach(var role in proc.Roles)
 					{
-						map.Add(role.Value);
+						map.Add(role.Role);
 					}
 				}
 
@@ -860,7 +860,7 @@ private static void ShowProcessesScreen(FdbSystemStatus status, HistoryMetric cu
 					map = new RoleMap();
 					foreach (var role in proc.Roles)
 					{
-						map.Add(role.Value);
+						map.Add(role.Role);
 					}
 					Console.ForegroundColor = ConsoleColor.DarkGray;
 					WriteAt(1, y,

FoundationDB.Client/Fdb.Bulk.cs

@@ -31,6 +31,7 @@ namespace FoundationDB.Client
 	using System;
 	using System.Collections.Generic;
 	using System.Diagnostics;
+	using System.Linq;
 	using System.Threading;
 	using System.Threading.Tasks;
 	using Doxense.Diagnostics.Contracts;
@@ -98,7 +99,7 @@ public static Task<long> WriteAsync([NotNull] IFdbDatabase db, [NotNull] IEnumer
 
 				return RunWriteOperationAsync(
 					db,
-					data,
+					data.Select(x => (x.Key, x.Value)),
 					new WriteOptions(),
 					ct
 				);
@@ -118,6 +119,49 @@ public static Task<long> WriteAsync([NotNull] IFdbDatabase db, [NotNull] IEnumer
 
 				ct.ThrowIfCancellationRequested();
 
+				return RunWriteOperationAsync(
+					db,
+					data.Select(x => (x.Key, x.Value)),
+					options ?? new WriteOptions(),
+					ct
+				);
+			}
+
+			/// <summary>Writes a potentially large sequence of key/value pairs into the database, by using as many transactions as necessary, and automatically scaling the size of each batch.</summary>
+			/// <param name="db">Database used for the operation</param>
+			/// <param name="data">Sequence of key/value pairs</param>
+			/// <param name="ct">Token used to cancel the operation</param>
+			/// <returns>Total number of values inserted in the database</returns>
+			/// <remarks>In case of a non-retryable error, some of the keys may remain in the database. Other transactions running at the same time may observe only a fraction of the keys until the operation completes.</remarks>
+			public static Task<long> WriteAsync([NotNull] IFdbDatabase db, [NotNull] IEnumerable<(Slice Key, Slice Value)> data, CancellationToken ct)
+			{
+				if (db == null) throw new ArgumentNullException(nameof(db));
+				if (data == null) throw new ArgumentNullException(nameof(data));
+
+				ct.ThrowIfCancellationRequested();
+
+				return RunWriteOperationAsync(
+					db,
+					data,
+					new WriteOptions(),
+					ct
+				);
+			}
+
+			/// <summary>Writes a potentially large sequence of key/value pairs into the database, by using as many transactions as necessary, and automatically scaling the size of each batch.</summary>
+			/// <param name="db">Database used for the operation</param>
+			/// <param name="data">Sequence of key/value pairs</param>
+			/// <param name="options">Custom options used to configure the behaviour of the operation</param>
+			/// <param name="ct">Token used to cancel the operation</param>
+			/// <returns>Total number of values inserted in the database</returns>
+			/// <remarks>In case of a non-retryable error, some of the keys may remain in the database. Other transactions running at the same time may observe only a fraction of the keys until the operation completes.</remarks>
+			public static Task<long> WriteAsync([NotNull] IFdbDatabase db, [NotNull] IEnumerable<(Slice Key, Slice Value)> data, WriteOptions options, CancellationToken ct)
+			{
+				if (db == null) throw new ArgumentNullException(nameof(db));
+				if (data == null) throw new ArgumentNullException(nameof(data));
+
+				ct.ThrowIfCancellationRequested();
+
 				return RunWriteOperationAsync(
 					db,
 					data,
@@ -126,7 +170,7 @@ public static Task<long> WriteAsync([NotNull] IFdbDatabase db, [NotNull] IEnumer
 				);
 			}
 
-			internal static async Task<long> RunWriteOperationAsync([NotNull] IFdbDatabase db, [NotNull] IEnumerable<KeyValuePair<Slice, Slice>> data, WriteOptions options, CancellationToken ct)
+			internal static async Task<long> RunWriteOperationAsync([NotNull] IFdbDatabase db, [NotNull] IEnumerable<(Slice Key, Slice Value)> data, WriteOptions options, CancellationToken ct)
 			{
 				Contract.Requires(db != null && data != null && options != null);
 
@@ -145,7 +189,7 @@ internal static async Task<long> RunWriteOperationAsync([NotNull] IFdbDatabase d
 					throw new NotImplementedException("Multiple concurrent transactions are not yet supported");
 				}
 
-				var chunk = new List<KeyValuePair<Slice, Slice>>();
+				var chunk = new List<(Slice Key, Slice Value)>();
 
 				long items = 0;
 				using (var iterator = data.GetEnumerator())
@@ -960,7 +1004,6 @@ public static Task ForEachAsync<TSource, TLocal>(
 			/// <param name="localFinally">Lambda function that will be called after the last batch, and will be passed the last known state.</param>
 			/// <param name="ct">Token used to cancel the operation</param>
 			/// <returns>Task that completes when all the elements of <paramref name="source"/> have been processed, a non-retryable error occurs, or <paramref name="ct"/> is triggered</returns>
-			[Obsolete("EXPERIMENTAL: do not use yet!")]
 			public static Task ForEachAsync<TSource, TLocal>(
 				[NotNull] IFdbDatabase db,
 				[NotNull] IEnumerable<TSource> source,

FoundationDB.Client/Fdb.System.cs

@@ -408,7 +408,7 @@ public static Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, KeyRange
 			/// <param name="ct">Token used to cancel the operation</param>
 			/// <returns>Number of keys k such that range.Begin &lt;= k &gt; range.End</returns>
 			/// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks>
-			public static Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, KeyRange range, IProgress<STuple<long, Slice>> onProgress, CancellationToken ct)
+			public static Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, KeyRange range, IProgress<(long Count, Slice Current)> onProgress, CancellationToken ct)
 			{
 				return EstimateCountAsync(db, range.Begin, range.End, onProgress, ct);
 				//REVIEW: BUGBUG: REFACTORING: deal with null value for End!
@@ -422,7 +422,7 @@ public static Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, KeyRange
 			/// <param name="ct">Token used to cancel the operation</param>
 			/// <returns>Number of keys k such that <paramref name="beginInclusive"/> &lt;= k &gt; <paramref name="endExclusive"/></returns>
 			/// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks>
-			public static async Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, IProgress<STuple<long, Slice>> onProgress, CancellationToken ct)
+			public static async Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, IProgress<(long Count, Slice Current)> onProgress, CancellationToken ct)
 			{
 				const int INIT_WINDOW_SIZE = 1 << 8; // start at 256 //1024
 				const int MAX_WINDOW_SIZE = 1 << 13; // never use more than 4096
@@ -538,7 +538,7 @@ public static async Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, Sli
 									.ConfigureAwait(false);
 
 								counter += n;
-								if (onProgress != null) onProgress.Report(STuple.Create(counter, end));
+								onProgress?.Report((counter, end));
 #if TRACE_COUNTING
 								++iter;
 #endif
@@ -552,7 +552,7 @@ public static async Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, Sli
 						// the range is not finished, advance the cursor
 						counter += windowSize;
 						cursor = next;
-						if (onProgress != null) onProgress.Report(STuple.Create(counter, cursor));
+						onProgress?.Report((counter, cursor));
 
 						if (!last)
 						{ // double the size of the window if we are not in the last segment

FoundationDB.Client/FdbDatabase.cs

@@ -456,8 +456,8 @@ internal void ChangeRoot(IKeySubspace subspace, IFdbDirectory directory, bool re
 			lock (this)//TODO: don't use this for locking
 			{
 				m_readOnly = readOnly;
-				m_globalSpace = KeySubspace.Copy(subspace).Using(TypeSystem.Tuples);
-				m_globalSpaceCopy = KeySubspace.Copy(subspace).Using(TypeSystem.Tuples); // keep another copy
+				m_globalSpace = KeySubspace.Copy(subspace, TypeSystem.Tuples);
+				m_globalSpaceCopy = KeySubspace.Copy(subspace, TypeSystem.Tuples); // keep another copy
 				m_directory = directory == null ? null : new FdbDatabasePartition(this, directory);
 			}
 		}

FoundationDB.Client/FdbDatabaseExtensions.cs

@@ -318,18 +318,18 @@ public static Task SetAsync([NotNull] this IFdbRetryable db, Slice key, Slice va
 			return db.WriteAsync((tr) => tr.Set(key, value), ct);
 		}
 
-		/// <summary>Set the values of a list of keys in the database, using a dedicated transaction.</summary>
+		/// <summary>Set the values of a sequence of keys in the database, using a dedicated transaction.</summary>
 		/// <param name="db">Database instance</param>
 		/// <remarks>
 		/// Use this method only if you intend to perform a single operation inside your execution context (ex: HTTP request).
 		/// If you need to combine multiple read or write operations, consider using on of the multiple <see cref="IFdbRetryable.WriteAsync"/> or <see cref="IFdbRetryable.ReadWriteAsync"/> overrides.
 		/// </remarks>
-		public static Task SetValuesAsync([NotNull] this IFdbRetryable db, KeyValuePair<Slice, Slice>[] keyValuePairs, CancellationToken ct)
+		public static Task SetValuesAsync([NotNull] this IFdbRetryable db, IEnumerable<KeyValuePair<Slice, Slice>> items, CancellationToken ct)
 		{
 			Contract.NotNull(db, nameof(db));
 			return db.WriteAsync((tr) =>
 			{
-				foreach (var kv in keyValuePairs)
+				foreach (var kv in items)
 				{
 					tr.Set(kv.Key, kv.Value);
 				}
@@ -342,12 +342,12 @@ public static Task SetValuesAsync([NotNull] this IFdbRetryable db, KeyValuePair<
 		/// Use this method only if you intend to perform a single operation inside your execution context (ex: HTTP request).
 		/// If you need to combine multiple read or write operations, consider using on of the multiple <see cref="IFdbRetryable.WriteAsync"/> or <see cref="IFdbRetryable.ReadWriteAsync"/> overrides.
 		/// </remarks>
-		public static Task SetValuesAsync([NotNull] this IFdbRetryable db, IEnumerable<KeyValuePair<Slice, Slice>> keyValuePairs, CancellationToken ct)
+		public static Task SetValuesAsync([NotNull] this IFdbRetryable db, IEnumerable<(Slice Key, Slice Value)> items, CancellationToken ct)
 		{
 			Contract.NotNull(db, nameof(db));
 			return db.WriteAsync((tr) =>
 			{
-				foreach (var kv in keyValuePairs)
+				foreach (var kv in items)
 				{
 					tr.Set(kv.Key, kv.Value);
 				}

FoundationDB.Client/FoundationDB.Client.csproj

@@ -32,4 +32,8 @@
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="System.ValueTuple" Version="4.4.0" />
+  </ItemGroup>
+
 </Project>

FoundationDB.Client/Layers/Directories/FdbDirectoryLayer.cs

@@ -153,34 +153,29 @@ internal FdbDirectoryLayer(IDynamicKeySubspace nodeSubspace, IDynamicKeySubspace
 			}
 		}
 
-		/// <summary>Create an instance of the default Directory Layer</summary>
-		[NotNull]
-		public static FdbDirectoryLayer Create()
-		{
-			return Create(Slice.Empty);
-		}
-
 		/// <summary>Create an instance of a Directory Layer located under a specific prefix and path</summary>
 		/// <param name="prefix">Prefix for the content. The nodes will be stored under <paramref name="prefix"/> + &lt;FE&gt;</param>
 		/// <param name="path">Optional path, if the Directory Layer is not located at the root of the database.</param>
+		/// <param name="encoding">Optional key encoding scheme. If not specified, will use the <see cref="TypeSystem.Tuples"/> type system by default.</param>
 		[NotNull]
-		public static FdbDirectoryLayer Create(Slice prefix, IEnumerable<string> path = null)
+		public static FdbDirectoryLayer Create(Slice prefix, IEnumerable<string> path = null, IKeyEncoding encoding = null)
 		{
-			var subspace = KeySubspace.FromKey(prefix).Using(TypeSystem.Tuples);
+			var subspace = KeySubspace.CreateDynamic(prefix, encoding ?? TypeSystem.Tuples);
 			var location = path != null ? ParsePath(path) : STuple.Empty;
 			return new FdbDirectoryLayer(subspace.Partition[FdbKey.Directory], subspace, location);
 		}
 
 		/// <summary>Create an instance of a Directory Layer located under a specific subspace and path</summary>
 		/// <param name="subspace">Subspace for the content. The nodes will be stored under <paramref name="subspace"/>.Key + &lt;FE&gt;</param>
 		/// <param name="path">Optional path, if the Directory Layer is not located at the root of the database.</param>
+		/// <param name="encoding">Optional key encoding scheme. If not specified, will use the <see cref="TypeSystem.Tuples"/> type system by default.</param>
 		[NotNull]
-		public static FdbDirectoryLayer Create(IKeySubspace subspace, IEnumerable<string> path = null)
+		public static FdbDirectoryLayer Create(IKeySubspace subspace, IEnumerable<string> path = null, IKeyEncoding encoding = null)
 		{
 			if (subspace == null) throw new ArgumentNullException(nameof(subspace));
 
 			var location = path != null ? ParsePath(path) : STuple.Empty;
-			var space = subspace.Using(TypeSystem.Tuples);
+			var space = subspace.Using(encoding ?? TypeSystem.Tuples);
 			return new FdbDirectoryLayer(space.Partition[FdbKey.Directory], space, location);
 		}
 
@@ -539,14 +534,6 @@ internal static ITuple ParsePath(IEnumerable<string> path, string argName = null
 			return STuple.FromArray<string>(pathCopy);
 		}
 
-		[NotNull]
-		internal static ITuple ParsePath([NotNull] string name, string argName = null)
-		{
-			if (name == null) throw new ArgumentNullException(argName ?? "name");
-
-			return STuple.Create<string>(name);
-		}
-
 		[NotNull]
 		internal static ITuple VerifyPath([NotNull] ITuple path, string argName = null)
 		{

FoundationDB.Client/Layers/Tuples/Encoding/TupleCodec`1.cs

@@ -55,7 +55,7 @@ public TupleCodec(T missingValue)
 
 		public override Slice EncodeOrdered(T value)
 		{
-			return TupleEncoder.EncodeKey(value);
+			return TupleEncoder.EncodeKey(default(Slice), value);
 		}
 
 		public override void EncodeOrderedSelfTerm(ref SliceWriter output, T value)