Npgsql: Use EnableDynamicJson to unlock better container type mappings

amotl · amotl · commit e88bbc566de9 · 2024-12-19T23:05:49.000+01:00
Vanilla Npgsql provides good enough support to handle CrateDB's ARRAY
and OBJECT types better than just plain strings. This patch demonstrates
type mappings to native .NET `List` and `Dictionary` types, as well as
type mappings to custom .NET POCO types.
diff --git a/by-language/csharp-npgsql/BasicPoco.cs b/by-language/csharp-npgsql/BasicPoco.cs
@@ -0,0 +1,22 @@
+namespace demo;
+
+public class BasicPoco
+{
+
+    public string? name { get; set; }
+    public int? age { get; set; }
+
+    public override bool Equals(object obj)
+    {
+        var other = (BasicPoco) obj;
+        return name == other.name && age == other.age;
+    }
+
+    public override int GetHashCode()
+    {
+        return base.GetHashCode();
+    }
+
+    public override string ToString() => "Name: " + name + " Age: " + age;
+
+}
diff --git a/by-language/csharp-npgsql/DemoProgram.cs b/by-language/csharp-npgsql/DemoProgram.cs
@@ -18,12 +18,18 @@ await Parser.Default.ParseArguments<Options>(args)
                     var connString = $"Host={options.Host};Port={options.Port};SSL Mode={options.SslMode};" +
                                      $"Username={options.Username};Password={options.Password};Database={options.Database}";
                     Console.WriteLine($"Connecting to {connString}\n");
-                    await using var conn = new NpgsqlConnection(connString);
-                    conn.Open();
+
+                    var dataSourceBuilder = new NpgsqlDataSourceBuilder(connString);
+                    dataSourceBuilder.EnableDynamicJson();
+                    await using var dataSource = dataSourceBuilder.Build();
+                    await using var conn = dataSource.OpenConnection();
+
                     await DatabaseWorkloads.SystemQueryExample(conn);
                     await DatabaseWorkloads.BasicConversationExample(conn);
                     await DatabaseWorkloads.UnnestExample(conn);
-                    await DatabaseWorkloadsMore.AllTypesExample(conn);
+                    await DatabaseWorkloadsMore.AllTypesNativeExample(conn);
+                    await DatabaseWorkloadsMore.ObjectPocoExample(conn);
+                    await DatabaseWorkloadsMore.ObjectPocoArrayExample(conn);
                     conn.Close();
                 });
 
diff --git a/by-language/csharp-npgsql/DemoTypes.cs b/by-language/csharp-npgsql/DemoTypes.cs
@@ -57,9 +57,9 @@ public class AllTypesRecord
     public class DatabaseWorkloadsMore
     {
 
-        public static async Task<DataTable> AllTypesExample(NpgsqlConnection conn)
+        public static async Task<DataTable> AllTypesNativeExample(NpgsqlConnection conn)
         {
-            Console.WriteLine("Running AllTypesExample");
+            Console.WriteLine("Running AllTypesNativeExample");
 
             // Submit DDL, create database schema.
             await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.example", conn))
@@ -154,11 +154,8 @@ INSERT INTO testdrive.example (
                 cmd.Parameters.AddWithValue("timestamp_tz", "1970-01-02T00:00:00+01:00");
                 cmd.Parameters.AddWithValue("timestamp_notz", "1970-01-02T00:00:00");
                 cmd.Parameters.AddWithValue("ip", "127.0.0.1");
-                cmd.Parameters.AddWithValue("array", new List<string>{"foo", "bar"});
-                // FIXME: System.NotSupportedException: Cannot resolve 'hstore' to a fully qualified datatype name. The datatype was not found in the current database info.
-                // https://github.com/crate/zk/issues/26
-                // cmd.Parameters.AddWithValue("object", new Dictionary<string, string>(){{"foo", "bar"}});
-                cmd.Parameters.AddWithValue("object", """{"foo": "bar"}""");
+                cmd.Parameters.AddWithValue("array", NpgsqlDbType.Json, new List<string>{"foo", "bar"});
+                cmd.Parameters.AddWithValue("object", NpgsqlDbType.Json, new Dictionary<string, string>{{"foo", "bar"}});
                 cmd.Parameters.AddWithValue("geopoint", new List<double>{85.43, 66.23});
                 // TODO: Check if `GEO_SHAPE` types can be represented by real .NET or Npgsql data types.
                 cmd.Parameters.AddWithValue("geoshape", "POLYGON ((5 5, 10 5, 10 10, 5 10, 5 5))");
@@ -185,20 +182,22 @@ INSERT INTO testdrive.example (
 
         }
 
-        public static async Task<DataTable> ContainerTypesExample(NpgsqlConnection conn)
+        public static async Task ProvisionPoco(NpgsqlConnection conn)
         {
-            Console.WriteLine("Running AllTypesExample");
+            /***
+             * Verify Npgsql POCO mapping with CrateDB.
+             * https://www.npgsql.org/doc/types/json.html#poco-mapping
+             */
 
             // Submit DDL, create database schema.
-            await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.container", conn))
+            await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.poco", conn))
             {
                 cmd.ExecuteNonQuery();
             }
 
             await using (var cmd = new NpgsqlCommand("""
-                CREATE TABLE testdrive.container (
-                    -- Container types
-                    "array" ARRAY(STRING),
+                CREATE TABLE testdrive.poco (
+                    "array" ARRAY(OBJECT(DYNAMIC)),
                     "object" OBJECT(DYNAMIC)
                 );
             """, conn))
@@ -208,7 +207,7 @@ CREATE TABLE testdrive.container (
 
             // Insert single data point.
             await using (var cmd = new NpgsqlCommand("""
-                INSERT INTO testdrive.container (
+                INSERT INTO testdrive.poco (
                     "array",
                     "object"
                 ) VALUES (
@@ -217,32 +216,58 @@ INSERT INTO testdrive.container (
                 );
             """, conn))
             {
-                Console.WriteLine(cmd);
-                // FIXME: While doing conversations with ARRAY types works natively,
-                //        it doesn't work for OBJECT types.
-                //        Yet, they can be submitted as STRING in JSON format.
-                cmd.Parameters.AddWithValue("array", new List<string>{"foo", "bar"});
-                cmd.Parameters.AddWithValue("object", """{"foo": "bar"}""");
+                cmd.Parameters.AddWithValue("object", NpgsqlDbType.Json, new BasicPoco { name = "Hotzenplotz" });
+                cmd.Parameters.AddWithValue("array", NpgsqlDbType.Json, new List<BasicPoco>
+                {
+                    new BasicPoco { name = "Hotzenplotz" },
+                    new BasicPoco { name = "Petrosilius", age = 42 },
+                });
                 cmd.ExecuteNonQuery();
             }
 
             // Flush data.
-            await using (var cmd = new NpgsqlCommand("REFRESH TABLE testdrive.container", conn))
+            await using (var cmd = new NpgsqlCommand("REFRESH TABLE testdrive.poco", conn))
             {
                 cmd.ExecuteNonQuery();
             }
 
+        }
+
+        public static async Task<BasicPoco> ObjectPocoExample(NpgsqlConnection conn)
+        {
+            Console.WriteLine("Running ObjectPocoExample");
+
+            // Provision data.
+            await ProvisionPoco(conn);
+
             // Query back data.
-            await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.container", conn))
+            await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.poco", conn))
             await using (var reader = cmd.ExecuteReader())
             {
-                var dataTable = new DataTable();
-                dataTable.Load(reader);
-                var payload = JsonConvert.SerializeObject(dataTable);
-                Console.WriteLine(payload);
-                return (DataTable) dataTable;
+                reader.Read();
+                var obj = reader.GetFieldValue<BasicPoco>("object");
+                Console.WriteLine(obj);
+                return obj;
             }
+        }
+
+        public static async Task<List<BasicPoco>> ObjectPocoArrayExample(NpgsqlConnection conn)
+        {
+            Console.WriteLine("Running ObjectPocoArrayExample");
 
+            // Provision data.
+            await ProvisionPoco(conn);
+
+            // Query back data.
+            await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.poco", conn))
+            await using (var reader = cmd.ExecuteReader())
+            {
+                reader.Read();
+                var obj = reader.GetFieldValue<List<BasicPoco>>("array");
+                Console.WriteLine(obj[0]);
+                Console.WriteLine(obj[1]);
+                return obj;
+            }
         }
 
     }
diff --git a/by-language/csharp-npgsql/tests/DemoProgramTest.cs b/by-language/csharp-npgsql/tests/DemoProgramTest.cs
@@ -21,8 +21,11 @@ public DatabaseFixture()
                 CRATEDB_DSN = $"Host=localhost;Port=5432;Username=crate;Password=;Database=testdrive";
             }
             Console.WriteLine($"Connecting to {CRATEDB_DSN}\n");
-            Db = new NpgsqlConnection(CRATEDB_DSN);
-            Db.Open();
+            
+            var dataSourceBuilder = new NpgsqlDataSourceBuilder(CRATEDB_DSN);
+            dataSourceBuilder.EnableDynamicJson();
+            using var dataSource = dataSourceBuilder.Build();
+            Db = dataSource.OpenConnection();
         }
 
         public void Dispose()
@@ -83,12 +86,12 @@ public async Task TestUnnestExample()
         }
 
         [Fact]
-        public async Task TestAllTypesExample()
+        public async Task TestAllTypesNativeExample()
         {
             var conn = fixture.Db;
 
-            // Invoke database workload.
-            var task = DatabaseWorkloadsMore.AllTypesExample(conn);
+            // Provision data.
+            var task = DatabaseWorkloadsMore.AllTypesNativeExample(conn);
             var dt = await task.WaitAsync(TimeSpan.FromSeconds(0.5));
 
             // Check results.
@@ -112,12 +115,18 @@ public async Task TestAllTypesExample()
             Assert.Equal("127.0.0.1", row["ip"]);
 
             // Container types
-            // FIXME: While doing conversations with ARRAY types works natively,
-            //        it doesn't work for OBJECT types.
-            //        Yet, they can be submitted as STRING in JSON format.
             Assert.Equal(new List<string>{"foo", "bar"}, row["array"]);
             Assert.Equal("""{"foo":"bar"}""", row["object"]);
 
+            // Note: While it works on the ingress side to communicate `Dictionary` types,
+            //       this kind of equality check does not work on the egress side,
+            //       presenting an error that indicates a different internal representation,
+            //       or a programming error ;].
+            //
+            //       Expected: [["foo"] = "bar"]
+            //       Actual:   {"foo":"bar"}
+            // Assert.Equal(new Dictionary<string, string>{{"foo", "bar"}}, row["object"]);
+
             // Geospatial types
             // TODO: Unlock native data types?
             //       GEO_POINT and GEO_SHAPE types can be marshalled back and forth using STRING.
@@ -135,29 +144,21 @@ public async Task TestContainerTypesExample()
         {
             var conn = fixture.Db;
 
-            // Invoke database workload.
-            var task = DatabaseWorkloadsMore.ContainerTypesExample(conn);
-            var dt = await task.WaitAsync(TimeSpan.FromSeconds(0.5));
-
-            // Check results.
-            var row = dt.Rows[0];
-            // FIXME: While doing conversations with ARRAY types works natively,
-            //        it doesn't work for OBJECT types.
-            //        Yet, they can be submitted as STRING in JSON format.
-            Assert.Equal(new List<string>{"foo", "bar"}, row["array"]);
-            Assert.Equal("""{"foo":"bar"}""", row["object"]);
+            // Provision data.
+            var task = DatabaseWorkloadsMore.AllTypesNativeExample(conn);
+            await task.WaitAsync(TimeSpan.FromSeconds(0.5));
 
-            // Run a special query indexing into ARRAY types.
-            await using (var cmd = new NpgsqlCommand("""SELECT "array[2]" AS foo FROM testdrive.container""", conn))
+            // Run an SQL query indexing into ARRAY types.
+            await using (var cmd = new NpgsqlCommand("""SELECT "array[2]" AS foo FROM testdrive.example""", conn))
             await using (var reader = cmd.ExecuteReader())
             {
                 var dataTable = new DataTable();
                 dataTable.Load(reader);
                 Assert.Equal("bar", dataTable.Rows[0]["foo"]);
             }
 
-            // Run a special query indexing into OBJECT types.
-            await using (var cmd = new NpgsqlCommand("""SELECT "object['foo']" AS foo FROM testdrive.container""", conn))
+            // Run an SQL query indexing into OBJECT types.
+            await using (var cmd = new NpgsqlCommand("""SELECT "object['foo']" AS foo FROM testdrive.example""", conn))
             await using (var reader = cmd.ExecuteReader())
             {
                 var dataTable = new DataTable();
@@ -167,5 +168,38 @@ public async Task TestContainerTypesExample()
 
         }
 
+        [Fact]
+        public async Task TestObjectPocoExample()
+        {
+            var conn = fixture.Db;
+
+            // Invoke database workload.
+            var task = DatabaseWorkloadsMore.ObjectPocoExample(conn);
+            var obj = await task.WaitAsync(TimeSpan.FromSeconds(0.5));
+
+            // Validate the outcome.
+            Assert.Equal(new BasicPoco { name = "Hotzenplotz" }, obj);
+
+        }
+
+        [Fact]
+        public async Task TestObjectPocoArrayExample()
+        {
+            var conn = fixture.Db;
+
+            // Invoke database workload.
+            var task = DatabaseWorkloadsMore.ObjectPocoArrayExample(conn);
+            var obj = await task.WaitAsync(TimeSpan.FromSeconds(0.5));
+
+            // Validate the outcome.
+            var reference = new List<BasicPoco>
+            {
+                new BasicPoco { name = "Hotzenplotz" },
+                new BasicPoco { name = "Petrosilius", age = 42 },
+            };
+            Assert.Equal(reference, obj);
+
+        }
+
     }
 }
