From 752cff691f0715f3a02e4cd3446cfc281751d618 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 9 Nov 2025 20:36:30 +0000 Subject: [PATCH 1/3] Initial plan From d058eb0479e56e606e3ef408a109ccc413f8033b Mon Sep 17 00:00:00 2001 From: Test User Date: Sun, 9 Nov 2025 20:52:12 +0000 Subject: [PATCH 2/3] Optimize performance: reduce allocations in VarInt parsing and parameter binding Co-authored-by: Giorgi <580749+Giorgi@users.noreply.github.com> --- .../Reader/NumericVectorDataReader.cs | 68 +++++++++++-------- .../PreparedStatement/PreparedStatement.cs | 13 +++- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs b/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs index d2b212b..9ce377e 100644 --- a/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs +++ b/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs @@ -117,56 +117,72 @@ private unsafe T GetBigInteger(ulong offset) var isNegative = (buffer[0] & 0x80) == 0; - var bytes = new List(data->Length - VarIntHeaderSize); + // Allocate byte array once with proper capacity + var byteCount = data->Length - VarIntHeaderSize; + var bytes = new byte[byteCount]; + var currentByteCount = byteCount; - for (var index = VarIntHeaderSize; index < buffer.Length; index++) + for (var index = 0; index < byteCount; index++) { if (isNegative) { - bytes.Add((byte)~buffer[index]); + bytes[index] = (byte)~buffer[index + VarIntHeaderSize]; } else { - bytes.Add(buffer[index]); + bytes[index] = buffer[index + VarIntHeaderSize]; } } - var bigIntegerDigits = new Stack(); + // Estimate maximum digit count: log10(256^n) = n * log10(256) ≈ n * 2.408 + var maxDigits = (int)(byteCount * 2.5) + 2; + var digitChars = new char[maxDigits]; + var digitCount = 0; - while (bytes.Count > 0) - { - var quotient = new List(); + // Preallocate quotient buffer + var quotient = new byte[byteCount]; + while (currentByteCount > 0) + { byte remainder = 0; + var quotientCount = 0; - foreach (var @byte in bytes) + for (var i = 0; i < currentByteCount; i++) { - var newValue = remainder * 256 + @byte; - quotient.Add(DigitToChar(newValue / 10)); + var newValue = remainder * 256 + bytes[i]; + var digit = (byte)(newValue / 10); + + // Only add non-zero or if we already have digits + if (digit != 0 || quotientCount > 0) + { + quotient[quotientCount++] = digit; + } remainder = (byte)(newValue % 10); } - bigIntegerDigits.Push(DigitToChar(remainder)); + digitChars[digitCount++] = DigitToChar(remainder); - // Remove leading zeros from the quotient - bytes.Clear(); - - foreach (var digit in quotient) - { - if (digit != '0' || bytes.Count > 0) - { - bytes.Add(CharToDigit(digit)); - } - } + // Swap buffers to avoid allocation + var temp = bytes; + bytes = quotient; + quotient = temp; + currentByteCount = quotientCount; } if (isNegative) { - bigIntegerDigits.Push('-'); + digitChars[digitCount++] = '-'; } - - var integer = BigInteger.Parse(new string(bigIntegerDigits.ToArray())); + + // Reverse the digits in place + Array.Reverse(digitChars, 0, digitCount); + +#if NET6_0_OR_GREATER + var integer = BigInteger.Parse(digitChars.AsSpan(0, digitCount)); +#else + var integer = BigInteger.Parse(new string(digitChars, 0, digitCount)); +#endif try { @@ -178,8 +194,6 @@ private unsafe T GetBigInteger(ulong offset) } char DigitToChar(int c) => (char)(c + '0'); - - byte CharToDigit(char digit) => (byte)(digit-'0'); } private T GetBigInteger(ulong offset, bool unsigned) diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index f9bd58b..2bdb668 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -85,7 +85,18 @@ private static void BindParameters(DuckDBPreparedStatement preparedStatement, Du throw new InvalidOperationException($"Invalid number of parameters. Expected {expectedParameters}, got {parameterCollection.Count}"); } - if (parameterCollection.OfType().Any(p => !string.IsNullOrEmpty(p.ParameterName))) + // Check if any parameter has a name (faster than LINQ) + var hasNamedParameters = false; + for (var i = 0; i < parameterCollection.Count; i++) + { + if (!string.IsNullOrEmpty(parameterCollection[i].ParameterName)) + { + hasNamedParameters = true; + break; + } + } + + if (hasNamedParameters) { foreach (DuckDBParameter param in parameterCollection) { From 3df4e6f9f9809d928853a2880f35f363cf1ec7f2 Mon Sep 17 00:00:00 2001 From: Test User Date: Sun, 9 Nov 2025 20:58:01 +0000 Subject: [PATCH 3/3] Optimize DuckDBDataReader: replace foreach with for loops for better performance Co-authored-by: Giorgi <580749+Giorgi@users.noreply.github.com> --- DuckDB.NET.Data/DuckDBDataReader.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBDataReader.cs b/DuckDB.NET.Data/DuckDBDataReader.cs index 75094fa..d7ebdf0 100644 --- a/DuckDB.NET.Data/DuckDBDataReader.cs +++ b/DuckDB.NET.Data/DuckDBDataReader.cs @@ -67,9 +67,10 @@ private bool InitNextReader() private bool InitChunkData() { - foreach (var reader in vectorReaders) + // Dispose existing readers using for loop (faster than foreach) + for (int i = 0; i < vectorReaders.Length; i++) { - reader.Dispose(); + vectorReaders[i]?.Dispose(); } currentChunk?.Dispose(); @@ -343,9 +344,10 @@ public override void Close() { if (closed) return; - foreach (var reader in vectorReaders) + // Dispose readers using for loop (faster than foreach) + for (int i = 0; i < vectorReaders.Length; i++) { - reader.Dispose(); + vectorReaders[i]?.Dispose(); } currentChunk?.Dispose();