8000 Fixed nested handling in BulkCopy. fixes #465 (#468) · DarkWanderer/ClickHouse.Client@6a14b69 · GitHub
[go: up one dir, main page]
Skip to content
This repository was archived by the owner on Jun 22, 2025. It is now read-only.
/ ClickHouse.Client Public archive

Commit 6a14b69

Browse files
DarkWandererDarkWanderer
authored
Fixed nested handling in BulkCopy. fixes #465 (#468)
1 parent 8987029 commit 6a14b69

File tree

8 files changed

+66
-80
lines changed

8 files changed

+66
-80
lines changed

‎ClickHouse.Client.Tests/BulkCopyTests.cs

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using System.Diagnostics;
44
using System.Linq;
5+
using System.Runtime.CompilerServices;
56
using System.Text;
67
using System.Threading;
78
using System.Threading.Tasks;
@@ -10,6 +11,7 @@
1011
using ClickHouse.Client.Tests.Attributes;
1112
using ClickHouse.Client.Utility;
1213
using NUnit.Framework;
14+
using NUnit.Framework.Internal;
1315

1416
namespace ClickHouse.Client.Tests;
1517

@@ -287,7 +289,7 @@ public async Task ShouldNotLoseRowsOnMultipleBatches()
287289
[Test]
288290
public async Task ShouldExecuteWithDBNullArrays()
289291
{
290-
var targetTable = $"test.dbnull_array";
292+
var targetTable = $"test.bulk_dbnull_array";
291293

292294
await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}");
293295
await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (stringValue Array(String), intValue Array(Int32)) ENGINE TinyLog");
@@ -307,6 +309,28 @@ await bulkCopy.WriteToServerAsync(new List<object[]>
307309
using var reader = await connection.ExecuteReaderAsync($"SELECT * from {targetTable}");
308310
}
309311

312+
[Test]
313+
public async Task ShouldInsertNestedTable()
314+
{
315+
var targetTable = "test.bulk_nested";
316+
317+
await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}");
318+
await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (`_id` UUID, `Comments` Nested(Id Nullable(String), Comment Nullable(String))) ENGINE TinyLog");
319+
320+
using var bulkCopy = new ClickHouseBulkCopy(connection)
321+
{
322+
DestinationTableName = targetTable,
323+
};
324+
325+
await bulkCopy.InitAsync();
326+
327+
await bulkCopy.WriteToServerAsync(new List<object[]>() { new object[] { Guid.NewGuid(), new ITuple[] {("1", "Comment1"),("2","Comment2"),("3","Comment3")}} });
328+
329+
using var reader = await connection.ExecuteReaderAsync($"SELECT * from {targetTable}");
330+
Assert.AreEqual(1, bulkCopy.RowsWritten);
331+
Assert.AreEqual(1, await connection.ExecuteScalarAsync($"SELECT count() FROM {targetTable}"));
332+
}
333+
310334
private static string SanitizeTableName(string input)
311335
{
312336
var builder = new StringBuilder();

‎ClickHouse.Client.Tests/NestedTableTests.cs

Lines changed: 0 additions & 65 deletions
This file was deleted.

‎ClickHouse.Client.Tests/ORM/DapperTests.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ private static bool ShouldBeSupportedByDapper(string clickHouseType)
4040
return false;
4141
if (clickHouseType.Contains("Int256"))
4242
return false;
43+
if (clickHouseType.Contains("Nested"))
44+
return false;
4345
switch (clickHouseType)
4446
{
4547
case "UUID":

‎ClickHouse.Client.Tests/TestUtilities.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public static ClickHouseConnection GetTestClickHouseConnection(bool compression
5252
builder.UseCustomDecimals = customDecimals;
5353
builder["set_session_timeout"] = 1; // Expire sessions quickly after test
5454
builder["set_allow_experimental_geo_types"] = 1; // Allow support for geo types
55+
builder["set_flatten_nested"] = 0; // Nested should be a single column, see https://clickhouse.com/docs/en/operations/settings/settings#flatten-nested
5556

5657
if (SupportedFeatures.HasFlag(Feature.Map))
5758
{
@@ -156,6 +157,8 @@ public static IEnumerable<DataTypeSample> GetDataTypeSamples()
156157

157158
yield return new DataTypeSample("Decimal128(30)", typeof(ClickHouseDecimal), "toDecimal128(1, 30)", new ClickHouseDecimal(BigInteger.Pow(10, 30), 30));
158159

160+
yield return new DataTypeSample("Nested(Id int, Comment String)", typeof(Tuple<int, string>[]), "CAST([(1, 'a')], 'Nested(Id int, Comment String)')", new[] { Tuple.Create(1, "a") });
161+
159162
if (SupportedFeatures.HasFlag(Feature.WideTypes))
160163
{
161164
// Code: 53. DB::Exception: Type mismatch in IN or VALUES section. Expected: Decimal(76, 25). Got: Decimal256:

‎ClickHouse.Client/ADO/Feature.cs

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,6 @@ public enum Feature
88
{
99
None = 0, // Special value
1010

11-
[Obsolete]
12-
[SinceVersion("20.1")]
13-
DateTime64 = 4,
14-
15-
[Obsolete]
16-
[SinceVersion("20.0")]
17-
Decimals = 8,
18-
19-
[Obsolete]
20-
[SinceVersion("20.0")]
21-
IPv6 = 16,
22-
2311
[SinceVersion("21.4")]
2412
UUIDParameters = 32,
2513

‎ClickHouse.Client/Formats/HttpParameterFormatter.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ internal static string Format(ClickHouseType type, object value, bool quote)
7777
case ArrayType arrayType when value is IEnumerable enumerable:
7878
return $"[{string.Join(",", enumerable.Cast<object>().Select(obj => Format(arrayType.UnderlyingType, obj, true)))}]";
7979

80+
case NestedType nestedType when value is IEnumerable enumerable:
81+
var values = enumerable.Cast<object>().Select(x => Format(nestedType, x, false));
82+
return $"[{string.Join(",", values)}]";
83+
8084
#if !NET462
8185
case TupleType tupleType when value is ITuple tuple:
8286
return $"({string.Join(",", tupleType.UnderlyingTypes.Select((x, i) => Format(x, tuple[i], true)))})";

‎ClickHouse.Client/Types/NestedType.cs

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
using System;
2+
using System.Collections;
3+
using System.Collections.Generic;
24
using System.Linq;
5+
using ClickHouse.Client.Formats;
36
using ClickHouse.Client.Types.Grammar;
47

58
namespace ClickHouse.Client.Types;
@@ -31,4 +34,31 @@ private static SyntaxTreeNode ClearFieldName(SyntaxTreeNode node)
3134
var lastSpaceIndex = name.LastIndexOf(' ');
3235
return lastSpaceIndex > 0 ? new SyntaxTreeNode { Value = name.Substring(lastSpaceIndex + 1) } : node;
3336
}
37+
38+
public override object Read(ExtendedBinaryReader reader)
39+
{
40+
var length = reader.Read7BitEncodedInt();
41+
var data = Array.CreateInstance(base.FrameworkType, length);
42+
for (var i = 0; i < length; i++)
43+
{
44+
data.SetValue(ClearDBNull(base.Read(reader)), i);
45+
}
46+
return data;
47+
}
48+
49+
public override void Write(ExtendedBinaryWriter writer, object value)
50+
{
51+
if (value is null || value is DBNull)
52+
{
53+
writer.Write7BitEncodedInt(0);
54+
return;
55+
}
56+
57+
var collection = (IList)value;
58+
writer.Write7BitEncodedInt(collection.Count);
59+
for (var i = 0; i < collection.Count; i++)
60+
{
61+
base.Write(writer, collection[i]);
62+
}
63+
}
3464
}

‎ClickHouse.Client/Utility/EnumToLowercaseStringCached.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ namespace ClickHouse.Client.Utility;
55
internal static class EnumToLowercaseStringCached<T>
66
where T : Enum
77
{
8-
private static readonly ConcurrentDictionary<T, string> values = new ConcurrentDictionary<T, string>();
8+
private static readonly ConcurrentDictionary<T, string> Values = new ConcurrentDictionary<T, string>();
99

1010
public static string ToString(T value)
1111
{
12-
return values.GetOrAdd(value, (v) => v.ToString().ToLowerInvariant());
12+
return Values.GetOrAdd(value, (v) => v.ToString().ToLowerInvariant());
1313
}
1414
}

0 commit comments

Comments
 (0)
0