Reject unknown type names inside containers

README.md

@@ -185,6 +185,7 @@ still might break.
 * String
 * LowCardinality(String) or LowCardinality(FixedString(N))
 * Tuple
+* Nested, read and written as Array(Tuple(...))
 * UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64
 * UInt128, Int128
 * UUID

clickhouse/columns/factory.cpp

@@ -166,7 +166,8 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti
             return CreateTerminalColumn(ast);
         }
 
-        case TypeAst::Tuple: {
+        case TypeAst::Tuple:
+        case TypeAst::Nested: {
             std::vector<ColumnRef> columns;
             std::vector<std::string> names;
 
@@ -185,10 +186,12 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti
                 }
             }
 
-            if (any_named) {
-                return std::make_shared<ColumnTuple>(columns, std::move(names));
+            auto tuple = any_named ? std::make_shared<ColumnTuple>(columns, std::move(names))
+                                   : std::make_shared<ColumnTuple>(columns);
+            if (ast.meta == TypeAst::Nested) {
+                return std::make_shared<ColumnArray>(tuple);
             }
-            return std::make_shared<ColumnTuple>(columns);
+            return tuple;
         }
 
         case TypeAst::Enum: {

clickhouse/types/type_parser.cpp

@@ -31,6 +31,7 @@ bool TypeAst::operator==(const TypeAst & other) const {
 
 static const std::unordered_map<std::string, Type::Code> kTypeCode = {
     { "Void",        Type::Void },
+    { "Nothing",     Type::Void },
     { "Int8",        Type::Int8 },
     { "Int16",       Type::Int16 },
     { "Int32",       Type::Int32 },
@@ -146,17 +147,41 @@ static TypeAst::Meta GetTypeMeta(const StringView& name) {
         return TypeAst::Map;
     }
 
+    if (name == "Nested") {
+        return TypeAst::Nested;
+    }
+
     return TypeAst::Terminal;
 }
 
+static bool IsKnownVoidName(const std::string& name) {
+    return CompateStringsCaseInsensitive(name, std::string_view("void")) == 0
+        || CompateStringsCaseInsensitive(name, std::string_view("nothing")) == 0;
+}
+
+static bool ElementsAreTypes(TypeAst::Meta meta) {
+    return meta == TypeAst::Array || meta == TypeAst::Nullable || meta == TypeAst::Tuple
+        || meta == TypeAst::Map || meta == TypeAst::LowCardinality || meta == TypeAst::Nested;
+}
+
 bool ValidateAST(const TypeAst& ast) {
-    // Void terminal that is not actually "void" produced when unknown type is encountered.
-    if (ast.meta == TypeAst::Terminal
-            && ast.code == Type::Void
-            && CompateStringsCaseInsensitive(ast.name, std::string_view("void")) != 0)
-        //throw UnimplementedError("Unsupported type: " + ast.name);
+    if (ast.name.empty()) {
+        return true;
+    }
+    if (ast.meta == TypeAst::Terminal && ast.code == Type::Void && !IsKnownVoidName(ast.name)) {
         return false;
-
+    }
+    if (ast.meta == TypeAst::SimpleAggregateFunction) {
+        return !ast.elements.empty() && ValidateAST(ast.elements.back());
+    }
+    if (!ElementsAreTypes(ast.meta)) {
+        return true;
+    }
+    for (const auto& element : ast.elements) {
+        if (!ValidateAST(element)) {
+            return false;
+        }
+    }
     return true;
 }
 

clickhouse/types/type_parser.h

@@ -22,7 +22,8 @@ struct TypeAst {
         Enum,
         LowCardinality,
         SimpleAggregateFunction,
-        Map
+        Map,
+        Nested
     };
 
     /// Type's category.

docs/index.mdx

@@ -164,6 +164,7 @@ client.Select(
 - `Nullable(T)`
 - `Array(T)`
 - `Tuple`
+- `Nested`, read and written as `Array(Tuple(...))`
 - `Map`
 - `IPv4`, `IPv6`
 - `Point`, `Ring`, `Polygon`, `MultiPolygon`

ut/CreateColumnByType_ut.cpp

@@ -9,12 +9,33 @@
 
 #include <gtest/gtest.h>
 
+#include <cstdlib>
+#include <iostream>
+#include <string>
 #include <tuple>
 
+#if !defined(_WIN32)
+#include <sys/resource.h>
+#endif
+
 namespace {
 using namespace clickhouse;
 }
 
+#if !defined(_WIN32)
+namespace {
+long MaxResidentKiB() {
+    struct rusage usage {};
+    getrusage(RUSAGE_SELF, &usage);
+#if defined(__APPLE__)
+    return usage.ru_maxrss / 1024;
+#else
+    return usage.ru_maxrss;
+#endif
+}
+}
+#endif
+
 TEST(CreateColumnByType, CreateSimpleAggregateFunction) {
     auto col = CreateColumnByType("SimpleAggregateFunction(funt, Int32)");
 
@@ -280,3 +301,62 @@ INSTANTIATE_TEST_SUITE_P(Nested, CreateColumnByTypeWithName, ::testing::Values(
     "Array(Nullable(LowCardinality(FixedString(10000))))",
     "Array(Enum8('ONE' = 1, 'TWO' = 2))"
 ));
+
+class CreateColumnByNestedType : public ::testing::TestWithParam<std::tuple<const char*, const char*>> {};
+
+TEST_P(CreateColumnByNestedType, CreateColumnByType) {
+    const auto & [type_name, expected_name] = GetParam();
+    const auto col = CreateColumnByType(type_name);
+    ASSERT_NE(nullptr, col) << "CreateColumnByType returned nullptr for " << type_name;
+    EXPECT_EQ(expected_name, col->GetType().GetName());
+}
+
+INSTANTIATE_TEST_SUITE_P(Nested, CreateColumnByNestedType, ::testing::Values(
+    std::make_tuple("Nested(key String, value UInt8)", "Array(Tuple(key String, value UInt8))"),
+    std::make_tuple("Array(Nested(key String, value UInt8))", "Array(Array(Tuple(key String, value UInt8)))"),
+    std::make_tuple("Map(String, Nested(key String, value UInt8))", "Map(String, Array(Tuple(key String, value UInt8)))"),
+    std::make_tuple("Nested(a Nested(b String))", "Array(Tuple(a Array(Tuple(b String))))")
+));
+
+TEST(CreateColumnByTypeDeathTest, DeepNestedDoesNotCopyAst) {
+#if defined(_WIN32)
+    GTEST_SKIP() << "peak RSS is read with getrusage";
+#else
+    const size_t depth = 200;
+    const size_t width = 8000;
+    std::string type_name;
+    for (size_t i = 0; i < depth; ++i) {
+        type_name += "Nested(n ";
+    }
+    type_name += "Nested(f0 UInt8";
+    for (size_t i = 1; i < width; ++i) {
+        type_name += ", f" + std::to_string(i) + " UInt8";
+    }
+    type_name += ")";
+    type_name.append(depth, ')');
+
+    const auto create_within_peak_budget = [&type_name] {
+        const long before = MaxResidentKiB();
+        const bool created = CreateColumnByType(type_name) != nullptr;
+        const long growth = MaxResidentKiB() - before;
+        std::cerr << "peak RSS grew by " << growth << " KiB";
+        std::exit(created && growth < 16 * 1024 ? EXIT_SUCCESS : EXIT_FAILURE);
+    };
+    EXPECT_EXIT(create_within_peak_budget(), ::testing::ExitedWithCode(EXIT_SUCCESS), "");
+#endif
+}
+
+TEST(CreateColumnByType, UnknownTypeInsideContainer) {
+    ASSERT_EQ(nullptr, CreateColumnByType("Array(Int256)"));
+    ASSERT_EQ(nullptr, CreateColumnByType("Array(Nested(key String, value Dynamic))"));
+}
+
+TEST(CreateColumnByType, Nothing) {
+    ASSERT_NE(nullptr, CreateColumnByType("Array(Nothing)"));
+    ASSERT_NE(nullptr, CreateColumnByType("Nullable(Nothing)"));
+}
+
+TEST(CreateColumnByType, ParameterisedJSON) {
+    ASSERT_NE(nullptr, CreateColumnByType("JSON(max_dynamic_paths=1)"));
+    ASSERT_NE(nullptr, CreateColumnByType("JSON(a Int256, SKIP b)"));
+}

ut/client_ut.cpp

@@ -1823,6 +1823,147 @@ const auto LocalHostEndpoint = ClientOptions()
         .SetPassword(       getEnvOrDefault("CLICKHOUSE_PASSWORD", ""))
         .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB",       "default"));
 
+namespace {
+
+using KeyValuePairs = std::vector<std::pair<std::string, uint8_t>>;
+
+KeyValuePairs KeyValuePairsOf(const ColumnRef& column) {
+    auto tuples = column->AsStrict<ColumnTuple>();
+    auto keys = (*tuples)[0]->AsStrict<ColumnString>();
+    auto values = (*tuples)[1]->AsStrict<ColumnUInt8>();
+    KeyValuePairs pairs;
+    for (size_t i = 0; i < tuples->Size(); ++i) {
+        pairs.emplace_back(std::string(keys->At(i)), values->At(i));
+    }
+    return pairs;
+}
+
+std::vector<KeyValuePairs> ArraysOfKeyValuePairsOf(const ColumnRef& column) {
+    auto arrays = column->AsStrict<ColumnArray>();
+    std::vector<KeyValuePairs> arrays_of_pairs;
+    for (size_t i = 0; i < arrays->Size(); ++i) {
+        arrays_of_pairs.push_back(KeyValuePairsOf(arrays->GetAsColumn(i)));
+    }
+    return arrays_of_pairs;
+}
+
+}
+
+TEST_P(ClientCase, ArrayOfNested_SelectServerTypedColumn) {
+    client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested");
+    client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested "
+                     "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))");
+    client_->Execute("INSERT INTO test_clickhouse_cpp_array_of_nested VALUES "
+                     "(1, [[('a', 1), ('b', 2)], []]), (2, []), (3, [[('c', 3)]])");
+
+    std::vector<std::vector<KeyValuePairs>> rows;
+    std::string type_name;
+    client_->Select("SELECT id, arrays_of_pairs FROM test_clickhouse_cpp_array_of_nested ORDER BY id", [&](const Block& block) {
+        if (block.GetRowCount() == 0) {
+            return;
+        }
+        type_name = block[1]->GetType().GetName();
+        auto arrays_of_pairs = block[1]->AsStrict<ColumnArray>();
+        for (size_t row = 0; row < block.GetRowCount(); ++row) {
+            rows.push_back(ArraysOfKeyValuePairsOf(arrays_of_pairs->GetAsColumn(row)));
+        }
+    });
+
+    EXPECT_EQ("Array(Array(Tuple(key String, value UInt8)))", type_name);
+    const std::vector<std::vector<KeyValuePairs>> expected = {
+        {{{"a", 1}, {"b", 2}}, {}},
+        {},
+        {{{"c", 3}}},
+    };
+    EXPECT_EQ(expected, rows);
+}
+
+TEST_P(ClientCase, ArrayOfNested_ReinsertSelectedRows) {
+    client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested_source");
+    client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested_copy");
+    client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested_source "
+                     "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))");
+    client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested_copy "
+                     "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))");
+    client_->Execute("INSERT INTO test_clickhouse_cpp_array_of_nested_source VALUES (1, [[('a', 1), ('b', 2)], []]), (2, [])");
+
+    std::vector<Block> selected_blocks;
+    client_->Select("SELECT id, arrays_of_pairs FROM test_clickhouse_cpp_array_of_nested_source ORDER BY id", [&](const Block& block) {
+        if (block.GetRowCount() > 0) {
+            selected_blocks.push_back(block);
+        }
+    });
+    for (const auto& block : selected_blocks) {
+        client_->Insert("test_clickhouse_cpp_array_of_nested_copy", block);
+    }
+
+    std::vector<std::string> rendered_rows;
+    client_->Select("SELECT toString(arrays_of_pairs) FROM test_clickhouse_cpp_array_of_nested_copy ORDER BY id",
+                    [&](const Block& block) {
+        for (size_t row = 0; row < block.GetRowCount(); ++row) {
+            rendered_rows.emplace_back(block[0]->AsStrict<ColumnString>()->At(row));
+        }
+    });
+
+    const std::vector<std::string> expected = {"[[('a',1),('b',2)],[]]", "[]"};
+    EXPECT_EQ(expected, rendered_rows);
+}
+
+TEST_P(ClientCase, Nested_UnflattenedColumn) {
+    client_->Execute("SET flatten_nested = 0");
+    client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_nested_unflattened");
+    client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_nested_unflattened "
+                     "(id UInt8, pairs Nested(key String, value UInt8))");
+    client_->Execute("INSERT INTO test_clickhouse_cpp_nested_unflattened VALUES (1, [('a', 1), ('b', 2)]), (2, [])");
+
+    std::vector<KeyValuePairs> rows;
+    std::string type_name;
+    client_->Select("SELECT id, pairs FROM test_clickhouse_cpp_nested_unflattened ORDER BY id", [&](const Block& block) {
+        if (block.GetRowCount() == 0) {
+            return;
+        }
+        type_name = block[1]->GetType().GetName();
+        auto pairs = block[1]->AsStrict<ColumnArray>();
+        for (size_t row = 0; row < block.GetRowCount(); ++row) {
+            rows.push_back(KeyValuePairsOf(pairs->GetAsColumn(row)));
+        }
+    });
+
+    EXPECT_EQ("Array(Tuple(key String, value UInt8))", type_name);
+    const std::vector<KeyValuePairs> expected = {{{"a", 1}, {"b", 2}}, {}};
+    EXPECT_EQ(expected, rows);
+}
+
+TEST_P(ClientCase, EmptyArrayAndNullDecode) {
+    size_t rows = 0;
+    client_->Select("SELECT [] AS empty, NULL AS null_value, [NULL] AS nulls", [&](const Block& block) {
+        rows += block.GetRowCount();
+    });
+    EXPECT_EQ(1u, rows);
+}
+
+TEST_P(ClientCase, ParameterisedJSONDecodes) {
+    const auto & server_info = client_->GetServerInfo();
+    if (versionNumber(server_info) < versionNumber(25, 3)) {
+        GTEST_SKIP() << "Test is skipped since server '" << server_info << "' does not support the JSON type" << std::endl;
+    }
+
+    std::vector<std::string> documents;
+    client_->Select("SELECT CAST('{\"a\":1}', 'JSON(max_dynamic_paths=1)') AS document "
+                    "SETTINGS output_format_native_write_json_as_string = 1", [&](const Block& block) {
+        for (size_t row = 0; row < block.GetRowCount(); ++row) {
+            documents.emplace_back(block[0]->AsStrict<ColumnJSON>()->At(row));
+        }
+    });
+
+    const std::vector<std::string> expected = {"{\"a\":1}"};
+    EXPECT_EQ(expected, documents);
+}
+
+TEST_P(ClientCase, UnknownTypeInsideArrayIsReported) {
+    EXPECT_THROW(client_->Select("SELECT [CAST(1, 'Int256')] AS wide", [](const Block&) {}), UnimplementedError);
+}
+
 INSTANTIATE_TEST_SUITE_P(
     Client, ClientCase,
     ::testing::Values(