Allow use DECIMAL(22,9) in EXTERNAL TABLE (ydb-platform#7945)

Hor911 · Aug 29, 2024 · e17a4b8 · e17a4b8
1 parent 3985116
commit e17a4b8
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 1 deletion.
diff --git a/ydb/core/kqp/provider/read_attributes_utils.cpp b/ydb/core/kqp/provider/read_attributes_utils.cpp
@@ -203,7 +203,19 @@ static Ydb::Type CreateYdbType(const NKikimr::NScheme::TTypeInfo& typeInfo, bool
         auto& item = notNull
             ? ydbType
             : *ydbType.mutable_optional_type()->mutable_item();
-        item.set_type_id((Ydb::Type::PrimitiveTypeId)typeInfo.GetTypeId());
+        //
+        // DECIMAL is PrimitiveType with (22,9) defaults in Scheme
+        // and separate (non-primitive) type everywhere else
+        //
+        // NKikimr::NScheme::NTypeIds::Decimal is omitted in public API intentionally
+        //
+        if (typeInfo.GetTypeId() == NKikimr::NScheme::NTypeIds::Decimal) {
+            auto* decimal = item.mutable_decimal_type();
+            decimal->set_precision(NKikimr::NScheme::DECIMAL_PRECISION);
+            decimal->set_scale(NKikimr::NScheme::DECIMAL_SCALE);
+        } else {
+            item.set_type_id((Ydb::Type::PrimitiveTypeId)typeInfo.GetTypeId());
+        }
     }
     return ydbType;
 }

diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common_external_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_common_external_table.cpp
@@ -113,6 +113,14 @@ std::pair<TExternalTableInfo::TPtr, TMaybe<TString>> CreateExternalTable(
         }
 
         auto typeName = NMiniKQL::AdaptLegacyYqlType(col.GetType());
+        if (typeName == "Decimal(22,9)"sv) {
+            //
+            // typename is reformatted as above
+            // should discard (SCALE,PRECISION)
+            // they are validated to be (22,9)
+            //
+            typeName = "Decimal"sv;
+        }
         const NScheme::IType* type = typeRegistry->GetType(typeName);
 
         NScheme::TTypeInfo typeInfo;

diff --git a/ydb/tests/fq/s3/test_bindings_1.py b/ydb/tests/fq/s3/test_bindings_1.py
@@ -325,3 +325,56 @@ def test_binding_with_backslash_in_location(self, s3, client, unique_prefix):
         assert result_set.columns[0].name == "data"
         assert len(result_set.rows) == 1
         assert result_set.rows[0].items[0].text_value == "test"
+
+    @yq_all
+    @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True)
+    def test_decimal_binding(self, kikimr, s3, client, unique_prefix):
+        resource = boto3.resource(
+            "s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key"
+        )
+
+        bucket = resource.Bucket("decimal_binding")
+        bucket.create(ACL='public-read')
+
+        s3_client = boto3.client(
+            "s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key"
+        )
+
+        row = R'''
+            { "a": 0.1111111115, "b": 0.111111115 },
+            { "a": 0.1111111115, "b": 0.111111115 },
+        '''
+        s3_client.put_object(Body=row, Bucket='decimal_binding', Key='abc.json', ContentType='text/json')
+
+        kikimr.control_plane.wait_bootstrap(1)
+        connection_response = client.create_storage_connection(unique_prefix + "_decc", "decimal_binding")
+
+        aType = ydb.Column(name="a", type=ydb.Type(decimal_type=ydb.DecimalType(precision=22, scale=9)))
+        bType = ydb.Column(name="b", type=ydb.Type(decimal_type=ydb.DecimalType(precision=22, scale=9)))
+
+        storage_binding_name = unique_prefix + "_decb"
+        client.create_object_storage_binding(
+            name=storage_binding_name,
+            path="abc.json",
+            format="json_each_row",
+            connection_id=connection_response.result.connection_id,
+            columns=[aType, bType],
+        )
+
+        sql = fR'''
+            SELECT Unwrap(SUM(a)) as a, Unwrap(SUM(b)) as b
+            FROM bindings.{storage_binding_name};
+            '''
+
+        query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id
+        client.wait_query_status(query_id, fq.QueryMeta.COMPLETED)
+
+        data = client.get_result_data(query_id)
+        result_set = data.result.result_set
+        logging.debug(str(result_set))
+        assert len(result_set.columns) == 2
+        assert len(result_set.rows) == 1
+        assert result_set.columns[0].type == ydb.Type(decimal_type=ydb.DecimalType(precision=35, scale=9))
+        assert result_set.columns[1].type == ydb.Type(decimal_type=ydb.DecimalType(precision=35, scale=9))
+        assert result_set.rows[0].items[0].low_128 == 222222222
+        assert result_set.rows[0].items[1].low_128 == 222222230