From b36c390b3f9fef7835ef35c21f7c6142590c6016 Mon Sep 17 00:00:00 2001 From: Grigoriy Pisarenko Date: Wed, 21 Aug 2024 11:21:46 +0000 Subject: [PATCH] Fixed internal error for missing format --- .../providers/s3/provider/yql_s3_datasink.cpp | 9 ++++++- ydb/tests/fq/s3/test_insert.py | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_datasink.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_datasink.cpp index 4e83e373ce01..c631f86a3c23 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_datasink.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_datasink.cpp @@ -90,12 +90,19 @@ class TS3DataSinkProvider : public TDataProviderBase { TExprNode::TPtr RewriteIO(const TExprNode::TPtr& write, TExprContext& ctx) override { const TS3Write w(write); auto settings = write->Tail().ChildrenList(); + + TExprNode::TPtr format = ExtractFormat(settings); + if (!format) { + ctx.AddError(TIssue(ctx.GetPosition(write->Pos()), "Missing format - please use WITH FORMAT when writing into S3")); + return nullptr; + } + return Build(ctx, w.Pos()) .World(w.World()) .DataSink(w.DataSink()) .Target() .Path(write->Child(2U)->Head().Tail().HeadPtr()) - .Format(ExtractFormat(settings)) + .Format(std::move(format)) .Settings(ctx.NewList(w.Pos(), std::move(settings))) .Build() .Input(write->ChildPtr(3)) diff --git a/ydb/tests/fq/s3/test_insert.py b/ydb/tests/fq/s3/test_insert.py index 9a9946e224d9..45fea6d79092 100644 --- a/ydb/tests/fq/s3/test_insert.py +++ b/ydb/tests/fq/s3/test_insert.py @@ -510,3 +510,28 @@ def test_insert_empty_object(self, kikimr, s3, client, unique_prefix): assert result_set.columns[0].type.type_id == ydb.Type.STRING assert len(result_set.rows) == 1 assert result_set.rows[0].items[0].text_value == "" + + @yq_all + @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) + def test_insert_without_format_error(self, kikimr, s3, client, unique_prefix): + resource = boto3.resource( + "s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key" + ) + + bucket = resource.Bucket("insert_bucket") + bucket.create(ACL='public-read-write') + bucket.objects.all().delete() + + storage_connection_name = unique_prefix + "ibucket" + client.create_storage_connection(storage_connection_name, "insert_bucket") + + sql = f''' + insert into `{storage_connection_name}`.`/test/` + select * from AS_TABLE([<|foo:123, bar:"xxx"u|>,<|foo:456, bar:"yyy"u|>]); + ''' + + query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id + client.wait_query_status(query_id, fq.QueryMeta.FAILED) + issues = str(client.describe_query(query_id).result.query.issue) + + assert "Missing format - please use WITH FORMAT when writing into S3" in issues, "Incorrect Issues: " + issues