YQL-17485 initial version of splitter for generated pb files from grammar

ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make

@@ -7,6 +7,7 @@ IF (CPP_PROTO)
 
     SET(ANTLR_PACKAGE_NAME NJsonPathGenerated)
     SET(PROTOBUF_HEADER_PATH ${MODDIR})
+    SET(PROTOBUF_SUFFIX_PATH .pb.h)
     SET(LEXER_PARSER_NAMESPACE NALP)
 
 

ydb/library/yql/parser/proto_ast/gen/multiproto.py

@@ -0,0 +1,110 @@
+import os
+import sys
+
+NSPLIT=10
+
+def main(argv):
+    input_dir="."
+    output_dir="."
+    name=sys.argv[1]
+    if len(argv)>2:
+        input_dir=argv[2]
+    if len(argv)>3:
+        output_dir=argv[3]
+    print("name:",name)
+    print("input_dir:",input_dir)
+    print("output_dir:",output_dir)
+
+    in_h=os.path.join(input_dir,name + ".pb.h")
+    in_cpp=os.path.join(input_dir,name + ".pb.cc")
+    out_h=os.path.join(output_dir,name + ".pb.main.h")
+    out_cpp_template=os.path.join(output_dir,name + ".pb.I")
+
+    with open(out_h,"w") as out_file:
+       with open(in_h,"r") as in_file:
+           for line in in_file:
+              line = line.replace("inline void RegisterArenaDtor","void RegisterArenaDtor")
+              out_file.write(line) 
+
+    for i in range(0,2 + NSPLIT):
+        with open(out_cpp_template.replace("I","code" + str(i) + ".cc" if i<NSPLIT else "data.cc" if i==NSPLIT else "classes.h"),"w") as out_file:
+            with open(in_cpp,"r") as in_file:
+                line = line.replace("inline ","")
+                statement_index=0
+                current_types=set()
+                is_data_stmt=False
+                extern_data=False
+                extern_code=False
+                in_class_def=False
+                for line in in_file:
+                    if line.startswith("#include") and name + ".pb.h" in line:
+                        out_file.write('#include "' + name + '.pb.main.h"\n')
+                        if i!=NSPLIT+1:
+                            out_file.write('#include "' + name + '.pb.classes.h"\n')
+                        continue
+                    if line.strip()=="PROTOBUF_PRAGMA_INIT_SEG":
+                        out_file.write(line)
+                        break
+                    out_file.write(line)
+                for line in in_file:
+                    line=line.replace("inline ","")
+                    if line.startswith("#"):
+                        out_file.write(line)
+                        continue
+                    if line.startswith("namespace") or line.startswith("PROTOBUF_NAMESPACE_OPEN"):
+                        open_namespace = True
+                        out_file.write(line)
+                        continue
+                    if (line.startswith("}  // namespace") or line.startswith("PROTOBUF_NAMESPACE_CLOSE")) and open_namespace:
+                        open_namespace = False
+                        out_file.write(line)
+                        continue
+                    if in_class_def:
+                        if (i==NSPLIT+1):
+                           out_file.write(line)
+                        if line.startswith("};"):
+                           in_class_def=False
+                        continue
+                    if line.startswith("PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT"):
+                        type_name=line.split(" ")[2]
+                        if type_name in current_types:
+                            out_file.write(line)
+                        continue
+                    if line.startswith("static ") or (line.startswith("const ") and ("[]" in line or "=" in line)) or line.startswith("PROTOBUF_ATTRIBUTE_WEAK"):
+                        is_data_stmt = True
+                        extern_data = "file_level_metadata" in line or ("descriptor_table" in line and "once" in line)
+                        extern_code = line.startswith("PROTOBUF_ATTRIBUTE_WEAK")
+                    if line.startswith("class"):
+                       in_class_def=True
+                       if i==NSPLIT+1:
+                          out_file.write(line)
+                       continue
+                    if not is_data_stmt and (statement_index % NSPLIT)==i:
+                        if line.startswith("struct"):
+                            current_types.add(line.split(" ")[1])
+                        out_file.write(line)
+                    if is_data_stmt and i==NSPLIT:
+                        if extern_data:
+                           line = line.replace("static ","")
+                        out_file.write(line)
+                    if is_data_stmt and i<NSPLIT:
+                        if extern_data or extern_code:
+                            if extern_data:
+                                line = "extern " + line.replace("static ","").replace(" = {",";")
+                            if extern_code:
+                                if not "PROTOBUF_ATTRIBUTE_WEAK" in line:
+                                    continue
+                                line = "extern " + line.replace(" {",";")
+                            out_file.write(line)
+                            extern_data = False
+                            extern_code = False
+                    if line.startswith("}"):
+                        if is_data_stmt:
+                            is_data_stmt=False
+                            extern_data = False
+                            extern_code = False
+                        else:
+                            statement_index += 1
+
+if __name__ == "__main__":
+    main(sys.argv)

ydb/library/yql/parser/proto_ast/gen/v0/ya.make

@@ -1,48 +1,36 @@
-PROTO_LIBRARY()
+LIBRARY()
 
-EXCLUDE_TAGS(GO_PROTO JAVA_PROTO)
+PEERDIR (
+    ydb/library/yql/parser/proto_ast/gen/v0_proto_split
+)
 
-IF (CPP_PROTO)
+SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR})
+SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates)
+SET(sql_grammar ${ARCADIA_ROOT}/ydb/library/yql/sql/v0/SQL.g)
 
-    SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR})
-    SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates)
-    SET(sql_grammar ${ARCADIA_ROOT}/ydb/library/yql/sql/v0/SQL.g)
+SET(ANTLR_PACKAGE_NAME NSQLGenerated)
+SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v0_proto_split)
+SET(PROTOBUF_SUFFIX_PATH .pb.main.h)
+SET(LEXER_PARSER_NAMESPACE NALP)
 
-    SET(ANTLR_PACKAGE_NAME NSQLGenerated)
-    SET(PROTOBUF_HEADER_PATH ${MODDIR})
-    SET(LEXER_PARSER_NAMESPACE NALP)
 
+CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg)
 
-    CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg)
-    CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/protobuf/protobuf.stg.in ${antlr_templates}/protobuf/protobuf.stg)
+NO_COMPILER_WARNINGS()
 
-    RUN_ANTLR(
-        ${sql_grammar}
-        -lib .
-        -fo ${antlr_output}
-        -language protobuf
-        IN ${sql_grammar} ${antlr_templates}/protobuf/protobuf.stg
-        OUT_NOAUTO SQLParser.proto
-        CWD ${antlr_output}
-    )
+INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl)
 
-    NO_COMPILER_WARNINGS()
-
-    INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl)
-
-    RUN_ANTLR(
-        ${sql_grammar}
-        -lib .
-        -fo ${antlr_output}
-        IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg
-        OUT SQLParser.cpp SQLLexer.cpp SQLParser.h SQLLexer.h
-        OUTPUT_INCLUDES
-        SQLParser.pb.h
-        ${STG_INCLUDES}
-        CWD ${antlr_output}
-    )
-ENDIF()
-
-SRCS(SQLParser.proto)
+RUN_ANTLR(
+    ${sql_grammar}
+    -lib .
+    -fo ${antlr_output}
+    IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg
+    OUT SQLParser.cpp SQLLexer.cpp SQLParser.h SQLLexer.h
+    OUTPUT_INCLUDES
+    ${PROTOBUF_HEADER_PATH}/SQLParser.pb.main.h
+    ${STG_INCLUDES}
+    CWD ${antlr_output}
+)
 
 END()
+

ydb/library/yql/parser/proto_ast/gen/v0_proto/ya.make.gen

@@ -0,0 +1,29 @@
+PROTO_LIBRARY()
+
+IF (GEN_PROTO)
+
+    SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR})
+    SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates)
+    SET(sql_grammar ${ARCADIA_ROOT}/ydb/library/yql/sql/v0/SQL.g)
+
+    SET(ANTLR_PACKAGE_NAME NSQLGenerated)
+
+    CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/protobuf/protobuf.stg.in ${antlr_templates}/protobuf/protobuf.stg)
+
+    RUN_ANTLR(
+        ${sql_grammar}
+        -lib .
+        -fo ${antlr_output}
+        -language protobuf
+        IN ${sql_grammar} ${antlr_templates}/protobuf/protobuf.stg
+        OUT_NOAUTO SQLParser.proto
+        CWD ${antlr_output}
+    )
+
+ENDIF()
+
+SRCS(SQLParser.proto)
+
+EXCLUDE_TAGS(GO_PROTO JAVA_PROTO)
+
+END()

ydb/library/yql/parser/proto_ast/gen/v0_proto_split/update.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -eux
+cp ../v0_proto/ya.make.gen ../v0_proto/ya.make
+yag make ../v0_proto --add-result=".h" --add-result=".cc"
+rm ../v0_proto/ya.make
+python3 ../multiproto.py SQLParser ../v0_proto .
+

ydb/library/yql/parser/proto_ast/gen/v0_proto_split/ya.make

@@ -0,0 +1,79 @@
+LIBRARY()
+
+OWNER(g:yql g:yql_ydb_core)
+
+SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR})
+SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates)
+SET(sql_grammar ${ARCADIA_ROOT}/ydb/library/yql/sql/v0/SQL.g)
+
+SET(ANTLR_PACKAGE_NAME NSQLGenerated)
+
+CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/protobuf/protobuf.stg.in ${antlr_templates}/protobuf/protobuf.stg)
+
+RUN_ANTLR(
+    ${sql_grammar}
+    -lib .
+    -fo ${antlr_output}
+    -language protobuf
+    IN ${sql_grammar} ${antlr_templates}/protobuf/protobuf.stg
+    OUT_NOAUTO SQLParser.proto
+    CWD ${antlr_output}
+)
+
+IF (USE_VANILLA_PROTOC)
+    SET(PROTOC_PATH contrib/tools/protoc_std)
+ELSE()
+    SET(PROTOC_PATH contrib/tools/protoc)
+ENDIF()
+
+
+RUN_PROGRAM(
+    $PROTOC_PATH -I=$CURDIR -I=$ARCADIA_ROOT -I=$ARCADIA_BUILD_ROOT -I=$ARCADIA_ROOT/contrib/libs/protobuf/src
+    --cpp_out=$ARCADIA_BUILD_ROOT --cpp_styleguide_out=$ARCADIA_BUILD_ROOT
+    --plugin=protoc-gen-cpp_styleguide=contrib/tools/protoc/plugins/cpp_styleguide
+    SQLParser.proto
+    IN SQLParser.proto
+    TOOL contrib/tools/protoc/plugins/cpp_styleguide
+    OUT_NOAUTO SQLParser.pb.h SQLParser.pb.cc
+    CWD $ARCADIA_BUILD_ROOT
+)
+
+RUN_PYTHON3(
+    ${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/gen/multiproto.py SQLParser
+    IN SQLParser.pb.h
+    IN SQLParser.pb.cc
+    OUT_NOAUTO
+    SQLParser.pb.code0.cc
+    SQLParser.pb.code1.cc
+    SQLParser.pb.code2.cc
+    SQLParser.pb.code3.cc
+    SQLParser.pb.code4.cc
+    SQLParser.pb.code5.cc
+    SQLParser.pb.code6.cc
+    SQLParser.pb.code7.cc
+    SQLParser.pb.code8.cc
+    SQLParser.pb.code9.cc
+    SQLParser.pb.data.cc
+    SQLParser.pb.classes.h
+    SQLParser.pb.main.h
+    CWD $ARCADIA_BUILD_ROOT/ydb/library/yql/parser/proto_ast/gen/v0_proto_split
+)
+
+PEERDIR(contrib/libs/protobuf)
+
+SRCS(
+    SQLParser.pb.code0.cc
+    SQLParser.pb.code1.cc
+    SQLParser.pb.code2.cc
+    SQLParser.pb.code3.cc
+    SQLParser.pb.code4.cc
+    SQLParser.pb.code5.cc
+    SQLParser.pb.code6.cc
+    SQLParser.pb.code7.cc
+    SQLParser.pb.code8.cc
+    SQLParser.pb.code9.cc
+    SQLParser.pb.data.cc
+)
+
+END()
+

ydb/library/yql/parser/proto_ast/gen/v1/ya.make

@@ -1,15 +1,16 @@
 LIBRARY()
 
 PEERDIR (
-    ydb/library/yql/parser/proto_ast/gen/v1_proto
+    ydb/library/yql/parser/proto_ast/gen/v1_proto_split
 )
 
 SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR})
 SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates)
 SET(sql_grammar ${antlr_output}/SQLv1.g)
 
 SET(ANTLR_PACKAGE_NAME NSQLv1Generated)
-SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto)
+SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto_split)
+SET(PROTOBUF_SUFFIX_PATH .pb.main.h)
 
 SET(LEXER_PARSER_NAMESPACE NALPDefault)
 
@@ -36,7 +37,7 @@ RUN_ANTLR(
     IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg
     OUT SQLv1Parser.cpp SQLv1Lexer.cpp SQLv1Parser.h SQLv1Lexer.h
     OUTPUT_INCLUDES
-    ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.h
+    ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.main.h
     ${STG_INCLUDES}
     CWD ${antlr_output}
 )

ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make

@@ -1,15 +1,16 @@
 LIBRARY()
 
 PEERDIR (
-    ydb/library/yql/parser/proto_ast/gen/v1_proto
+    ydb/library/yql/parser/proto_ast/gen/v1_proto_split
 )
 
 SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR})
 SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates)
 SET(sql_grammar ${antlr_output}/SQLv1.g)
 
 SET(ANTLR_PACKAGE_NAME NSQLv1Generated)
-SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto)
+SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto_split)
+SET(PROTOBUF_SUFFIX_PATH .pb.main.h)
 
 SET(LEXER_PARSER_NAMESPACE NALPAnsi)
 
@@ -36,7 +37,7 @@ RUN_ANTLR(
     IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg
     OUT SQLv1Parser.cpp SQLv1Lexer.cpp SQLv1Parser.h SQLv1Lexer.h
     OUTPUT_INCLUDES
-    ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.h
+    ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.main.h
     ${STG_INCLUDES}
     CWD ${antlr_output}
 )

ydb/library/yql/parser/proto_ast/gen/v1_proto_split/update.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -eux
+cp ../v1_proto/ya.make.gen ../v1_proto/ya.make
+yag make ../v1_proto --add-result=".h" --add-result=".cc"
+rm ../v1_proto/ya.make
+python3 ../multiproto.py SQLv1Parser ../v1_proto .
+