From f57399c359acdcc815f315a588f651d49e7bdf9f Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 15 Aug 2024 03:44:47 +0300 Subject: [PATCH 01/34] start adding grpc runtime --- libs/common/src/typegraph/runtimes/grpc.rs | 9 +++++++ libs/common/src/typegraph/runtimes/mod.rs | 4 +++ typegraph/core/src/conversion/runtimes.rs | 24 +++++++++++++++++- typegraph/core/src/runtimes/mod.rs | 29 +++++++++++++++++++--- typegraph/core/wit/typegraph.wit | 16 ++++++++++++ 5 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 libs/common/src/typegraph/runtimes/grpc.rs diff --git a/libs/common/src/typegraph/runtimes/grpc.rs b/libs/common/src/typegraph/runtimes/grpc.rs new file mode 100644 index 0000000000..ed715b2935 --- /dev/null +++ b/libs/common/src/typegraph/runtimes/grpc.rs @@ -0,0 +1,9 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GrpcRuntimeData { + pub url: String, +} diff --git a/libs/common/src/typegraph/runtimes/mod.rs b/libs/common/src/typegraph/runtimes/mod.rs index 82468ee2a1..11882f2f80 100644 --- a/libs/common/src/typegraph/runtimes/mod.rs +++ b/libs/common/src/typegraph/runtimes/mod.rs @@ -9,6 +9,7 @@ use substantial::SubstantialRuntimeData; use self::deno::DenoRuntimeData; use self::graphql::GraphQLRuntimeData; +use self::grpc::GrpcRuntimeData; use self::http::HTTPRuntimeData; use self::kv::KvRuntimeData; use self::prisma::PrismaRuntimeData; @@ -20,6 +21,7 @@ use self::wasm::WasmRuntimeData; pub mod deno; pub mod graphql; +pub mod grpc; pub mod http; pub mod kv; pub mod prisma; @@ -59,6 +61,7 @@ pub enum KnownRuntime { Typegraph(TypegraphRuntimeData), Substantial(SubstantialRuntimeData), Kv(KvRuntimeData), + Grpc(GrpcRuntimeData), } #[derive(Serialize, Deserialize, Clone, Debug, Default)] @@ -93,6 +96,7 @@ impl TGRuntime { KnownRuntime::Typegraph(_) => "typegraph", KnownRuntime::Substantial(_) => "substantial", KnownRuntime::Kv(_) => "kv", + KnownRuntime::Grpc(_) => "grpc", }, TGRuntime::Unknown(UnknownRuntime { name, .. }) => name, } diff --git a/typegraph/core/src/conversion/runtimes.rs b/typegraph/core/src/conversion/runtimes.rs index 7a2482495e..bc6f6c025c 100644 --- a/typegraph/core/src/conversion/runtimes.rs +++ b/typegraph/core/src/conversion/runtimes.rs @@ -8,10 +8,11 @@ use crate::runtimes::{ Runtime, TemporalMaterializer, WasmMaterializer, }; use crate::wit::core::{Artifact as WitArtifact, RuntimeId}; -use crate::wit::runtimes::{HttpMethod, KvMaterializer, MaterializerHttpRequest}; +use crate::wit::runtimes::{GrpcMaterializer, HttpMethod, KvMaterializer, MaterializerHttpRequest}; use crate::{typegraph::TypegraphContext, wit::runtimes::Effect as WitEffect}; use common::typegraph::runtimes::deno::DenoRuntimeData; use common::typegraph::runtimes::graphql::GraphQLRuntimeData; +use common::typegraph::runtimes::grpc::GrpcRuntimeData; use common::typegraph::runtimes::http::HTTPRuntimeData; use common::typegraph::runtimes::kv::KvRuntimeData; use common::typegraph::runtimes::python::PythonRuntimeData; @@ -404,6 +405,24 @@ impl MaterializerConverter for KvMaterializer { } } +impl MaterializerConverter for GrpcMaterializer { + fn convert( + &self, + c: &mut TypegraphContext, + runtime_id: RuntimeId, + effect: WitEffect, + ) -> Result<common::typegraph::Materializer> { + let runtime = c.register_runtime(runtime_id)?; + let data = serde_json::from_value(json!({})).map_err(|e| e.to_string())?; + Ok(Materializer { + name: "grpc".into(), + runtime, + effect: effect.into(), + data, + }) + } +} + pub fn convert_materializer( c: &mut TypegraphContext, mat: RawMaterializer, @@ -505,5 +524,8 @@ pub fn convert_runtime(_c: &mut TypegraphContext, runtime: Runtime) -> Result<Co .into()) } Runtime::Kv(d) => Ok(TGRuntime::Known(Rt::Kv(KvRuntimeData { url: d.url.clone() })).into()), + Runtime::Grpc(d) => { + Ok(TGRuntime::Known(Rt::Grpc(GrpcRuntimeData { url: d.url.clone() })).into()) + } } } diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index 679e208b60..28b61f789d 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -27,10 +27,11 @@ use crate::validation::types::validate_value; use crate::wit::aws::S3RuntimeData; use crate::wit::core::{FuncParams, MaterializerId, RuntimeId, TypeId as CoreTypeId}; use crate::wit::runtimes::{ - self as wit, BaseMaterializer, Error as TgError, GraphqlRuntimeData, HttpRuntimeData, - KvMaterializer, KvRuntimeData, MaterializerHttpRequest, PrismaLinkData, - PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, SubstantialRuntimeData, + self as wit, BaseMaterializer, Error as TgError, GraphqlRuntimeData, GrpcMaterializer, + GrpcRuntimeData, HttpRuntimeData, KvMaterializer, KvRuntimeData, MaterializerHttpRequest, + PrismaLinkData, PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, TemporalOperationData, TemporalRuntimeData, WasmRuntimeData, + PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, SubstantialRuntimeData, }; use crate::{typegraph::TypegraphContext, wit::runtimes::Effect as WitEffect}; use enum_dispatch::enum_dispatch; @@ -70,6 +71,7 @@ pub enum Runtime { S3(Rc<S3RuntimeData>), Substantial(Rc<SubstantialRuntimeData>), Kv(Rc<KvRuntimeData>), + Grpc(Rc<GrpcRuntimeData>), } #[derive(Debug, Clone)] @@ -191,6 +193,14 @@ impl Materializer { data: Rc::new(data).into(), } } + + fn grpc(runtime_id: RuntimeId, data: GrpcMaterializer, effect: wit::Effect) -> Self { + Self { + runtime_id, + effect, + data: Rc::new(data).into(), + } + } } #[derive(Debug, Clone)] @@ -210,6 +220,7 @@ pub enum MaterializerData { S3(Rc<S3Materializer>), Substantial(Rc<SubstantialMaterializer>), Kv(Rc<KvMaterializer>), + Grpc(Rc<GrpcMaterializer>), } macro_rules! prisma_op { @@ -695,4 +706,16 @@ impl crate::wit::runtimes::Guest for crate::Lib { let mat = Materializer::kv(base.runtime, data, base.effect); Ok(Store::register_materializer(mat)) } + + fn register_grpc_runtime(data: GrpcRuntimeData) -> Result<RuntimeId, wit::Error> { + Ok(Store::register_runtime(Runtime::Grpc(data.into()))) + } + + fn call_grpc_methode( + base: BaseMaterializer, + data: GrpcMaterializer, + ) -> Result<MaterializerId, wit::Error> { + let mat = Materializer::grpc(base.runtime, data, base.effect); + Ok(Store::register_materializer(mat)) + } } diff --git a/typegraph/core/wit/typegraph.wit b/typegraph/core/wit/typegraph.wit index 8fc4b1a5bd..ad44df1ebd 100644 --- a/typegraph/core/wit/typegraph.wit +++ b/typegraph/core/wit/typegraph.wit @@ -534,6 +534,22 @@ interface runtimes { } kv-operation: func(base: base-materializer, data: kv-materializer) -> result<materializer-id, error>; + + // Grpc + record grpc-runtime-data { + url: string + } + + register-grpc-runtime: func(data: grpc-runtime-data) -> result<runtime-id, error>; + + record grpc-materializer { + proto-file: string, + method: string, + payload: string, + endpoint: string, + } + + call-grpc-methode: func(base: base-materializer, data: grpc-materializer) -> result<materializer-id, error>; } interface aws { From 0dd7112bac55376a77c52e2370bf22d98c34c893 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 15 Aug 2024 15:57:04 +0300 Subject: [PATCH 02/34] add grpc runtime in typegate --- typegate/engine/00_runtime.js | 3 ++ typegate/engine/bindings.ts | 13 +++++++++ typegate/engine/runtime.d.ts | 16 +++++++++++ typegate/src/runtimes/grpc.ts | 52 +++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 typegate/src/runtimes/grpc.ts diff --git a/typegate/engine/00_runtime.js b/typegate/engine/00_runtime.js index 62497888d7..ea0cf0db2b 100644 --- a/typegate/engine/00_runtime.js +++ b/typegate/engine/00_runtime.js @@ -51,4 +51,7 @@ globalThis.Meta = { destroy: getOp("op_wit_wire_destroy"), handle: getOp("op_wit_wire_handle"), }, + grpc: { + gprcRegister: getOp("op_grpc_register"), + }, }; diff --git a/typegate/engine/bindings.ts b/typegate/engine/bindings.ts index 491f21bc75..29bc64867f 100644 --- a/typegate/engine/bindings.ts +++ b/typegate/engine/bindings.ts @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Elastic-2.0 import type { + GrpcRegisterInput, + GrpcRegisterOutput, ParsedDiff, PrismaApplyOut, PrismaCreateOut, @@ -453,3 +455,14 @@ export function archive(a0: ArchiveInp): ArchiveResult { return { Err: { message: err.toString() } }; } } + +export async function grpc_register( + a0: GrpcRegisterInput, +): Promise<GrpcRegisterOutput> { + try { + await Meta.grpc.gprcRegister(a0); + return "Ok"; + } catch (err) { + return { Err: { message: err.to_string() } }; + } +} diff --git a/typegate/engine/runtime.d.ts b/typegate/engine/runtime.d.ts index 3cd963550e..9c6a352f4f 100644 --- a/typegate/engine/runtime.d.ts +++ b/typegate/engine/runtime.d.ts @@ -61,6 +61,10 @@ type MetaNS = { args: WitWireReq, ) => Promise<WitWireHandleResponse>; }; + + grpc: { + grpcRegister: (inp: GrpcRegisterInput) => Promise<GrpcRegisterOutput>; + }; }; interface WasmInput { @@ -259,3 +263,15 @@ export type WitWireHandleResponse = | { HandlerErr: string; }; + +export type GrpcRegisterInput = { + url: string; +}; + +export type GrpcRegisterOutput = + | "Ok" + | { + Err: { + message: string; + }; + }; diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts new file mode 100644 index 0000000000..40587e8888 --- /dev/null +++ b/typegate/src/runtimes/grpc.ts @@ -0,0 +1,52 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { Runtime } from "@typegate/runtimes/Runtime.ts"; +import * as native from "native"; +import { ComputeStage } from "@typegate/engine/query_engine.ts"; +import { RuntimeInitParams } from "@typegate/types.ts"; +import { getLogger, Logger } from "@typegate/log.ts"; +import { TypeGraph } from "@typegate/typegraph/mod.ts"; +import { nativeVoid } from "@typegate/utils.ts"; + +const logger = getLogger(import.meta); + +interface GrpcRuntimeData { + url: string; +} + +export class GrpcRuntime extends Runtime { + private logger: Logger; + + private constructor(typegraphName: string) { + super(typegraphName); + this.logger = getLogger(`grpc: '${typegraphName}'`); + } + + static async init(params: RuntimeInitParams): Promise<Runtime> { + logger.info("Initliazing GrpcRuntime"); + logger.debug(`Init params: ${JSON.stringify(params)}`); + + const { typegraph, args } = params as RuntimeInitParams< + GrpcRuntimeData + >; + const typegraphName = TypeGraph.formatName(typegraph); + const instance = new GrpcRuntime(typegraphName); + instance.logger.info("registering GrpcRuntime"); + nativeVoid( + await native.grpc_register({ url: args.url }), + ); + throw new Error("Method not implemented."); + } + + deinit(): Promise<void> { + throw new Error("Method not implemented."); + } + materialize( + _stage: ComputeStage, + _waitlist: ComputeStage[], + _verbose: boolean, + ): ComputeStage[] | Promise<ComputeStage[]> { + throw new Error("Method not implemented."); + } +} From f0944ef8476eb718077cf4273ae42b9c69bea1af Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Tue, 20 Aug 2024 15:50:07 +0300 Subject: [PATCH 03/34] finish add grpc typegate --- Cargo.lock | 201 +++++++++++++++++--- libs/common/src/typegraph/runtimes/grpc.rs | 4 +- typegate/engine/00_runtime.js | 2 +- typegate/engine/Cargo.toml | 10 +- typegate/engine/bindings.ts | 14 +- typegate/engine/runtime.d.ts | 17 +- typegate/engine/src/runtimes.rs | 1 + typegate/engine/src/runtimes/grpc.rs | 206 +++++++++++++++++++++ typegate/src/runtimes/grpc.ts | 31 ++-- typegraph/core/src/conversion/runtimes.rs | 4 +- typegraph/core/src/runtimes/mod.rs | 6 +- typegraph/core/wit/typegraph.wit | 6 +- 12 files changed, 429 insertions(+), 73 deletions(-) create mode 100644 typegate/engine/src/runtimes/grpc.rs diff --git a/Cargo.lock b/Cargo.lock index 9cf0b129bc..af7ff00eb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -640,7 +640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.3.4", "bitflags 1.3.2", "bytes", "futures-util", @@ -655,7 +655,34 @@ dependencies = [ "pin-project-lite", "rustversion", "serde 1.0.204", - "sync_wrapper", + "sync_wrapper 0.1.2", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core 0.4.3", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde 1.0.204", + "sync_wrapper 1.0.1", "tower", "tower-layer", "tower-service", @@ -678,6 +705,26 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backoff" version = "0.4.0" @@ -1093,9 +1140,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "bzip2" @@ -2720,7 +2767,7 @@ dependencies = [ "http 1.1.0", "log", "num-bigint", - "prost", + "prost 0.11.9", "prost-build", "rand 0.8.5", "rusqlite", @@ -3164,7 +3211,7 @@ dependencies = [ "chrono", "futures", "num-bigint", - "prost", + "prost 0.11.9", "serde 1.0.204", "uuid", ] @@ -3184,7 +3231,7 @@ dependencies = [ "futures", "http 1.1.0", "log", - "prost", + "prost 0.11.9", "rand 0.8.5", "serde 1.0.204", "serde_json", @@ -5306,7 +5353,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.7", "tokio", "tower-service", "tracing", @@ -5363,6 +5410,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.4.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -8386,6 +8446,15 @@ dependencies = [ "prost-derive", ] +[[package]] +name = "prost" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" +dependencies = [ + "bytes", +] + [[package]] name = "prost-build" version = "0.11.9" @@ -8400,7 +8469,7 @@ dependencies = [ "multimap", "petgraph 0.6.5", "prettyplease 0.1.25", - "prost", + "prost 0.11.9", "prost-types", "regex", "syn 1.0.109", @@ -8427,7 +8496,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ - "prost", + "prost 0.11.9", ] [[package]] @@ -8438,7 +8507,7 @@ checksum = "562788060bcf2bfabe055194bd991ed2442457661744c88e0a0828ff9a08c08b" dependencies = [ "chrono", "inventory", - "prost", + "prost 0.11.9", "serde 1.0.204", "serde_derive", "serde_json", @@ -8452,7 +8521,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4dca8bcead3b728a6a7da017cc95e7f4cb2320ec4f6896bc593a1c4700f7328" dependencies = [ "heck 0.4.1", - "prost", + "prost 0.11.9", "prost-build", "prost-types", "quote", @@ -8465,7 +8534,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2377c5680f2342871823045052e791b4487f7c90aae17e0feaee24cf59578a34" dependencies = [ "chrono", - "prost", + "prost 0.11.9", "prost-build", "prost-types", "prost-wkt", @@ -8476,6 +8545,53 @@ dependencies = [ "serde_json", ] +[[package]] +name = "protobuf" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bcc343da15609eaecd65f8aa76df8dc4209d325131d8219358c0aaaebab0bf6" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror", +] + +[[package]] +name = "protobuf-json-mapping" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07993d14c66dfb74c639dc1b90381773b85cff66ef47bff35bad0778150a3aa" +dependencies = [ + "protobuf", + "protobuf-support", + "thiserror", +] + +[[package]] +name = "protobuf-parse" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b0e9b447d099ae2c4993c0cbb03c7a9d6c937b17f2d56cfc0b1550e6fcfdb76" +dependencies = [ + "anyhow", + "indexmap 2.2.6", + "log", + "protobuf", + "protobuf-support", + "tempfile", + "thiserror", + "which 4.4.2", +] + +[[package]] +name = "protobuf-support" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0766e3675a627c327e4b3964582594b0e8741305d628a98a5de75a1d15f99b9" +dependencies = [ + "thiserror", +] + [[package]] name = "psl" version = "0.1.0" @@ -9057,7 +9173,7 @@ dependencies = [ "serde 1.0.204", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", @@ -11154,6 +11270,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "synstructure" version = "0.12.6" @@ -11308,7 +11430,7 @@ dependencies = [ "temporal-sdk-core-protos", "thiserror", "tokio", - "tonic", + "tonic 0.8.3", "tower", "tracing", "url", @@ -11323,14 +11445,14 @@ dependencies = [ "anyhow", "base64 0.21.7", "derive_more", - "prost", + "prost 0.11.9", "prost-wkt", "prost-wkt-build", "prost-wkt-types", "serde 1.0.204", "serde_json", "thiserror", - "tonic", + "tonic 0.8.3", "tonic-build", ] @@ -11764,7 +11886,7 @@ checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.6.20", "base64 0.13.1", "bytes", "futures-core", @@ -11773,10 +11895,10 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", - "hyper-timeout", + "hyper-timeout 0.4.1", "percent-encoding", "pin-project", - "prost", + "prost 0.11.9", "prost-derive", "rustls-native-certs 0.6.3", "rustls-pemfile 1.0.4", @@ -11791,6 +11913,36 @@ dependencies = [ "tracing-futures", ] +[[package]] +name = "tonic" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38659f4a91aba8598d27821589f5db7dddd94601e7a01b1e485a50e5484c7401" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.7.5", + "base64 0.22.1", + "bytes", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.0", + "hyper-timeout 0.5.1", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.1", + "socket2 0.5.7", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic-build" version = "0.8.4" @@ -12149,6 +12301,7 @@ version = "0.4.8" dependencies = [ "anyhow", "base64 0.22.1", + "bytes", "common", "connection-string", "convert_case 0.6.0", @@ -12161,6 +12314,9 @@ dependencies = [ "mt_deno", "once_cell", "prisma-models", + "protobuf", + "protobuf-json-mapping", + "protobuf-parse", "psl", "query-connector", "query-core", @@ -12177,6 +12333,7 @@ dependencies = [ "temporal-sdk-core-protos", "thiserror", "tokio", + "tonic 0.12.1", "user-facing-errors", "wasmtime", "wasmtime-wasi", @@ -13247,7 +13404,7 @@ dependencies = [ "log", "naga", "once_cell", - "parking_lot 0.11.2", + "parking_lot 0.12.3", "profiling", "raw-window-handle", "ron", @@ -13289,7 +13446,7 @@ dependencies = [ "ndk-sys", "objc", "once_cell", - "parking_lot 0.11.2", + "parking_lot 0.12.3", "profiling", "range-alloc", "raw-window-handle", diff --git a/libs/common/src/typegraph/runtimes/grpc.rs b/libs/common/src/typegraph/runtimes/grpc.rs index ed715b2935..6ae04a5664 100644 --- a/libs/common/src/typegraph/runtimes/grpc.rs +++ b/libs/common/src/typegraph/runtimes/grpc.rs @@ -4,6 +4,4 @@ use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GrpcRuntimeData { - pub url: String, -} +pub struct GrpcRuntimeData {} diff --git a/typegate/engine/00_runtime.js b/typegate/engine/00_runtime.js index ea0cf0db2b..2864cc4fd4 100644 --- a/typegate/engine/00_runtime.js +++ b/typegate/engine/00_runtime.js @@ -52,6 +52,6 @@ globalThis.Meta = { handle: getOp("op_wit_wire_handle"), }, grpc: { - gprcRegister: getOp("op_grpc_register"), + callGrpcMethod: getOp("op_call_grpc_method"), }, }; diff --git a/typegate/engine/Cargo.toml b/typegate/engine/Cargo.toml index fbd99298c0..b18304f1e7 100644 --- a/typegate/engine/Cargo.toml +++ b/typegate/engine/Cargo.toml @@ -42,15 +42,21 @@ mt_deno.workspace = true deno_core.workspace = true common.workspace = true -wasmtime = { workspace = true, features = ["component-model"] } +wasmtime = { workspace = true, features = ["component-model"] } wasmtime-wasi.workspace = true shadow-rs.workspace = true +tonic = "0.12.1" +protobuf = "3.5.0" +protobuf-parse = "3.5.1" +bytes = "1.7.1" +protobuf-json-mapping = "3.5.1" + [dev-dependencies] env_logger.workspace = true [build-dependencies] shadow-rs.workspace = true -wasmtime = { workspace = true, features = ["component-model"] } +wasmtime = { workspace = true, features = ["component-model"] } zstd = "0.13.1" diff --git a/typegate/engine/bindings.ts b/typegate/engine/bindings.ts index 29bc64867f..10494f1e85 100644 --- a/typegate/engine/bindings.ts +++ b/typegate/engine/bindings.ts @@ -2,8 +2,7 @@ // SPDX-License-Identifier: Elastic-2.0 import type { - GrpcRegisterInput, - GrpcRegisterOutput, + GrpcCallMethodInput, ParsedDiff, PrismaApplyOut, PrismaCreateOut, @@ -456,13 +455,12 @@ export function archive(a0: ArchiveInp): ArchiveResult { } } -export async function grpc_register( - a0: GrpcRegisterInput, -): Promise<GrpcRegisterOutput> { +export async function call_grpc_method( + a0: GrpcCallMethodInput, +): Promise<string> { try { - await Meta.grpc.gprcRegister(a0); - return "Ok"; + return await Meta.grpc.callGrpcMethod(a0); } catch (err) { - return { Err: { message: err.to_string() } }; + return err.toString(); } } diff --git a/typegate/engine/runtime.d.ts b/typegate/engine/runtime.d.ts index 9c6a352f4f..7ecddd63f8 100644 --- a/typegate/engine/runtime.d.ts +++ b/typegate/engine/runtime.d.ts @@ -63,7 +63,7 @@ type MetaNS = { }; grpc: { - grpcRegister: (inp: GrpcRegisterInput) => Promise<GrpcRegisterOutput>; + callGrpcMethod: (inp: GrpcCallMethodInput) => Promise<string>; }; }; @@ -264,14 +264,9 @@ export type WitWireHandleResponse = HandlerErr: string; }; -export type GrpcRegisterInput = { - url: string; +export type GrpcCallMethodInput = { + proto_file: string; + method: string; + payload: string; + endpoint: string; }; - -export type GrpcRegisterOutput = - | "Ok" - | { - Err: { - message: string; - }; - }; diff --git a/typegate/engine/src/runtimes.rs b/typegate/engine/src/runtimes.rs index f6164f79a7..0e17de154d 100644 --- a/typegate/engine/src/runtimes.rs +++ b/typegate/engine/src/runtimes.rs @@ -1,6 +1,7 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 +pub mod grpc; pub mod prisma; pub mod temporal; pub mod wasm; diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs new file mode 100644 index 0000000000..61104185e6 --- /dev/null +++ b/typegate/engine/src/runtimes/grpc.rs @@ -0,0 +1,206 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +use std::{ + ops::Deref, + path::{Path, PathBuf}, + str::FromStr, +}; + +use protobuf::{ + descriptor::{FileDescriptorProto, MethodDescriptorProto}, + reflect::FileDescriptor, + MessageDyn, +}; + +use anyhow::{Context, Result}; +use bytes::{Buf, BufMut}; +use serde::Deserialize; +use tonic::codegen::http::uri::PathAndQuery; +use tonic::{ + client::Grpc, + codec::{Codec, DecodeBuf, Decoder, EncodeBuf, Encoder}, + transport::{Channel, Endpoint}, +}; +use tonic::{IntoRequest, Status}; + +#[rustfmt::skip] +use deno_core as deno_core; + +type DynRequest = Box<dyn MessageDyn>; +type DynResponse = Box<dyn MessageDyn>; + +#[derive(Clone)] +pub struct DynCodec { + file_descriptor: FileDescriptor, + method_descriptor_proto: MethodDescriptorProto, +} + +impl Codec for DynCodec { + type Encode = DynRequest; + type Decode = DynResponse; + + type Encoder = DynCodec; + type Decoder = DynCodec; + + fn encoder(&mut self) -> Self::Encoder { + self.clone() + } + + fn decoder(&mut self) -> Self::Decoder { + self.clone() + } +} + +impl Encoder for DynCodec { + type Item = DynRequest; + + type Error = Status; + + fn encode( + &mut self, + item: Self::Item, + dst: &mut EncodeBuf<'_>, + ) -> std::prelude::v1::Result<(), Self::Error> { + item.write_to_bytes_dyn() + .map(|buf| dst.put(buf.as_slice())) + .map_err(|err| Status::internal(format!("{:?}", err))) + } +} + +impl Decoder for DynCodec { + type Item = DynResponse; + type Error = Status; + + fn decode(&mut self, src: &mut DecodeBuf<'_>) -> Result<Option<Self::Item>, Self::Error> { + let buf = src.chunk(); + let length = buf.len(); + + let response_message = + get_relative_message_name(self.method_descriptor_proto.output_type()).unwrap(); + + let response = buf2response(buf, response_message, self.file_descriptor.clone()) + .map(Some) + .map_err(|err| Status::internal(format!("{:?}", err))); + src.advance(length); + response + } +} + +fn json2request( + json: String, + input_message: String, + file_descriptor: FileDescriptor, +) -> anyhow::Result<DynRequest> { + let msg_descriptor = file_descriptor + .message_by_package_relative_name(&input_message) + .with_context(|| format!("Input message {input_message} not found"))?; + let mut msg = msg_descriptor.new_instance(); + protobuf_json_mapping::merge_from_str(&mut *msg, &json)?; + + Ok(msg) +} + +fn buf2response( + buffer: &[u8], + output_message: String, + file_descriptor: FileDescriptor, +) -> anyhow::Result<DynResponse> { + let msg_descriptor = file_descriptor + .message_by_package_relative_name(&output_message) + .with_context(|| format!("Output message {output_message} not found"))?; + + let mut msg = msg_descriptor.new_instance(); + msg.merge_from_bytes_dyn(buffer)?; + + Ok(msg) +} + +fn get_file_descriptor(proto_file: &Path) -> Result<FileDescriptor> { + let proto_folder = proto_file + .parent() + .context("Proto file is not within a folder")?; + + let mut file_descriptors_protos = protobuf_parse::Parser::new() + .include(proto_folder) + .input(proto_file) + .parse_and_typecheck() + .unwrap() + .file_descriptors; + + let file_descriptor_proto: FileDescriptorProto = file_descriptors_protos.pop().unwrap(); + + let file_descriptor = FileDescriptor::new_dynamic(file_descriptor_proto, &[])?; + + Ok(file_descriptor) +} + +fn get_method_descriptor_proto( + file_descriptor: FileDescriptor, + method_name: &str, +) -> Result<MethodDescriptorProto> { + let method = file_descriptor + .proto() + .service + .iter() + .flat_map(|service| &service.method) + .find(|method| method.name.as_ref().is_some_and(|name| name == method_name)) + .context("method descriptor not found")?; + + Ok(method.clone()) +} + +fn get_relative_method_name(absolute_method_name: &str) -> anyhow::Result<String> { + let path: Vec<&str> = absolute_method_name.split('/').collect(); + let method = path.get(2).context("Invalid path")?; + + Ok(method.to_string()) +} + +fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result<String> { + let path: Vec<&str> = absolute_message_name.split('.').collect(); + let message = path.get(2).context("Invalid path")?; + + Ok(message.to_string()) +} + +#[derive(Deserialize)] +#[serde(crate = "serde")] +pub struct GrpcCallMethodInput { + proto_file: String, + method: String, + payload: String, + endpoint: String, +} + +#[deno_core::op2(async)] +#[string] +pub async fn op_call_grpc_method(#[serde] input: GrpcCallMethodInput) -> Result<String> { + let endpoint = Endpoint::from_str(&input.endpoint)?; + let channel = Channel::builder(endpoint.uri().to_owned()); + let channel = channel.connect().await?; + let mut client = Grpc::new(channel); + + let path = PathBuf::from_str(&input.proto_file).unwrap(); + let file_descriptor = get_file_descriptor(&path).unwrap(); + + let method_name = get_relative_method_name(&input.method).unwrap(); + let method_descriptor_proto = + get_method_descriptor_proto(file_descriptor.clone(), &method_name)?; + + let request_message = get_relative_message_name(method_descriptor_proto.input_type())?; + let req = json2request(input.payload, request_message, file_descriptor.clone())?.into_request(); + + let path_query = PathAndQuery::from_str(input.method.as_str()).unwrap(); + client.ready().await.unwrap(); + + let codec = DynCodec { + method_descriptor_proto, + file_descriptor, + }; + + let response = client.unary(req, path_query, codec).await.unwrap(); + let response = response.get_ref().deref(); + let json_response = protobuf_json_mapping::print_to_string(response).unwrap(); + Ok(json_response) +} diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index 40587e8888..fb2b0d45ed 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -4,17 +4,12 @@ import { Runtime } from "@typegate/runtimes/Runtime.ts"; import * as native from "native"; import { ComputeStage } from "@typegate/engine/query_engine.ts"; -import { RuntimeInitParams } from "@typegate/types.ts"; import { getLogger, Logger } from "@typegate/log.ts"; import { TypeGraph } from "@typegate/typegraph/mod.ts"; -import { nativeVoid } from "@typegate/utils.ts"; +import { Resolver, RuntimeInitParams } from "../types.ts"; const logger = getLogger(import.meta); -interface GrpcRuntimeData { - url: string; -} - export class GrpcRuntime extends Runtime { private logger: Logger; @@ -23,30 +18,36 @@ export class GrpcRuntime extends Runtime { this.logger = getLogger(`grpc: '${typegraphName}'`); } + // deno-lint-ignore require-await static async init(params: RuntimeInitParams): Promise<Runtime> { logger.info("Initliazing GrpcRuntime"); logger.debug(`Init params: ${JSON.stringify(params)}`); - const { typegraph, args } = params as RuntimeInitParams< - GrpcRuntimeData - >; + const { typegraph } = params as RuntimeInitParams; const typegraphName = TypeGraph.formatName(typegraph); const instance = new GrpcRuntime(typegraphName); instance.logger.info("registering GrpcRuntime"); - nativeVoid( - await native.grpc_register({ url: args.url }), - ); - throw new Error("Method not implemented."); + return instance; } deinit(): Promise<void> { throw new Error("Method not implemented."); } materialize( - _stage: ComputeStage, + stage: ComputeStage, _waitlist: ComputeStage[], _verbose: boolean, ): ComputeStage[] | Promise<ComputeStage[]> { - throw new Error("Method not implemented."); + const resolver: Resolver = async (args) => { + const { proto_file, method, payload, endpoint } = args; + return await native.call_grpc_method({ + proto_file, + method, + payload, + endpoint, + }); + }; + + return [new ComputeStage({ ...stage.props, resolver })]; } } diff --git a/typegraph/core/src/conversion/runtimes.rs b/typegraph/core/src/conversion/runtimes.rs index bc6f6c025c..43e03159e7 100644 --- a/typegraph/core/src/conversion/runtimes.rs +++ b/typegraph/core/src/conversion/runtimes.rs @@ -524,8 +524,6 @@ pub fn convert_runtime(_c: &mut TypegraphContext, runtime: Runtime) -> Result<Co .into()) } Runtime::Kv(d) => Ok(TGRuntime::Known(Rt::Kv(KvRuntimeData { url: d.url.clone() })).into()), - Runtime::Grpc(d) => { - Ok(TGRuntime::Known(Rt::Grpc(GrpcRuntimeData { url: d.url.clone() })).into()) - } + Runtime::Grpc => Ok(TGRuntime::Known(Rt::Grpc(GrpcRuntimeData {})).into()), } } diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index 28b61f789d..65679ed575 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -71,7 +71,7 @@ pub enum Runtime { S3(Rc<S3RuntimeData>), Substantial(Rc<SubstantialRuntimeData>), Kv(Rc<KvRuntimeData>), - Grpc(Rc<GrpcRuntimeData>), + Grpc, } #[derive(Debug, Clone)] @@ -707,8 +707,8 @@ impl crate::wit::runtimes::Guest for crate::Lib { Ok(Store::register_materializer(mat)) } - fn register_grpc_runtime(data: GrpcRuntimeData) -> Result<RuntimeId, wit::Error> { - Ok(Store::register_runtime(Runtime::Grpc(data.into()))) + fn register_grpc_runtime() -> Result<RuntimeId, wit::Error> { + Ok(Store::register_runtime(Runtime::Grpc)) } fn call_grpc_methode( diff --git a/typegraph/core/wit/typegraph.wit b/typegraph/core/wit/typegraph.wit index ad44df1ebd..8b6514a0ad 100644 --- a/typegraph/core/wit/typegraph.wit +++ b/typegraph/core/wit/typegraph.wit @@ -536,11 +536,7 @@ interface runtimes { kv-operation: func(base: base-materializer, data: kv-materializer) -> result<materializer-id, error>; // Grpc - record grpc-runtime-data { - url: string - } - - register-grpc-runtime: func(data: grpc-runtime-data) -> result<runtime-id, error>; + register-grpc-runtime: func() -> result<runtime-id, error>; record grpc-materializer { proto-file: string, From 51c297bd29fed876bdc1e63142a5696d7a66b4dc Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Tue, 20 Aug 2024 16:13:30 +0300 Subject: [PATCH 04/34] fix: wit import --- typegraph/core/src/runtimes/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index 65679ed575..f15098d4ae 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -28,10 +28,9 @@ use crate::wit::aws::S3RuntimeData; use crate::wit::core::{FuncParams, MaterializerId, RuntimeId, TypeId as CoreTypeId}; use crate::wit::runtimes::{ self as wit, BaseMaterializer, Error as TgError, GraphqlRuntimeData, GrpcMaterializer, - GrpcRuntimeData, HttpRuntimeData, KvMaterializer, KvRuntimeData, MaterializerHttpRequest, - PrismaLinkData, PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, - TemporalOperationData, TemporalRuntimeData, WasmRuntimeData, + HttpRuntimeData, KvMaterializer, KvRuntimeData, MaterializerHttpRequest, PrismaLinkData, PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, SubstantialRuntimeData, + TemporalOperationData, TemporalRuntimeData, WasmRuntimeData, }; use crate::{typegraph::TypegraphContext, wit::runtimes::Effect as WitEffect}; use enum_dispatch::enum_dispatch; From 35f7cef9fe863ffd269a5453c9c8fd6a60f45b37 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Wed, 21 Aug 2024 09:14:43 +0300 Subject: [PATCH 05/34] add python & deno grpc runtime --- typegate/src/runtimes/grpc.ts | 14 ++++-- typegraph/core/wit/typegraph.wit | 1 - typegraph/deno/sdk/src/runtimes/grpc.ts | 48 +++++++++++++++++++ .../python/typegraph/runtimes/__init__.py | 1 + typegraph/python/typegraph/runtimes/grpc.py | 39 +++++++++++++++ 5 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 typegraph/deno/sdk/src/runtimes/grpc.ts create mode 100644 typegraph/python/typegraph/runtimes/grpc.py diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index fb2b0d45ed..3caa5cb6ce 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -38,13 +38,19 @@ export class GrpcRuntime extends Runtime { _waitlist: ComputeStage[], _verbose: boolean, ): ComputeStage[] | Promise<ComputeStage[]> { + const { + proto_file, + method, + endpoint, + } = stage.props.materializer?.data ?? {}; + const resolver: Resolver = async (args) => { - const { proto_file, method, payload, endpoint } = args; + const { payload } = args; return await native.call_grpc_method({ - proto_file, - method, + proto_file: String(proto_file), + method: String(method), payload, - endpoint, + endpoint: String(endpoint), }); }; diff --git a/typegraph/core/wit/typegraph.wit b/typegraph/core/wit/typegraph.wit index 8b6514a0ad..7a518c662c 100644 --- a/typegraph/core/wit/typegraph.wit +++ b/typegraph/core/wit/typegraph.wit @@ -541,7 +541,6 @@ interface runtimes { record grpc-materializer { proto-file: string, method: string, - payload: string, endpoint: string, } diff --git a/typegraph/deno/sdk/src/runtimes/grpc.ts b/typegraph/deno/sdk/src/runtimes/grpc.ts new file mode 100644 index 0000000000..528b4b799e --- /dev/null +++ b/typegraph/deno/sdk/src/runtimes/grpc.ts @@ -0,0 +1,48 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + +import { Effect, GrpcMaterializer } from "../gen/typegraph_core.d.ts"; +import * as t from "../types.ts"; +import { runtimes } from "../wit.ts"; +import { Materializer, Runtime } from "./mod.ts"; + +class CallGrpcMethodMat implements Materializer { + mat: GrpcMaterializer; + _id: number; + constructor(id: number, mat: GrpcMaterializer) { + this._id = id; + this.mat = mat; + } +} + +export class GrpcRuntime extends Runtime { + constructor() { + const id = runtimes.registerGrpcRuntime(); + super(id); + } + + call_grpc_method( + proto_file: string, + method: string, + endpoint: string, + fx: Effect, + ) { + const grpc_materializer: GrpcMaterializer = { + protoFile: proto_file, + method: method, + endpoint: endpoint, + }; + const mat_id = runtimes.callGrpcMethode( + { runtime: this._id, effect: fx }, + grpc_materializer, + ); + + const mat = new CallGrpcMethodMat(mat_id, grpc_materializer); + + return t.func( + t.struct({ "payload": t.string().optional() }), + t.string(), + mat, + ); + } +} diff --git a/typegraph/python/typegraph/runtimes/__init__.py b/typegraph/python/typegraph/runtimes/__init__.py index 88d9bb01ac..d21ee52856 100644 --- a/typegraph/python/typegraph/runtimes/__init__.py +++ b/typegraph/python/typegraph/runtimes/__init__.py @@ -9,3 +9,4 @@ from typegraph.runtimes.random import RandomRuntime # noqa from typegraph.runtimes.wasm import WasmRuntime # noqa from typegraph.runtimes.kv import KvRuntime # noqa +from typegraph.runtimes.grpc import GrpcRuntime # noqa diff --git a/typegraph/python/typegraph/runtimes/grpc.py b/typegraph/python/typegraph/runtimes/grpc.py new file mode 100644 index 0000000000..771322d70a --- /dev/null +++ b/typegraph/python/typegraph/runtimes/grpc.py @@ -0,0 +1,39 @@ +# Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +# SPDX-License-Identifier: MPL-2.0 + +from dataclasses import dataclass +from typegraph import t +from typegraph.runtimes.base import Materializer, Runtime +from typegraph.gen.exports.runtimes import ( + BaseMaterializer, + Effect, + GrpcMaterializer, +) +from typegraph.gen.types import Err +from typegraph.wit import runtimes, store + + +class GrpcRuntime(Runtime): + def __init__(self, url: str): + runtime_id = runtimes.register_grpc_runtime(store) + if isinstance(runtime_id, Err): + raise Exception(runtime_id.value) + + super().__init__(runtime_id.value) + + def call_grpc_method(self, proto_file: str, method: str, endpoint: str, fx: Effect): + base = BaseMaterializer(self.id, fx) + grpc_materialier = GrpcMaterializer(proto_file, method, endpoint) + mat_id = runtimes.call_grpc_methode(store, base, grpc_materialier) + + if isinstance(mat_id, Err): + raise Exception(mat_id.value) + + mat = CallGrpcMethodMat(mat_id.value, effect=fx, mat=grpc_materialier) + + return t.func(t.struct({"payload": t.optional(t.string())}), t.string(), mat) + + +@dataclass +class CallGrpcMethodMat(Materializer): + mat: GrpcMaterializer From 6af81784ce4a7415f6a9fbf9640421ab7c799b14 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Sun, 25 Aug 2024 09:16:17 +0300 Subject: [PATCH 06/34] add op_grpc_register --- typegate/engine/src/runtimes/grpc.rs | 62 ++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 61104185e6..84c2e972c0 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -2,11 +2,16 @@ // SPDX-License-Identifier: Elastic-2.0 use std::{ + cell::RefCell, ops::Deref, path::{Path, PathBuf}, + rc::Rc, str::FromStr, + sync::Arc, }; +use dashmap::DashMap; +use deno_core::OpState; use protobuf::{ descriptor::{FileDescriptorProto, MethodDescriptorProto}, reflect::FileDescriptor, @@ -164,24 +169,67 @@ fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result<Stri Ok(message.to_string()) } +#[derive(Default)] +pub struct Ctx { + clients: Arc<DashMap<String, Grpc<Channel>>>, + proto_files: Arc<DashMap<String, String>>, +} + #[derive(Deserialize)] #[serde(crate = "serde")] -pub struct GrpcCallMethodInput { +pub struct GrpcRegisterInput { proto_file: String, - method: String, - payload: String, endpoint: String, + client_id: String, } #[deno_core::op2(async)] -#[string] -pub async fn op_call_grpc_method(#[serde] input: GrpcCallMethodInput) -> Result<String> { +pub async fn op_grpc_register( + state: Rc<RefCell<OpState>>, + #[serde] input: GrpcRegisterInput, +) -> Result<()> { let endpoint = Endpoint::from_str(&input.endpoint)?; let channel = Channel::builder(endpoint.uri().to_owned()); let channel = channel.connect().await?; - let mut client = Grpc::new(channel); + let client = Grpc::new(channel); + + let state = state.borrow(); + let ctx = state.borrow::<Ctx>(); + ctx.clients.insert(input.client_id.clone(), client); + ctx.proto_files.insert(input.client_id, input.proto_file); + Ok(()) +} + +#[derive(Deserialize)] +#[serde(crate = "serde")] +pub struct GrpcCallMethodInput { + method: String, + payload: String, + client_id: String, +} + +#[deno_core::op2(async)] +#[string] +pub async fn op_call_grpc_method( + state: Rc<RefCell<OpState>>, + #[serde] input: GrpcCallMethodInput, +) -> Result<String> { + let (clients, proto_files) = { + let state = state.borrow(); + let ctx = state.borrow::<Ctx>(); + (ctx.clients.clone(), ctx.proto_files.clone()) + }; + + let client_id = input.client_id.clone(); + let mut client = clients + .get_mut(&client_id) + .with_context(|| format!("Could not find client '{client_id}'"))?; + + let proto_file = proto_files + .get(&client_id) + .with_context(|| format!("Could not find proto_file '{client_id}'"))?; - let path = PathBuf::from_str(&input.proto_file).unwrap(); + let path = PathBuf::from_str(&proto_file).unwrap(); let file_descriptor = get_file_descriptor(&path).unwrap(); let method_name = get_relative_method_name(&input.method).unwrap(); From ec402ed8edcdfe9d3733f2fa759158c4ddb2a924 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Sun, 25 Aug 2024 15:18:35 +0300 Subject: [PATCH 07/34] fix effect && add grpc register native --- libs/common/src/typegraph/runtimes/grpc.rs | 5 ++- typegate/engine/00_runtime.js | 2 + typegate/engine/bindings.ts | 23 ++++++++++ typegate/engine/runtime.d.ts | 9 +++- typegate/engine/src/runtimes/grpc.rs | 52 +++++++++++++++------- typegate/src/runtimes/grpc.ts | 31 ++++++++----- typegraph/core/src/conversion/runtimes.rs | 6 ++- typegraph/core/src/runtimes/mod.rs | 12 ++--- typegraph/core/wit/typegraph.wit | 9 ++-- typegraph/deno/sdk/src/effects.ts | 2 +- typegraph/deno/sdk/src/runtimes/kv.ts | 2 +- typegraph/python/typegraph/effects.py | 8 ++-- 12 files changed, 115 insertions(+), 46 deletions(-) diff --git a/libs/common/src/typegraph/runtimes/grpc.rs b/libs/common/src/typegraph/runtimes/grpc.rs index 6ae04a5664..4bc5523e3c 100644 --- a/libs/common/src/typegraph/runtimes/grpc.rs +++ b/libs/common/src/typegraph/runtimes/grpc.rs @@ -4,4 +4,7 @@ use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GrpcRuntimeData {} +pub struct GrpcRuntimeData { + pub proto_file: String, + pub endpoint: String, +} diff --git a/typegate/engine/00_runtime.js b/typegate/engine/00_runtime.js index 2864cc4fd4..9ee1803002 100644 --- a/typegate/engine/00_runtime.js +++ b/typegate/engine/00_runtime.js @@ -52,6 +52,8 @@ globalThis.Meta = { handle: getOp("op_wit_wire_handle"), }, grpc: { + register: getOp("op_grpc_register"), + unregister: getOp("op_grpc_unregister"), callGrpcMethod: getOp("op_call_grpc_method"), }, }; diff --git a/typegate/engine/bindings.ts b/typegate/engine/bindings.ts index 10494f1e85..c54c3baa6c 100644 --- a/typegate/engine/bindings.ts +++ b/typegate/engine/bindings.ts @@ -3,6 +3,7 @@ import type { GrpcCallMethodInput, + GrpcRegisterInput, ParsedDiff, PrismaApplyOut, PrismaCreateOut, @@ -455,6 +456,28 @@ export function archive(a0: ArchiveInp): ArchiveResult { } } +export async function grpc_register(a0: GrpcRegisterInput): Promise<string> { + try { + return await Meta.grpc.register(a0); + } catch (err) { + return err.toString(); + } +} + +export type GrpcUnregisterInput = { + client_id: string; +}; + +export async function grpc_unregister( + a0: GrpcUnregisterInput, +): Promise<string> { + try { + return await Meta.grpc.unregister(a0.client_id); + } catch (err) { + return err.toString(); + } +} + export async function call_grpc_method( a0: GrpcCallMethodInput, ): Promise<string> { diff --git a/typegate/engine/runtime.d.ts b/typegate/engine/runtime.d.ts index 7ecddd63f8..a3479f3dfb 100644 --- a/typegate/engine/runtime.d.ts +++ b/typegate/engine/runtime.d.ts @@ -63,6 +63,8 @@ type MetaNS = { }; grpc: { + register: (inp: GrpcRegisterInput) => Promise<void>; + unregister: (client_id: string) => void; callGrpcMethod: (inp: GrpcCallMethodInput) => Promise<string>; }; }; @@ -265,8 +267,13 @@ export type WitWireHandleResponse = }; export type GrpcCallMethodInput = { - proto_file: string; method: string; payload: string; + client_id: string; +}; + +export type GrpcRegisterInput = { + protoFile: string; endpoint: string; + client_id: string; }; diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 84c2e972c0..a0bdda0e39 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -169,10 +169,14 @@ fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result<Stri Ok(message.to_string()) } +struct GrpcClient { + client: Grpc<Channel>, + proto_file: String, +} + #[derive(Default)] pub struct Ctx { - clients: Arc<DashMap<String, Grpc<Channel>>>, - proto_files: Arc<DashMap<String, String>>, + grpc_client: Arc<DashMap<String, GrpcClient>>, } #[derive(Deserialize)] @@ -195,8 +199,23 @@ pub async fn op_grpc_register( let state = state.borrow(); let ctx = state.borrow::<Ctx>(); - ctx.clients.insert(input.client_id.clone(), client); - ctx.proto_files.insert(input.client_id, input.proto_file); + + let grpc_client = GrpcClient { + client, + proto_file: input.proto_file, + }; + ctx.grpc_client.insert(input.client_id.clone(), grpc_client); + + Ok(()) +} + +#[deno_core::op2(fast)] +pub fn op_grpc_unregister(#[state] ctx: &mut Ctx, #[string] client_id: &str) -> Result<()> { + let Some((_, _client)) = ctx.grpc_client.remove(client_id) else { + anyhow::bail!("Could not remove engine {:?}: entry not found.", { + client_id + }); + }; Ok(()) } @@ -214,22 +233,17 @@ pub async fn op_call_grpc_method( state: Rc<RefCell<OpState>>, #[serde] input: GrpcCallMethodInput, ) -> Result<String> { - let (clients, proto_files) = { + let grpc_client = { let state = state.borrow(); let ctx = state.borrow::<Ctx>(); - (ctx.clients.clone(), ctx.proto_files.clone()) + ctx.grpc_client.clone() }; - let client_id = input.client_id.clone(); - let mut client = clients - .get_mut(&client_id) - .with_context(|| format!("Could not find client '{client_id}'"))?; - - let proto_file = proto_files - .get(&client_id) - .with_context(|| format!("Could not find proto_file '{client_id}'"))?; + let mut grpc_client = grpc_client + .get_mut(&input.client_id) + .with_context(|| format!("Could not find client '{}'", &input.client_id))?; - let path = PathBuf::from_str(&proto_file).unwrap(); + let path = PathBuf::from_str(&grpc_client.proto_file).unwrap(); let file_descriptor = get_file_descriptor(&path).unwrap(); let method_name = get_relative_method_name(&input.method).unwrap(); @@ -240,14 +254,18 @@ pub async fn op_call_grpc_method( let req = json2request(input.payload, request_message, file_descriptor.clone())?.into_request(); let path_query = PathAndQuery::from_str(input.method.as_str()).unwrap(); - client.ready().await.unwrap(); + grpc_client.client.ready().await.unwrap(); let codec = DynCodec { method_descriptor_proto, file_descriptor, }; - let response = client.unary(req, path_query, codec).await.unwrap(); + let response = grpc_client + .client + .unary(req, path_query, codec) + .await + .unwrap(); let response = response.get_ref().deref(); let json_response = protobuf_json_mapping::print_to_string(response).unwrap(); Ok(json_response) diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index 3caa5cb6ce..cff391f288 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -10,6 +10,11 @@ import { Resolver, RuntimeInitParams } from "../types.ts"; const logger = getLogger(import.meta); +interface GrpcRuntimeData { + protoFile: string; + endpoint: string; +} + export class GrpcRuntime extends Runtime { private logger: Logger; @@ -18,39 +23,43 @@ export class GrpcRuntime extends Runtime { this.logger = getLogger(`grpc: '${typegraphName}'`); } - // deno-lint-ignore require-await static async init(params: RuntimeInitParams): Promise<Runtime> { logger.info("Initliazing GrpcRuntime"); logger.debug(`Init params: ${JSON.stringify(params)}`); - const { typegraph } = params as RuntimeInitParams; + const { typegraph, args } = params as RuntimeInitParams<GrpcRuntimeData>; const typegraphName = TypeGraph.formatName(typegraph); const instance = new GrpcRuntime(typegraphName); + + await native.grpc_register({ + protoFile: args.protoFile, + endpoint: args.endpoint, + client_id: instance.id, + }); + instance.logger.info("registering GrpcRuntime"); + return instance; } - deinit(): Promise<void> { - throw new Error("Method not implemented."); + // deno-lint-ignore require-await + async deinit(): Promise<void> { + native.grpc_unregister({ client_id: this.id }); } + materialize( stage: ComputeStage, _waitlist: ComputeStage[], _verbose: boolean, ): ComputeStage[] | Promise<ComputeStage[]> { - const { - proto_file, - method, - endpoint, - } = stage.props.materializer?.data ?? {}; + const { method } = stage.props.materializer?.data ?? {}; const resolver: Resolver = async (args) => { const { payload } = args; return await native.call_grpc_method({ - proto_file: String(proto_file), method: String(method), payload, - endpoint: String(endpoint), + client_id: this.id, }); }; diff --git a/typegraph/core/src/conversion/runtimes.rs b/typegraph/core/src/conversion/runtimes.rs index 43e03159e7..a22e280cce 100644 --- a/typegraph/core/src/conversion/runtimes.rs +++ b/typegraph/core/src/conversion/runtimes.rs @@ -524,6 +524,10 @@ pub fn convert_runtime(_c: &mut TypegraphContext, runtime: Runtime) -> Result<Co .into()) } Runtime::Kv(d) => Ok(TGRuntime::Known(Rt::Kv(KvRuntimeData { url: d.url.clone() })).into()), - Runtime::Grpc => Ok(TGRuntime::Known(Rt::Grpc(GrpcRuntimeData {})).into()), + Runtime::Grpc(d) => Ok(TGRuntime::Known(Rt::Grpc(GrpcRuntimeData { + proto_file: d.proto_file.clone(), + endpoint: d.endpoint.clone(), + })) + .into()), } } diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index f15098d4ae..71875ea734 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -28,9 +28,9 @@ use crate::wit::aws::S3RuntimeData; use crate::wit::core::{FuncParams, MaterializerId, RuntimeId, TypeId as CoreTypeId}; use crate::wit::runtimes::{ self as wit, BaseMaterializer, Error as TgError, GraphqlRuntimeData, GrpcMaterializer, - HttpRuntimeData, KvMaterializer, KvRuntimeData, MaterializerHttpRequest, PrismaLinkData, - PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, SubstantialRuntimeData, - TemporalOperationData, TemporalRuntimeData, WasmRuntimeData, + GrpcRuntimeData, HttpRuntimeData, KvMaterializer, KvRuntimeData, MaterializerHttpRequest, + PrismaLinkData, PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, + SubstantialRuntimeData, TemporalOperationData, TemporalRuntimeData, WasmRuntimeData, }; use crate::{typegraph::TypegraphContext, wit::runtimes::Effect as WitEffect}; use enum_dispatch::enum_dispatch; @@ -70,7 +70,7 @@ pub enum Runtime { S3(Rc<S3RuntimeData>), Substantial(Rc<SubstantialRuntimeData>), Kv(Rc<KvRuntimeData>), - Grpc, + Grpc(Rc<GrpcRuntimeData>), } #[derive(Debug, Clone)] @@ -706,8 +706,8 @@ impl crate::wit::runtimes::Guest for crate::Lib { Ok(Store::register_materializer(mat)) } - fn register_grpc_runtime() -> Result<RuntimeId, wit::Error> { - Ok(Store::register_runtime(Runtime::Grpc)) + fn register_grpc_runtime(data: GrpcRuntimeData) -> Result<RuntimeId, wit::Error> { + Ok(Store::register_runtime(Runtime::Grpc(data.into()))) } fn call_grpc_methode( diff --git a/typegraph/core/wit/typegraph.wit b/typegraph/core/wit/typegraph.wit index 7a518c662c..8e42a86607 100644 --- a/typegraph/core/wit/typegraph.wit +++ b/typegraph/core/wit/typegraph.wit @@ -536,12 +536,15 @@ interface runtimes { kv-operation: func(base: base-materializer, data: kv-materializer) -> result<materializer-id, error>; // Grpc - register-grpc-runtime: func() -> result<runtime-id, error>; + record grpc-runtime-data { + proto-file: string, + endpoint: string, + } + + register-grpc-runtime: func(data: grpc-runtime-data) -> result<runtime-id, error>; record grpc-materializer { - proto-file: string, method: string, - endpoint: string, } call-grpc-methode: func(base: base-materializer, data: grpc-materializer) -> result<materializer-id, error>; diff --git a/typegraph/deno/sdk/src/effects.ts b/typegraph/deno/sdk/src/effects.ts index a92133be7e..8c5dbcdf7e 100644 --- a/typegraph/deno/sdk/src/effects.ts +++ b/typegraph/deno/sdk/src/effects.ts @@ -20,7 +20,7 @@ export function delete_(idempotent = true): EffectDelete { return { tag: "delete", val: idempotent }; } -export function update(idempotent = true): EffectUpdate { +export function update(idempotent = false): EffectUpdate { return { tag: "update", val: idempotent }; } diff --git a/typegraph/deno/sdk/src/runtimes/kv.ts b/typegraph/deno/sdk/src/runtimes/kv.ts index e510619dd8..14d5a81597 100644 --- a/typegraph/deno/sdk/src/runtimes/kv.ts +++ b/typegraph/deno/sdk/src/runtimes/kv.ts @@ -35,7 +35,7 @@ export class KvRuntime extends Runtime { } set() { - const mat = this.#operation("set", fx.update(false)); + const mat = this.#operation("set", fx.update()); return t.func( t.struct({ "key": t.string(), "value": t.string() }), t.string(), diff --git a/typegraph/python/typegraph/effects.py b/typegraph/python/typegraph/effects.py index 40ffeb6058..8156959e5c 100644 --- a/typegraph/python/typegraph/effects.py +++ b/typegraph/python/typegraph/effects.py @@ -19,14 +19,14 @@ def create(idempotent: bool = False): return EffectCreate(idempotent) -def update(idempotent: bool = False): - return EffectUpdate(idempotent) - - def delete(idempotent: bool = True): return EffectDelete(idempotent) +def update(idempotent: bool = False): + return EffectUpdate(idempotent) + + # For injections class EffectType(Enum): CREATE = auto() From 5c0f539a78de268903140daa5e58e306d88c2213 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Mon, 26 Aug 2024 17:55:55 +0300 Subject: [PATCH 08/34] improve the grpc impl --- typegate/engine/bindings.ts | 50 ++++++++++++++++++++------ typegate/engine/runtime.d.ts | 14 ++++---- typegate/engine/src/runtimes/grpc.rs | 53 ++++++++++++++++------------ typegate/src/runtimes/grpc.ts | 30 +++++++++------- 4 files changed, 96 insertions(+), 51 deletions(-) diff --git a/typegate/engine/bindings.ts b/typegate/engine/bindings.ts index c54c3baa6c..a87e542f9c 100644 --- a/typegate/engine/bindings.ts +++ b/typegate/engine/bindings.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Elastic-2.0 import type { - GrpcCallMethodInput, + CallGrpcMethodInput, GrpcRegisterInput, ParsedDiff, PrismaApplyOut, @@ -456,11 +456,22 @@ export function archive(a0: ArchiveInp): ArchiveResult { } } -export async function grpc_register(a0: GrpcRegisterInput): Promise<string> { +export type GrpcRegisterOutput = + | "Ok" + | { + Err: { + message: string; + }; + }; + +export async function grpc_register( + a0: GrpcRegisterInput, +): Promise<GrpcRegisterOutput> { try { - return await Meta.grpc.register(a0); + await Meta.grpc.register(a0); + return "Ok"; } catch (err) { - return err.toString(); + return { Err: { message: err.toString() } }; } } @@ -468,22 +479,41 @@ export type GrpcUnregisterInput = { client_id: string; }; +export type GrpcUnRegisterOutput = + | "Ok" + | { + Err: { + message: string; + }; + }; + export async function grpc_unregister( a0: GrpcUnregisterInput, -): Promise<string> { +): Promise<GrpcUnRegisterOutput> { try { - return await Meta.grpc.unregister(a0.client_id); + await Meta.grpc.unregister(a0.client_id); + return "Ok"; } catch (err) { - return err.toString(); + return { Err: { message: err.toString() } }; } } +export type CallGrpcMethodOutput = + | { + Ok: string; + } + | { + Err: { + message: string; + }; + }; + export async function call_grpc_method( - a0: GrpcCallMethodInput, -): Promise<string> { + a0: CallGrpcMethodInput, +): Promise<CallGrpcMethodOutput> { try { return await Meta.grpc.callGrpcMethod(a0); } catch (err) { - return err.toString(); + return { Err: { message: err.toString() } }; } } diff --git a/typegate/engine/runtime.d.ts b/typegate/engine/runtime.d.ts index a3479f3dfb..1992daeb3c 100644 --- a/typegate/engine/runtime.d.ts +++ b/typegate/engine/runtime.d.ts @@ -65,7 +65,7 @@ type MetaNS = { grpc: { register: (inp: GrpcRegisterInput) => Promise<void>; unregister: (client_id: string) => void; - callGrpcMethod: (inp: GrpcCallMethodInput) => Promise<string>; + callGrpcMethod: (inp: CallGrpcMethodInput) => Promise<string>; }; }; @@ -266,14 +266,14 @@ export type WitWireHandleResponse = HandlerErr: string; }; -export type GrpcCallMethodInput = { - method: string; - payload: string; - client_id: string; -}; - export type GrpcRegisterInput = { protoFile: string; endpoint: string; client_id: string; }; + +export type CallGrpcMethodInput = { + method: string; + payload: string; + client_id: string; +}; diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index a0bdda0e39..736507e9dc 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -176,7 +176,7 @@ struct GrpcClient { #[derive(Default)] pub struct Ctx { - grpc_client: Arc<DashMap<String, GrpcClient>>, + grpc_clients: Arc<DashMap<String, GrpcClient>>, } #[derive(Deserialize)] @@ -192,30 +192,32 @@ pub async fn op_grpc_register( state: Rc<RefCell<OpState>>, #[serde] input: GrpcRegisterInput, ) -> Result<()> { - let endpoint = Endpoint::from_str(&input.endpoint)?; - let channel = Channel::builder(endpoint.uri().to_owned()); - let channel = channel.connect().await?; - let client = Grpc::new(channel); + let endpoint = Endpoint::from_str(&input.endpoint).context("Failed to parse endpoint")?; + + let channel = Channel::builder(endpoint.uri().to_owned()) + .connect() + .await + .context("Failed to estabilish channel connection")?; let state = state.borrow(); let ctx = state.borrow::<Ctx>(); let grpc_client = GrpcClient { - client, + client: Grpc::new(channel), proto_file: input.proto_file, }; - ctx.grpc_client.insert(input.client_id.clone(), grpc_client); + ctx.grpc_clients + .insert(input.client_id.clone(), grpc_client); Ok(()) } #[deno_core::op2(fast)] pub fn op_grpc_unregister(#[state] ctx: &mut Ctx, #[string] client_id: &str) -> Result<()> { - let Some((_, _client)) = ctx.grpc_client.remove(client_id) else { - anyhow::bail!("Could not remove engine {:?}: entry not found.", { - client_id - }); - }; + ctx.grpc_clients + .remove(client_id) + .with_context(|| format!("Failed to remove gRPC client with ID: {}", client_id))?; + Ok(()) } @@ -233,28 +235,32 @@ pub async fn op_call_grpc_method( state: Rc<RefCell<OpState>>, #[serde] input: GrpcCallMethodInput, ) -> Result<String> { - let grpc_client = { + let grpc_clients = { let state = state.borrow(); let ctx = state.borrow::<Ctx>(); - ctx.grpc_client.clone() + ctx.grpc_clients.clone() }; - let mut grpc_client = grpc_client + let mut grpc_client = grpc_clients .get_mut(&input.client_id) - .with_context(|| format!("Could not find client '{}'", &input.client_id))?; + .with_context(|| format!("Could not find gRPC client '{}'", &input.client_id))?; + + let path = PathBuf::from_str(&grpc_client.proto_file)?; + + let file_descriptor = get_file_descriptor(&path)?; - let path = PathBuf::from_str(&grpc_client.proto_file).unwrap(); - let file_descriptor = get_file_descriptor(&path).unwrap(); + let method_name = get_relative_method_name(&input.method)?; - let method_name = get_relative_method_name(&input.method).unwrap(); let method_descriptor_proto = get_method_descriptor_proto(file_descriptor.clone(), &method_name)?; let request_message = get_relative_message_name(method_descriptor_proto.input_type())?; + let req = json2request(input.payload, request_message, file_descriptor.clone())?.into_request(); - let path_query = PathAndQuery::from_str(input.method.as_str()).unwrap(); - grpc_client.client.ready().await.unwrap(); + let path_query = PathAndQuery::from_str(input.method.as_str())?; + + grpc_client.client.ready().await?; let codec = DynCodec { method_descriptor_proto, @@ -265,8 +271,11 @@ pub async fn op_call_grpc_method( .client .unary(req, path_query, codec) .await - .unwrap(); + .context("Failed to perform unary gRPC call")?; + let response = response.get_ref().deref(); + let json_response = protobuf_json_mapping::print_to_string(response).unwrap(); + Ok(json_response) } diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index cff391f288..c4b3f09a4e 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -7,6 +7,7 @@ import { ComputeStage } from "@typegate/engine/query_engine.ts"; import { getLogger, Logger } from "@typegate/log.ts"; import { TypeGraph } from "@typegate/typegraph/mod.ts"; import { Resolver, RuntimeInitParams } from "../types.ts"; +import { nativeResult, nativeVoid } from "@typegate/utils.ts"; const logger = getLogger(import.meta); @@ -31,20 +32,23 @@ export class GrpcRuntime extends Runtime { const typegraphName = TypeGraph.formatName(typegraph); const instance = new GrpcRuntime(typegraphName); - await native.grpc_register({ - protoFile: args.protoFile, - endpoint: args.endpoint, - client_id: instance.id, - }); + nativeVoid( + await native.grpc_register({ + protoFile: args.protoFile, + endpoint: args.endpoint, + client_id: instance.id, + }), + ); instance.logger.info("registering GrpcRuntime"); return instance; } - // deno-lint-ignore require-await async deinit(): Promise<void> { - native.grpc_unregister({ client_id: this.id }); + nativeVoid( + await native.grpc_unregister({ client_id: this.id }), + ); } materialize( @@ -56,11 +60,13 @@ export class GrpcRuntime extends Runtime { const resolver: Resolver = async (args) => { const { payload } = args; - return await native.call_grpc_method({ - method: String(method), - payload, - client_id: this.id, - }); + return nativeResult( + await native.call_grpc_method({ + method: String(method), + payload, + client_id: this.id, + }), + ); }; return [new ComputeStage({ ...stage.props, resolver })]; From 5c6c90cc14e3cda1e6b8be24ffea285a430d0f4c Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Mon, 26 Aug 2024 22:09:53 +0300 Subject: [PATCH 09/34] finish impl grpcruntime for deno && python next step, auto gener type for input --- .ghjk/deno.lock | 42 +++++++++++++++++++++ Cargo.lock | 2 +- typegate/src/runtimes/grpc.ts | 2 + typegraph/deno/sdk/src/runtimes/grpc.ts | 16 +++----- typegraph/python/typegraph/runtimes/grpc.py | 16 +++++--- 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/.ghjk/deno.lock b/.ghjk/deno.lock index 3ff9615ac0..bae200298d 100644 --- a/.ghjk/deno.lock +++ b/.ghjk/deno.lock @@ -724,6 +724,48 @@ "https://raw.githubusercontent.com/levibostian/deno-udd/ignore-prerelease/registry.ts": "fd8e1b05f14cb988fee7a72a51e68131a920f7d4b72f949d9b86794b3c699671", "https://raw.githubusercontent.com/levibostian/deno-udd/ignore-prerelease/search.ts": "52f9a539ca76893c47d01f8c6d401487ea286d54d1305b079b8727598e4c847a", "https://raw.githubusercontent.com/levibostian/deno-udd/ignore-prerelease/semver.ts": "c051a906405dd72b55434eb0f390f678881379d57847abe4ec60d8a02af4f6f2", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/deps/cli.ts": "aac025f9372ad413b9c2663dc7f61affd597820d9448f010a510d541df3b56ea", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/deps/common.ts": "f775710b66a9099b98651cd3831906466e9b83ef98f2e5c080fd59ee801c28d4", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/deps/ports.ts": "3c60d1f7ab626ffdd81b37f4e83a780910936480da8fe24f4ccceaefa207d339", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/files/deno/mod.ts": "1b8204c3df18b908408b2148b48af788e669d0debbeb8ba119418ab1ddf1ab8f", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/files/deno/worker.ts": "8ded400d70a0bd40e281ceb1ffcdc82578443caf9c481b9eee77166472784282", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/host/mod.ts": "cc25d1f82e54e6a27eef4571145c3f34c4c8ad9148b3aa48bd3b53d1e078d95d", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/host/types.ts": "f450d9b9c0eced2650262d02455aa6f794de0edd6b052aade256882148e5697f", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/install/mod.ts": "aa54eb3e119f28d33e61645c89669da292ee00376068ead8f45be2807e7a9989", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/install/utils.ts": "d4634d4fc0e963f540402b4ca7eb5dcba340eaa0d8fceb43af57d722ad267115", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/main.ts": "ecd5e83be2d8f351058ad44424cad1f36dd2e3d76f6e8409afc47682a9eff01a", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/envs/inter.ts": "84805fa208754a08f185dca7a5236de3760bbc1d0df96af86ea5fd7778f827a2", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/envs/mod.ts": "5f37b9f155808f8d6d51e1f16f58c07914d8c7d8070bc5c2fb5076ab748798a7", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/envs/posix.ts": "09e410e3fea9c303a5148ff2a22697474320442b9fea0bd3fc932d6828fe820f", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/envs/reducer.ts": "50517084caaf73ce6618141ee4d97795060a0d3169651da7abd7251a3204465a", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/envs/types.ts": "ab9715cf02e9d73f553ae757db347863be23e1e9daf94d18aab716fc27b3dbc1", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/mod.ts": "fc1cb9176c6557b44ae9c6536fa51c6c4f80ac01fc476d15b0a217e70cb0d176", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/ambient.ts": "823ec8d98702a60e6bfcdbeb64b69dc9f5039e73a1f10e87cd51210c1aaf52d5", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/base.ts": "8ef8a8de372420bddcd63a1b363937f43d898059e99478a58621e8432bcd5891", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/db.ts": "a309d1058f66079a481141c3f1733d928b9af8a37b7ce911b1228f70fd24df0f", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/ghrel.ts": "ebbc30a5c31244131d937eadca73fbc099c9e7bdf0ad4f668766d4388ede143c", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/inter.ts": "b3999e73d73d7f928a8de86e5e2261fe6b1450ceedfb54f24537bf0803532ed0", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/mod.ts": "78db7040e724f84c95b1a0fdeaf0cfc53382482e8905cd352189756b953556cc", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/reducers.ts": "d04e813652101f67f946242df68429ed5540e499fbdb7776b8be5703f16754c8", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/sync.ts": "a7a297f6b098360d56af168692f3cff96f8ceeb5189e5baa249e094f8d9c42ef", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/types.ts": "f4dbd1a3f4b7f539b3a85418617d25adbf710b54144161880d48f6c4ec032eee", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/types/platform.ts": "0ecffeda71919293f9ffdb6c564ddea4f23bc85c4e640b08ea78225d34387fdc", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/utils.ts": "6b14b331cce66bd46e7aec51f02424327d819150f16d3f72a6b0aaf7aee43c09", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/ports/worker.ts": "6b76ba1efb2e47a82582fc48bcc6264fe153a166beffccde1a9a3a185024c337", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/std.ts": "419d6b04680f73f7b252257ab287d68c1571cee4347301c53278e2b53df21c4a", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/tasks/deno.ts": "2b9f33253ac1257eb79a4981cd221509aa9ecf8a3c36d7bd8be1cd6c1150100b", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/tasks/exec.ts": "6adcfe13f8d2da5d65331fd1601d4f950d9fc6f164bc9592204e5b08c23c5c30", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/tasks/inter.ts": "63e8f2860f7e3b4d95b6f61ca56aeb8567e4f265aa9c22cace6c8075edd6210f", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/tasks/mod.ts": "334b18d7c110cc05483be96353e342425c0033b7410c271a8a47d2b18308c73e", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/tasks/types.ts": "072a34bd0749428bad4d612cc86abe463d4d4f74dc56cf0a48a1f41650e2399b", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/modules/types.ts": "c0f212b686a2721d076e9aeb127596c7cbc939758e2cc32fd1d165a8fb320a87", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/port.ts": "c039a010dee7dfd978478cf4c5e2256c643135e10f33c30a09f8db9915e9d89d", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/setup_logger.ts": "f8a206bda0595497d6f4718032d4a959000b32ef3346d4b507777eec6a169458", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/utils/logger.ts": "fcbafb35ae4b812412b9b301ce6d06b8b9798f94ebebe3f92677e25e4b19af3c", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/utils/mod.ts": "25bfdd222d6afec5b3f0a7e647e3d9b12abed6d222b49a4b2e95c6bbe266f533", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/utils/unarchive.ts": "f6d0e9e75f470eeef5aecd0089169f4350fc30ebfdc05466bb7b30042294d6d3", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/utils/url.ts": "e1ada6fd30fc796b8918c88456ea1b5bbd87a07d0a0538b092b91fd2bb9b7623", + "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/utils/worker.ts": "ac4caf72a36d2e4af4f4e92f2e0a95f9fc2324b568640f24c7c2ff6dc0c11d62", "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/deps/cli.ts": "aac025f9372ad413b9c2663dc7f61affd597820d9448f010a510d541df3b56ea", "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/deps/common.ts": "f775710b66a9099b98651cd3831906466e9b83ef98f2e5c080fd59ee801c28d4", "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/deps/ports.ts": "3c60d1f7ab626ffdd81b37f4e83a780910936480da8fe24f4ccceaefa207d339", diff --git a/Cargo.lock b/Cargo.lock index af7ff00eb7..736e02cd82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9220,7 +9220,7 @@ dependencies = [ "serde 1.0.204", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index c4b3f09a4e..b07c749056 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -8,6 +8,7 @@ import { getLogger, Logger } from "@typegate/log.ts"; import { TypeGraph } from "@typegate/typegraph/mod.ts"; import { Resolver, RuntimeInitParams } from "../types.ts"; import { nativeResult, nativeVoid } from "@typegate/utils.ts"; +import { registerRuntime } from "@typegate/runtimes/mod.ts"; const logger = getLogger(import.meta); @@ -16,6 +17,7 @@ interface GrpcRuntimeData { endpoint: string; } +@registerRuntime("grpc") export class GrpcRuntime extends Runtime { private logger: Logger; diff --git a/typegraph/deno/sdk/src/runtimes/grpc.ts b/typegraph/deno/sdk/src/runtimes/grpc.ts index 528b4b799e..62297dc7cc 100644 --- a/typegraph/deno/sdk/src/runtimes/grpc.ts +++ b/typegraph/deno/sdk/src/runtimes/grpc.ts @@ -16,21 +16,17 @@ class CallGrpcMethodMat implements Materializer { } export class GrpcRuntime extends Runtime { - constructor() { - const id = runtimes.registerGrpcRuntime(); + constructor(protoFile: string, endpoint: string) { + const id = runtimes.registerGrpcRuntime({ + protoFile, + endpoint, + }); super(id); } - call_grpc_method( - proto_file: string, - method: string, - endpoint: string, - fx: Effect, - ) { + call_grpc_method(method: string, fx: Effect) { const grpc_materializer: GrpcMaterializer = { - protoFile: proto_file, method: method, - endpoint: endpoint, }; const mat_id = runtimes.callGrpcMethode( { runtime: this._id, effect: fx }, diff --git a/typegraph/python/typegraph/runtimes/grpc.py b/typegraph/python/typegraph/runtimes/grpc.py index 771322d70a..a53b649531 100644 --- a/typegraph/python/typegraph/runtimes/grpc.py +++ b/typegraph/python/typegraph/runtimes/grpc.py @@ -8,28 +8,32 @@ BaseMaterializer, Effect, GrpcMaterializer, + GrpcRuntimeData, ) from typegraph.gen.types import Err from typegraph.wit import runtimes, store class GrpcRuntime(Runtime): - def __init__(self, url: str): - runtime_id = runtimes.register_grpc_runtime(store) + def __init__(self, proto_file: str, endpoint: str): + data = GrpcRuntimeData(proto_file, endpoint) + runtime_id = runtimes.register_grpc_runtime(store, data) if isinstance(runtime_id, Err): raise Exception(runtime_id.value) super().__init__(runtime_id.value) - def call_grpc_method(self, proto_file: str, method: str, endpoint: str, fx: Effect): - base = BaseMaterializer(self.id, fx) - grpc_materialier = GrpcMaterializer(proto_file, method, endpoint) + def call_grpc_method(self, method: str, effect: Effect): + base = BaseMaterializer(self.id, effect) + + grpc_materialier = GrpcMaterializer(method) + mat_id = runtimes.call_grpc_methode(store, base, grpc_materialier) if isinstance(mat_id, Err): raise Exception(mat_id.value) - mat = CallGrpcMethodMat(mat_id.value, effect=fx, mat=grpc_materialier) + mat = CallGrpcMethodMat(mat_id.value, effect, mat=grpc_materialier) return t.func(t.struct({"payload": t.optional(t.string())}), t.string(), mat) From 69c67cc5282c4e22a062ba730636c24c93001b49 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Wed, 28 Aug 2024 00:22:29 +0300 Subject: [PATCH 10/34] finished setting up type generation --- typegraph/core/src/conversion/runtimes.rs | 20 +------- typegraph/core/src/runtimes/grpc/mod.rs | 46 +++++++++++++++++++ .../core/src/runtimes/grpc/type_generation.rs | 2 + typegraph/core/src/runtimes/mod.rs | 35 ++++++++++---- typegraph/core/wit/typegraph.wit | 4 +- typegraph/deno/sdk/src/runtimes/grpc.ts | 33 ++----------- typegraph/python/typegraph/runtimes/grpc.py | 29 ++++-------- 7 files changed, 89 insertions(+), 80 deletions(-) create mode 100644 typegraph/core/src/runtimes/grpc/mod.rs create mode 100644 typegraph/core/src/runtimes/grpc/type_generation.rs diff --git a/typegraph/core/src/conversion/runtimes.rs b/typegraph/core/src/conversion/runtimes.rs index a22e280cce..dad19397ca 100644 --- a/typegraph/core/src/conversion/runtimes.rs +++ b/typegraph/core/src/conversion/runtimes.rs @@ -8,7 +8,7 @@ use crate::runtimes::{ Runtime, TemporalMaterializer, WasmMaterializer, }; use crate::wit::core::{Artifact as WitArtifact, RuntimeId}; -use crate::wit::runtimes::{GrpcMaterializer, HttpMethod, KvMaterializer, MaterializerHttpRequest}; +use crate::wit::runtimes::{HttpMethod, KvMaterializer, MaterializerHttpRequest}; use crate::{typegraph::TypegraphContext, wit::runtimes::Effect as WitEffect}; use common::typegraph::runtimes::deno::DenoRuntimeData; use common::typegraph::runtimes::graphql::GraphQLRuntimeData; @@ -405,24 +405,6 @@ impl MaterializerConverter for KvMaterializer { } } -impl MaterializerConverter for GrpcMaterializer { - fn convert( - &self, - c: &mut TypegraphContext, - runtime_id: RuntimeId, - effect: WitEffect, - ) -> Result<common::typegraph::Materializer> { - let runtime = c.register_runtime(runtime_id)?; - let data = serde_json::from_value(json!({})).map_err(|e| e.to_string())?; - Ok(Materializer { - name: "grpc".into(), - runtime, - effect: effect.into(), - data, - }) - } -} - pub fn convert_materializer( c: &mut TypegraphContext, mat: RawMaterializer, diff --git a/typegraph/core/src/runtimes/grpc/mod.rs b/typegraph/core/src/runtimes/grpc/mod.rs new file mode 100644 index 0000000000..cf64c70f7a --- /dev/null +++ b/typegraph/core/src/runtimes/grpc/mod.rs @@ -0,0 +1,46 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + +use std::rc::Rc; + +use crate::errors::Result; +use crate::global_store::Store; +use crate::wit::core::RuntimeId; +use crate::wit::runtimes::{Effect as WitEffect, GrpcRuntimeData}; +use crate::{conversion::runtimes::MaterializerConverter, typegraph::TypegraphContext}; + +use common::typegraph::Materializer; + +use serde_json::{from_value, json}; + +use super::Runtime; + +pub fn get_gprc_data(runtime_id: RuntimeId) -> Rc<GrpcRuntimeData> { + match Store::get_runtime(runtime_id).unwrap() { + Runtime::Grpc(data) => data, + _ => unreachable!(), + } +} + +#[derive(Debug)] +pub struct GrpcMaterializer { + pub method: String, +} + +impl MaterializerConverter for GrpcMaterializer { + fn convert( + &self, + c: &mut TypegraphContext, + runtime_id: RuntimeId, + effect: WitEffect, + ) -> Result<common::typegraph::Materializer> { + let runtime = c.register_runtime(runtime_id)?; + let data = from_value(json!({"method": self.method})).map_err(|e| e.to_string())?; + Ok(Materializer { + name: "grpc".into(), + runtime, + effect: effect.into(), + data, + }) + } +} diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs new file mode 100644 index 0000000000..23d9e15ccf --- /dev/null +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -0,0 +1,2 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index 71875ea734..ccfb8a1dc9 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -4,6 +4,7 @@ pub mod aws; pub mod deno; pub mod graphql; +pub mod grpc; pub mod prisma; pub mod python; pub mod random; @@ -18,6 +19,7 @@ use std::rc::Rc; use crate::conversion::runtimes::MaterializerConverter; use crate::global_store::Store; +use crate::runtimes::grpc::get_gprc_data; use crate::runtimes::prisma::migration::{ prisma_apply, prisma_create, prisma_deploy, prisma_diff, prisma_reset, }; @@ -27,10 +29,10 @@ use crate::validation::types::validate_value; use crate::wit::aws::S3RuntimeData; use crate::wit::core::{FuncParams, MaterializerId, RuntimeId, TypeId as CoreTypeId}; use crate::wit::runtimes::{ - self as wit, BaseMaterializer, Error as TgError, GraphqlRuntimeData, GrpcMaterializer, - GrpcRuntimeData, HttpRuntimeData, KvMaterializer, KvRuntimeData, MaterializerHttpRequest, - PrismaLinkData, PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, - SubstantialRuntimeData, TemporalOperationData, TemporalRuntimeData, WasmRuntimeData, + self as wit, BaseMaterializer, Error as TgError, GraphqlRuntimeData, GrpcData, GrpcRuntimeData, + HttpRuntimeData, KvMaterializer, KvRuntimeData, MaterializerHttpRequest, PrismaLinkData, + PrismaMigrationOperation, PrismaRuntimeData, RandomRuntimeData, SubstantialRuntimeData, + TemporalOperationData, TemporalRuntimeData, WasmRuntimeData, }; use crate::{typegraph::TypegraphContext, wit::runtimes::Effect as WitEffect}; use enum_dispatch::enum_dispatch; @@ -39,6 +41,7 @@ use substantial::{substantial_operation, SubstantialMaterializer}; use self::aws::S3Materializer; pub use self::deno::{DenoMaterializer, MaterializerDenoImport, MaterializerDenoModule}; pub use self::graphql::GraphqlMaterializer; +use self::grpc::GrpcMaterializer; use self::prisma::context::PrismaContext; use self::prisma::get_prisma_context; use self::prisma::relationship::prisma_link; @@ -710,11 +713,23 @@ impl crate::wit::runtimes::Guest for crate::Lib { Ok(Store::register_runtime(Runtime::Grpc(data.into()))) } - fn call_grpc_methode( - base: BaseMaterializer, - data: GrpcMaterializer, - ) -> Result<MaterializerId, wit::Error> { - let mat = Materializer::grpc(base.runtime, data, base.effect); - Ok(Store::register_materializer(mat)) + fn call_grpc_method(runtime: RuntimeId, data: GrpcData) -> Result<FuncParams, wit::Error> { + let _grpc_runtime_data = get_gprc_data(runtime); + let mat = GrpcMaterializer { + method: data.method, + }; + + #[allow(unused_variables)] + let mat_id = + Store::register_materializer(Materializer::grpc(runtime, mat, WitEffect::Read)); + + // generate_type() + + Ok(FuncParams { + inp: todo!(), + #[allow(unreachable_code)] + out: todo!(), + mat: mat_id, + }) } } diff --git a/typegraph/core/wit/typegraph.wit b/typegraph/core/wit/typegraph.wit index 8e42a86607..4a07ca4460 100644 --- a/typegraph/core/wit/typegraph.wit +++ b/typegraph/core/wit/typegraph.wit @@ -543,11 +543,11 @@ interface runtimes { register-grpc-runtime: func(data: grpc-runtime-data) -> result<runtime-id, error>; - record grpc-materializer { + record grpc-data { method: string, } - call-grpc-methode: func(base: base-materializer, data: grpc-materializer) -> result<materializer-id, error>; + call-grpc-method: func(runtime: runtime-id, data: grpc-data) -> result<func-params, error>; } interface aws { diff --git a/typegraph/deno/sdk/src/runtimes/grpc.ts b/typegraph/deno/sdk/src/runtimes/grpc.ts index 62297dc7cc..db3feea641 100644 --- a/typegraph/deno/sdk/src/runtimes/grpc.ts +++ b/typegraph/deno/sdk/src/runtimes/grpc.ts @@ -1,19 +1,9 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -import { Effect, GrpcMaterializer } from "../gen/typegraph_core.d.ts"; -import * as t from "../types.ts"; +import { Func } from "../types.ts"; import { runtimes } from "../wit.ts"; -import { Materializer, Runtime } from "./mod.ts"; - -class CallGrpcMethodMat implements Materializer { - mat: GrpcMaterializer; - _id: number; - constructor(id: number, mat: GrpcMaterializer) { - this._id = id; - this.mat = mat; - } -} +import { Runtime } from "./mod.ts"; export class GrpcRuntime extends Runtime { constructor(protoFile: string, endpoint: string) { @@ -24,21 +14,8 @@ export class GrpcRuntime extends Runtime { super(id); } - call_grpc_method(method: string, fx: Effect) { - const grpc_materializer: GrpcMaterializer = { - method: method, - }; - const mat_id = runtimes.callGrpcMethode( - { runtime: this._id, effect: fx }, - grpc_materializer, - ); - - const mat = new CallGrpcMethodMat(mat_id, grpc_materializer); - - return t.func( - t.struct({ "payload": t.string().optional() }), - t.string(), - mat, - ); + call_grpc_method(method: string) { + const funcData = runtimes.callGrpcMethod(this._id, { method: method }); + return Func.fromTypeFunc(funcData); } } diff --git a/typegraph/python/typegraph/runtimes/grpc.py b/typegraph/python/typegraph/runtimes/grpc.py index a53b649531..d2ba98b156 100644 --- a/typegraph/python/typegraph/runtimes/grpc.py +++ b/typegraph/python/typegraph/runtimes/grpc.py @@ -1,13 +1,10 @@ # Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. # SPDX-License-Identifier: MPL-2.0 -from dataclasses import dataclass from typegraph import t -from typegraph.runtimes.base import Materializer, Runtime +from typegraph.runtimes.base import Runtime from typegraph.gen.exports.runtimes import ( - BaseMaterializer, - Effect, - GrpcMaterializer, + GrpcData, GrpcRuntimeData, ) from typegraph.gen.types import Err @@ -23,21 +20,11 @@ def __init__(self, proto_file: str, endpoint: str): super().__init__(runtime_id.value) - def call_grpc_method(self, method: str, effect: Effect): - base = BaseMaterializer(self.id, effect) + def call_grpc_method(self, method: str): + data = GrpcData(method) + func_data = runtimes.call_grpc_method(store, self.id, data) - grpc_materialier = GrpcMaterializer(method) + if isinstance(func_data, Err): + raise Exception(func_data.value) - mat_id = runtimes.call_grpc_methode(store, base, grpc_materialier) - - if isinstance(mat_id, Err): - raise Exception(mat_id.value) - - mat = CallGrpcMethodMat(mat_id.value, effect, mat=grpc_materialier) - - return t.func(t.struct({"payload": t.optional(t.string())}), t.string(), mat) - - -@dataclass -class CallGrpcMethodMat(Materializer): - mat: GrpcMaterializer + return t.func.from_type_func(func_data.value) From 4f7f0a6f13fbcd8249145c32ea48fb21bf0ca943 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 29 Aug 2024 09:05:38 +0300 Subject: [PATCH 11/34] fix: return on call_grpc_method --- typegate/engine/bindings.ts | 2 +- .../core/src/runtimes/grpc/type_generation.rs | 54 +++++++++++++++++++ typegraph/core/src/runtimes/mod.rs | 8 ++- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/typegate/engine/bindings.ts b/typegate/engine/bindings.ts index a87e542f9c..fa5df7d634 100644 --- a/typegate/engine/bindings.ts +++ b/typegate/engine/bindings.ts @@ -512,7 +512,7 @@ export async function call_grpc_method( a0: CallGrpcMethodInput, ): Promise<CallGrpcMethodOutput> { try { - return await Meta.grpc.callGrpcMethod(a0); + return { Ok: await Meta.grpc.callGrpcMethod(a0) }; } catch (err) { return { Err: { message: err.toString() } }; } diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 23d9e15ccf..303d5e7c27 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -1,2 +1,56 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 +use protobuf::descriptor::{FileDescriptorProto, MethodDescriptorProto}; +use protobuf::reflect::MessageDescriptor; + +struct TypeInfo { + inp: Type, + out: Type, +} + +fn generate_type( + proto_file: &FileDescriptorProto, + method_name: &str, +) -> Result<TypeInfo, Box<dyn std::error::Error>> { + // Find the service and method + let method = find_method(proto_file, method_name)?; + + // Generate input type based on method arguments + let input_type = generate(proto_file, &method.get_input_type())?; + + // Generate output type based on method return + let output_type = generate(proto_file, &method.get_output_type())?; + + Ok(TypeInfo { + inp: input_type, + out: output_type, + }) +} + +fn find_method( + file: &FileDescriptorProto, + method_name: &str, +) -> Result<&MethodDescriptorProto, Box<dyn std::error::Error>> { + for service in file.get_service() { + if let Some(method) = service + .get_method() + .iter() + .find(|m| m.get_name() == method_name) + { + return Ok(method); + } + } + Err("Method not found".into()) +} + +fn find_message( + file: &FileDescriptorProto, + type_name: &str, +) -> Result<&protobuf::descriptor::DescriptorProto, Box<dyn std::error::Error>> { + for message in file.get_message_type() { + if message.get_name() == type_name { + return Ok(message); + } + } + Err("Message type not found".into()) +} diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index ccfb8a1dc9..540d1c9685 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -723,7 +723,13 @@ impl crate::wit::runtimes::Guest for crate::Lib { let mat_id = Store::register_materializer(Materializer::grpc(runtime, mat, WitEffect::Read)); - // generate_type() + // let type = generate_type(grpc_runtime_data.proto_file, data.method); + + // Ok(FuncParams { + // inp: type.inp, + // out: type.out, + // mat: mat_id, + // }) Ok(FuncParams { inp: todo!(), From b1abcb53d0dc0cf43d173be170389d245811cc78 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 29 Aug 2024 12:52:54 +0300 Subject: [PATCH 12/34] refact: move all grpc utils to libs/common/src/grpc.rs --- Cargo.lock | 12 +- libs/common/Cargo.toml | 6 + libs/common/src/grpc.rs | 175 +++++++++++++++++ libs/common/src/lib.rs | 1 + typegate/engine/Cargo.toml | 6 - typegate/engine/src/runtimes/grpc.rs | 178 ++---------------- typegraph/core/Cargo.toml | 3 + typegraph/core/src/runtimes/grpc/mod.rs | 2 + .../core/src/runtimes/grpc/type_generation.rs | 54 ------ 9 files changed, 205 insertions(+), 232 deletions(-) create mode 100644 libs/common/src/grpc.rs diff --git a/Cargo.lock b/Cargo.lock index 736e02cd82..cbc7ae8ad3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1577,18 +1577,23 @@ dependencies = [ "anyhow", "async-trait", "base64 0.22.1", + "bytes", "colored", "flate2", "ignore", "indexmap 2.2.6", "indoc", "itertools 0.11.0", + "protobuf", + "protobuf-json-mapping", + "protobuf-parse", "reqwest 0.11.27", "serde 1.0.204", "serde_json", "serde_with 3.9.0", "tar", "thiserror", + "tonic 0.12.1", "url", ] @@ -12301,7 +12306,6 @@ version = "0.4.8" dependencies = [ "anyhow", "base64 0.22.1", - "bytes", "common", "connection-string", "convert_case 0.6.0", @@ -12314,9 +12318,6 @@ dependencies = [ "mt_deno", "once_cell", "prisma-models", - "protobuf", - "protobuf-json-mapping", - "protobuf-parse", "psl", "query-connector", "query-core", @@ -12333,7 +12334,6 @@ dependencies = [ "temporal-sdk-core-protos", "thiserror", "tokio", - "tonic 0.12.1", "user-facing-errors", "wasmtime", "wasmtime-wasi", @@ -12357,6 +12357,8 @@ dependencies = [ "once_cell", "ordered-float 4.2.1", "paste", + "protobuf", + "protobuf-parse", "ptree", "regex", "seahash", diff --git a/libs/common/Cargo.toml b/libs/common/Cargo.toml index d783ebfe91..6564ac3d39 100644 --- a/libs/common/Cargo.toml +++ b/libs/common/Cargo.toml @@ -20,3 +20,9 @@ itertools = "0.11.0" colored = "2.0.4" indoc.workspace = true thiserror.workspace = true + +tonic = "0.12.1" +bytes = "1.7.1" +protobuf = "3.5.0" +protobuf-parse = "3.5.1" +protobuf-json-mapping = "3.5.1" diff --git a/libs/common/src/grpc.rs b/libs/common/src/grpc.rs new file mode 100644 index 0000000000..11a028161e --- /dev/null +++ b/libs/common/src/grpc.rs @@ -0,0 +1,175 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +use anyhow::{Context, Result}; +use bytes::{Buf, BufMut}; +use tonic::{ + client::Grpc, + codec::{Codec, DecodeBuf, Decoder, EncodeBuf, Encoder}, + transport::{Channel, Endpoint}, + IntoRequest, Request, Status, +}; + +pub use tonic::codegen::http::uri::PathAndQuery; + +use std::{path::Path, str::FromStr}; + +pub type Client = Grpc<Channel>; + +use protobuf::{ + descriptor::{FileDescriptorProto, MethodDescriptorProto}, + reflect::FileDescriptor, + MessageDyn, +}; + +type DynRequest = Box<dyn MessageDyn>; +type DynResponse = Box<dyn MessageDyn>; + +#[derive(Clone)] +pub struct DynCodec { + pub file_descriptor: FileDescriptor, + pub method_descriptor_proto: MethodDescriptorProto, +} + +impl Codec for DynCodec { + type Encode = DynRequest; + type Decode = DynResponse; + + type Encoder = DynCodec; + type Decoder = DynCodec; + + fn encoder(&mut self) -> Self::Encoder { + self.clone() + } + + fn decoder(&mut self) -> Self::Decoder { + self.clone() + } +} + +impl Encoder for DynCodec { + type Item = DynRequest; + + type Error = Status; + + fn encode( + &mut self, + item: Self::Item, + dst: &mut EncodeBuf<'_>, + ) -> std::prelude::v1::Result<(), Self::Error> { + item.write_to_bytes_dyn() + .map(|buf| dst.put(buf.as_slice())) + .map_err(|err| Status::internal(format!("{:?}", err))) + } +} + +impl Decoder for DynCodec { + type Item = DynResponse; + type Error = Status; + + fn decode(&mut self, src: &mut DecodeBuf<'_>) -> Result<Option<Self::Item>, Self::Error> { + let buf = src.chunk(); + let length = buf.len(); + + let response_message = + get_relative_message_name(self.method_descriptor_proto.output_type()).unwrap(); + + let response = buf2response(buf, response_message, self.file_descriptor.clone()) + .map(Some) + .map_err(|err| Status::internal(format!("{:?}", err))); + src.advance(length); + response + } +} + +pub fn json2request( + json: String, + input_message: String, + file_descriptor: FileDescriptor, +) -> anyhow::Result<Request<DynRequest>> { + let msg_descriptor = file_descriptor + .message_by_package_relative_name(&input_message) + .with_context(|| format!("Input message {input_message} not found"))?; + let mut msg = msg_descriptor.new_instance(); + protobuf_json_mapping::merge_from_str(&mut *msg, &json)?; + + Ok(msg.into_request()) +} + +fn buf2response( + buffer: &[u8], + output_message: String, + file_descriptor: FileDescriptor, +) -> anyhow::Result<DynResponse> { + let msg_descriptor = file_descriptor + .message_by_package_relative_name(&output_message) + .with_context(|| format!("Output message {output_message} not found"))?; + + let mut msg = msg_descriptor.new_instance(); + msg.merge_from_bytes_dyn(buffer)?; + + Ok(msg) +} + +pub async fn create_client(endpoint: &str) -> Result<Client> { + let endpoint = Endpoint::from_str(endpoint).context("Failed to parse endpoint")?; + + let channel = Channel::builder(endpoint.uri().to_owned()) + .connect() + .await + .context("Failed to connect to endpoint")?; + + Ok(Grpc::new(channel)) +} + +pub fn get_file_descriptor(proto_file: &Path) -> Result<FileDescriptor> { + let proto_folder = proto_file + .parent() + .context("Proto file is not within a folder")?; + + let mut file_descriptors_protos = protobuf_parse::Parser::new() + .include(proto_folder) + .input(proto_file) + .parse_and_typecheck() + .unwrap() + .file_descriptors; + + let file_descriptor_proto: FileDescriptorProto = file_descriptors_protos.pop().unwrap(); + + let file_descriptor = FileDescriptor::new_dynamic(file_descriptor_proto, &[])?; + + Ok(file_descriptor) +} + +pub fn get_method_descriptor_proto( + file_descriptor: FileDescriptor, + method_name: &str, +) -> Result<MethodDescriptorProto> { + let method = file_descriptor + .proto() + .service + .iter() + .flat_map(|service| &service.method) + .find(|method| method.name.as_ref().is_some_and(|name| name == method_name)) + .context("method descriptor not found")?; + + Ok(method.clone()) +} + +pub fn get_relative_method_name(absolute_method_name: &str) -> anyhow::Result<String> { + let path: Vec<&str> = absolute_method_name.split('/').collect(); + let method = path.get(2).context("Invalid path")?; + + Ok(method.to_string()) +} + +pub fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result<String> { + let path: Vec<&str> = absolute_message_name.split('.').collect(); + let message = path.get(2).context("Invalid path")?; + + Ok(message.to_string()) +} + +pub fn response_print_to_string(response: &dyn MessageDyn) -> String { + protobuf_json_mapping::print_to_string(response).unwrap() +} diff --git a/libs/common/src/lib.rs b/libs/common/src/lib.rs index 6effa7065d..9e9eee51cb 100644 --- a/libs/common/src/lib.rs +++ b/libs/common/src/lib.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Elastic-2.0 pub mod archive; +pub mod grpc; pub mod typegraph; // Note: diff --git a/typegate/engine/Cargo.toml b/typegate/engine/Cargo.toml index b18304f1e7..3ab8d5ba7a 100644 --- a/typegate/engine/Cargo.toml +++ b/typegate/engine/Cargo.toml @@ -47,12 +47,6 @@ wasmtime-wasi.workspace = true shadow-rs.workspace = true -tonic = "0.12.1" -protobuf = "3.5.0" -protobuf-parse = "3.5.1" -bytes = "1.7.1" -protobuf-json-mapping = "3.5.1" - [dev-dependencies] env_logger.workspace = true diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 736507e9dc..e436ef7b3f 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -1,176 +1,25 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -use std::{ - cell::RefCell, - ops::Deref, - path::{Path, PathBuf}, - rc::Rc, - str::FromStr, - sync::Arc, +use std::{cell::RefCell, ops::Deref, path::PathBuf, rc::Rc, str::FromStr, sync::Arc}; + +use common::grpc::{ + create_client, get_file_descriptor, get_method_descriptor_proto, get_relative_message_name, + get_relative_method_name, json2request, response_print_to_string, Client, DynCodec, + PathAndQuery, }; use dashmap::DashMap; use deno_core::OpState; -use protobuf::{ - descriptor::{FileDescriptorProto, MethodDescriptorProto}, - reflect::FileDescriptor, - MessageDyn, -}; use anyhow::{Context, Result}; -use bytes::{Buf, BufMut}; use serde::Deserialize; -use tonic::codegen::http::uri::PathAndQuery; -use tonic::{ - client::Grpc, - codec::{Codec, DecodeBuf, Decoder, EncodeBuf, Encoder}, - transport::{Channel, Endpoint}, -}; -use tonic::{IntoRequest, Status}; #[rustfmt::skip] use deno_core as deno_core; -type DynRequest = Box<dyn MessageDyn>; -type DynResponse = Box<dyn MessageDyn>; - -#[derive(Clone)] -pub struct DynCodec { - file_descriptor: FileDescriptor, - method_descriptor_proto: MethodDescriptorProto, -} - -impl Codec for DynCodec { - type Encode = DynRequest; - type Decode = DynResponse; - - type Encoder = DynCodec; - type Decoder = DynCodec; - - fn encoder(&mut self) -> Self::Encoder { - self.clone() - } - - fn decoder(&mut self) -> Self::Decoder { - self.clone() - } -} - -impl Encoder for DynCodec { - type Item = DynRequest; - - type Error = Status; - - fn encode( - &mut self, - item: Self::Item, - dst: &mut EncodeBuf<'_>, - ) -> std::prelude::v1::Result<(), Self::Error> { - item.write_to_bytes_dyn() - .map(|buf| dst.put(buf.as_slice())) - .map_err(|err| Status::internal(format!("{:?}", err))) - } -} - -impl Decoder for DynCodec { - type Item = DynResponse; - type Error = Status; - - fn decode(&mut self, src: &mut DecodeBuf<'_>) -> Result<Option<Self::Item>, Self::Error> { - let buf = src.chunk(); - let length = buf.len(); - - let response_message = - get_relative_message_name(self.method_descriptor_proto.output_type()).unwrap(); - - let response = buf2response(buf, response_message, self.file_descriptor.clone()) - .map(Some) - .map_err(|err| Status::internal(format!("{:?}", err))); - src.advance(length); - response - } -} - -fn json2request( - json: String, - input_message: String, - file_descriptor: FileDescriptor, -) -> anyhow::Result<DynRequest> { - let msg_descriptor = file_descriptor - .message_by_package_relative_name(&input_message) - .with_context(|| format!("Input message {input_message} not found"))?; - let mut msg = msg_descriptor.new_instance(); - protobuf_json_mapping::merge_from_str(&mut *msg, &json)?; - - Ok(msg) -} - -fn buf2response( - buffer: &[u8], - output_message: String, - file_descriptor: FileDescriptor, -) -> anyhow::Result<DynResponse> { - let msg_descriptor = file_descriptor - .message_by_package_relative_name(&output_message) - .with_context(|| format!("Output message {output_message} not found"))?; - - let mut msg = msg_descriptor.new_instance(); - msg.merge_from_bytes_dyn(buffer)?; - - Ok(msg) -} - -fn get_file_descriptor(proto_file: &Path) -> Result<FileDescriptor> { - let proto_folder = proto_file - .parent() - .context("Proto file is not within a folder")?; - - let mut file_descriptors_protos = protobuf_parse::Parser::new() - .include(proto_folder) - .input(proto_file) - .parse_and_typecheck() - .unwrap() - .file_descriptors; - - let file_descriptor_proto: FileDescriptorProto = file_descriptors_protos.pop().unwrap(); - - let file_descriptor = FileDescriptor::new_dynamic(file_descriptor_proto, &[])?; - - Ok(file_descriptor) -} - -fn get_method_descriptor_proto( - file_descriptor: FileDescriptor, - method_name: &str, -) -> Result<MethodDescriptorProto> { - let method = file_descriptor - .proto() - .service - .iter() - .flat_map(|service| &service.method) - .find(|method| method.name.as_ref().is_some_and(|name| name == method_name)) - .context("method descriptor not found")?; - - Ok(method.clone()) -} - -fn get_relative_method_name(absolute_method_name: &str) -> anyhow::Result<String> { - let path: Vec<&str> = absolute_method_name.split('/').collect(); - let method = path.get(2).context("Invalid path")?; - - Ok(method.to_string()) -} - -fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result<String> { - let path: Vec<&str> = absolute_message_name.split('.').collect(); - let message = path.get(2).context("Invalid path")?; - - Ok(message.to_string()) -} - struct GrpcClient { - client: Grpc<Channel>, + client: Client, proto_file: String, } @@ -192,18 +41,13 @@ pub async fn op_grpc_register( state: Rc<RefCell<OpState>>, #[serde] input: GrpcRegisterInput, ) -> Result<()> { - let endpoint = Endpoint::from_str(&input.endpoint).context("Failed to parse endpoint")?; - - let channel = Channel::builder(endpoint.uri().to_owned()) - .connect() - .await - .context("Failed to estabilish channel connection")?; + let client = create_client(&input.endpoint).await?; let state = state.borrow(); let ctx = state.borrow::<Ctx>(); let grpc_client = GrpcClient { - client: Grpc::new(channel), + client, proto_file: input.proto_file, }; ctx.grpc_clients @@ -256,7 +100,7 @@ pub async fn op_call_grpc_method( let request_message = get_relative_message_name(method_descriptor_proto.input_type())?; - let req = json2request(input.payload, request_message, file_descriptor.clone())?.into_request(); + let req = json2request(input.payload, request_message, file_descriptor.clone())?; let path_query = PathAndQuery::from_str(input.method.as_str())?; @@ -275,7 +119,7 @@ pub async fn op_call_grpc_method( let response = response.get_ref().deref(); - let json_response = protobuf_json_mapping::print_to_string(response).unwrap(); + let json_response = response_print_to_string(response); Ok(json_response) } diff --git a/typegraph/core/Cargo.toml b/typegraph/core/Cargo.toml index b5bd7efb37..f92b9054ad 100644 --- a/typegraph/core/Cargo.toml +++ b/typegraph/core/Cargo.toml @@ -27,6 +27,9 @@ ordered-float = "4.2.0" glob = "0.3.1" unindent = "0.2.3" +protobuf = "3.5.1" +protobuf-parse = "3.5.1" + [dev-dependencies] insta = { version = "1.39.0", features = ["glob"] } ptree = "0.4.0" diff --git a/typegraph/core/src/runtimes/grpc/mod.rs b/typegraph/core/src/runtimes/grpc/mod.rs index cf64c70f7a..1e2c1efa78 100644 --- a/typegraph/core/src/runtimes/grpc/mod.rs +++ b/typegraph/core/src/runtimes/grpc/mod.rs @@ -15,6 +15,8 @@ use serde_json::{from_value, json}; use super::Runtime; +pub mod type_generation; + pub fn get_gprc_data(runtime_id: RuntimeId) -> Rc<GrpcRuntimeData> { match Store::get_runtime(runtime_id).unwrap() { Runtime::Grpc(data) => data, diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 303d5e7c27..23d9e15ccf 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -1,56 +1,2 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -use protobuf::descriptor::{FileDescriptorProto, MethodDescriptorProto}; -use protobuf::reflect::MessageDescriptor; - -struct TypeInfo { - inp: Type, - out: Type, -} - -fn generate_type( - proto_file: &FileDescriptorProto, - method_name: &str, -) -> Result<TypeInfo, Box<dyn std::error::Error>> { - // Find the service and method - let method = find_method(proto_file, method_name)?; - - // Generate input type based on method arguments - let input_type = generate(proto_file, &method.get_input_type())?; - - // Generate output type based on method return - let output_type = generate(proto_file, &method.get_output_type())?; - - Ok(TypeInfo { - inp: input_type, - out: output_type, - }) -} - -fn find_method( - file: &FileDescriptorProto, - method_name: &str, -) -> Result<&MethodDescriptorProto, Box<dyn std::error::Error>> { - for service in file.get_service() { - if let Some(method) = service - .get_method() - .iter() - .find(|m| m.get_name() == method_name) - { - return Ok(method); - } - } - Err("Method not found".into()) -} - -fn find_message( - file: &FileDescriptorProto, - type_name: &str, -) -> Result<&protobuf::descriptor::DescriptorProto, Box<dyn std::error::Error>> { - for message in file.get_message_type() { - if message.get_name() == type_name { - return Ok(message); - } - } - Err("Message type not found".into()) -} From 651c2d0c291262d7bee0200896ee8fd66aaa87e1 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 29 Aug 2024 16:26:24 +0300 Subject: [PATCH 13/34] add type_gen for grpc next step refactoring --- libs/common/src/grpc.rs | 15 +++++- .../core/src/runtimes/grpc/type_generation.rs | 54 +++++++++++++++++++ typegraph/core/src/runtimes/mod.rs | 21 +++----- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/libs/common/src/grpc.rs b/libs/common/src/grpc.rs index 11a028161e..cafbeb6970 100644 --- a/libs/common/src/grpc.rs +++ b/libs/common/src/grpc.rs @@ -18,7 +18,7 @@ pub type Client = Grpc<Channel>; use protobuf::{ descriptor::{FileDescriptorProto, MethodDescriptorProto}, - reflect::FileDescriptor, + reflect::{FieldDescriptor, FileDescriptor}, MessageDyn, }; @@ -173,3 +173,16 @@ pub fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result< pub fn response_print_to_string(response: &dyn MessageDyn) -> String { protobuf_json_mapping::print_to_string(response).unwrap() } + +pub type Fields = Vec<FieldDescriptor>; + +pub fn get_message_field_descriptor( + file_descriptor: &FileDescriptor, + type_name: &str, +) -> Result<Fields> { + let message_descriptor = file_descriptor + .message_by_full_name(type_name) + .context(format!("Message not found: {}", type_name))?; + + Ok(message_descriptor.fields().collect()) +} diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 23d9e15ccf..58070d793a 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -1,2 +1,56 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 + +use anyhow::Result; +use common::grpc::{ + get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, + get_relative_method_name, Fields, +}; + +use std::path::Path; + +use crate::{ + t::{self, TypeBuilder}, + types::TypeId, +}; + +pub struct Type { + pub input: TypeId, + pub output: TypeId, +} + +pub fn generate_type(proto_file: &str, method_name: &str) -> Result<Type> { + let proto_path = Path::new(proto_file); + let file_descriptor = get_file_descriptor(proto_path)?; + + let method_descriptor = get_method_descriptor_proto(file_descriptor.clone(), method_name)?; + + let input_message = get_relative_method_name(&method_descriptor.input_type.unwrap())?; + + let out_message = get_relative_method_name(&method_descriptor.output_type.unwrap())?; + + let input_fields = get_message_field_descriptor(&file_descriptor, &input_message)?; + let output_fields = get_message_field_descriptor(&file_descriptor, &out_message)?; + + Ok(Type { + input: traduct(input_fields)?, + output: traduct(output_fields)?, + }) +} + +fn traduct(fields: Fields) -> Result<TypeId> { + let mut r#type = t::struct_(); + for field in fields { + let the_type = match field.proto().type_name() { + "string" => t::string().build()?, + "int32" => t::integer().build()?, + "int64" => t::integer().build()?, + "bool" => t::boolean().build()?, + "float" => t::float().build()?, + tt => panic!("{tt} is not type"), + }; + r#type.prop(field.name(), the_type); + } + + Ok(r#type.build()?) +} diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index 540d1c9685..ce62e934a1 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -41,6 +41,7 @@ use substantial::{substantial_operation, SubstantialMaterializer}; use self::aws::S3Materializer; pub use self::deno::{DenoMaterializer, MaterializerDenoImport, MaterializerDenoModule}; pub use self::graphql::GraphqlMaterializer; +use self::grpc::type_generation::generate_type; use self::grpc::GrpcMaterializer; use self::prisma::context::PrismaContext; use self::prisma::get_prisma_context; @@ -714,27 +715,21 @@ impl crate::wit::runtimes::Guest for crate::Lib { } fn call_grpc_method(runtime: RuntimeId, data: GrpcData) -> Result<FuncParams, wit::Error> { - let _grpc_runtime_data = get_gprc_data(runtime); + let grpc_runtime_data = get_gprc_data(runtime); + let mat = GrpcMaterializer { - method: data.method, + method: data.method.clone(), }; - #[allow(unused_variables)] let mat_id = Store::register_materializer(Materializer::grpc(runtime, mat, WitEffect::Read)); - // let type = generate_type(grpc_runtime_data.proto_file, data.method); - - // Ok(FuncParams { - // inp: type.inp, - // out: type.out, - // mat: mat_id, - // }) + let t = generate_type(&grpc_runtime_data.proto_file, &data.method) + .map_err(|err| err.to_string())?; Ok(FuncParams { - inp: todo!(), - #[allow(unreachable_code)] - out: todo!(), + inp: t.input.0, + out: t.output.0, mat: mat_id, }) } From 1498f5de7177c970848f8f9170ce5cd5e7293065 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 29 Aug 2024 17:40:47 +0300 Subject: [PATCH 14/34] refact: make method call_grpc_method more minimal --- typegraph/core/src/runtimes/grpc/mod.rs | 52 +++++++++++++------ .../core/src/runtimes/grpc/type_generation.rs | 4 +- typegraph/core/src/runtimes/mod.rs | 22 +------- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/typegraph/core/src/runtimes/grpc/mod.rs b/typegraph/core/src/runtimes/grpc/mod.rs index 1e2c1efa78..2225e8446d 100644 --- a/typegraph/core/src/runtimes/grpc/mod.rs +++ b/typegraph/core/src/runtimes/grpc/mod.rs @@ -1,29 +1,22 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 +mod type_generation; + use std::rc::Rc; -use crate::errors::Result; +use super::Runtime; use crate::global_store::Store; -use crate::wit::core::RuntimeId; -use crate::wit::runtimes::{Effect as WitEffect, GrpcRuntimeData}; -use crate::{conversion::runtimes::MaterializerConverter, typegraph::TypegraphContext}; +use crate::wit::core::{FuncParams, RuntimeId}; +use crate::wit::runtimes::{Effect as WitEffect, GrpcData, GrpcRuntimeData}; +use crate::{ + conversion::runtimes::MaterializerConverter, errors::Result, typegraph::TypegraphContext, +}; use common::typegraph::Materializer; use serde_json::{from_value, json}; -use super::Runtime; - -pub mod type_generation; - -pub fn get_gprc_data(runtime_id: RuntimeId) -> Rc<GrpcRuntimeData> { - match Store::get_runtime(runtime_id).unwrap() { - Runtime::Grpc(data) => data, - _ => unreachable!(), - } -} - #[derive(Debug)] pub struct GrpcMaterializer { pub method: String, @@ -35,7 +28,7 @@ impl MaterializerConverter for GrpcMaterializer { c: &mut TypegraphContext, runtime_id: RuntimeId, effect: WitEffect, - ) -> Result<common::typegraph::Materializer> { + ) -> Result<Materializer> { let runtime = c.register_runtime(runtime_id)?; let data = from_value(json!({"method": self.method})).map_err(|e| e.to_string())?; Ok(Materializer { @@ -46,3 +39,30 @@ impl MaterializerConverter for GrpcMaterializer { }) } } + +fn get_gprc_data(runtime_id: RuntimeId) -> Rc<GrpcRuntimeData> { + match Store::get_runtime(runtime_id).unwrap() { + Runtime::Grpc(data) => data, + _ => unreachable!(), + } +} + +pub fn call_grpc_method(runtime: RuntimeId, data: GrpcData) -> Result<FuncParams> { + let grpc_runtime_data = get_gprc_data(runtime); + + let mat = GrpcMaterializer { + method: data.method.clone(), + }; + + let mat_id = + Store::register_materializer(super::Materializer::grpc(runtime, mat, WitEffect::Read)); + + let t = type_generation::generate_type(&grpc_runtime_data.proto_file, &data.method) + .map_err(|err| err.to_string())?; + + Ok(FuncParams { + inp: t.input.0, + out: t.output.0, + mat: mat_id, + }) +} diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 58070d793a..546c886f6a 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -1,14 +1,14 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 +use std::path::Path; + use anyhow::Result; use common::grpc::{ get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, get_relative_method_name, Fields, }; -use std::path::Path; - use crate::{ t::{self, TypeBuilder}, types::TypeId, diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index ce62e934a1..4afd7d4cfe 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -19,7 +19,6 @@ use std::rc::Rc; use crate::conversion::runtimes::MaterializerConverter; use crate::global_store::Store; -use crate::runtimes::grpc::get_gprc_data; use crate::runtimes::prisma::migration::{ prisma_apply, prisma_create, prisma_deploy, prisma_diff, prisma_reset, }; @@ -41,8 +40,7 @@ use substantial::{substantial_operation, SubstantialMaterializer}; use self::aws::S3Materializer; pub use self::deno::{DenoMaterializer, MaterializerDenoImport, MaterializerDenoModule}; pub use self::graphql::GraphqlMaterializer; -use self::grpc::type_generation::generate_type; -use self::grpc::GrpcMaterializer; +use self::grpc::{call_grpc_method, GrpcMaterializer}; use self::prisma::context::PrismaContext; use self::prisma::get_prisma_context; use self::prisma::relationship::prisma_link; @@ -715,22 +713,6 @@ impl crate::wit::runtimes::Guest for crate::Lib { } fn call_grpc_method(runtime: RuntimeId, data: GrpcData) -> Result<FuncParams, wit::Error> { - let grpc_runtime_data = get_gprc_data(runtime); - - let mat = GrpcMaterializer { - method: data.method.clone(), - }; - - let mat_id = - Store::register_materializer(Materializer::grpc(runtime, mat, WitEffect::Read)); - - let t = generate_type(&grpc_runtime_data.proto_file, &data.method) - .map_err(|err| err.to_string())?; - - Ok(FuncParams { - inp: t.input.0, - out: t.output.0, - mat: mat_id, - }) + call_grpc_method(runtime, data) } } From 1aabc5e77434ccf24d7454e75d27e9236a2ec375 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 29 Aug 2024 22:25:56 +0300 Subject: [PATCH 15/34] remove tonic from common --- .ghjk/lock.json | 430 +++++++++++++++------------ Cargo.lock | 7 +- libs/common/Cargo.toml | 3 - libs/common/src/grpc.rs | 118 +------- typegate/engine/Cargo.toml | 5 + typegate/engine/src/runtimes/grpc.rs | 118 +++++++- 6 files changed, 366 insertions(+), 315 deletions(-) diff --git a/.ghjk/lock.json b/.ghjk/lock.json index b7703c0204..27da743a0d 100644 --- a/.ghjk/lock.json +++ b/.ghjk/lock.json @@ -1,6 +1,6 @@ { "version": "0", - "platform": "aarch64-darwin", + "platform": "x86_64-linux", "moduleEntries": { "ports": { "version": "0", @@ -742,11 +742,38 @@ "ghjkEnvProvInstSet____rust": { "installs": [ "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya" ], "allowedBuildDeps": "bciqgdwpugwdrwifr6qqi3m6vs4y55edt7xkojyxk5zr5m3joowsgkki" }, + "ghjkEnvProvInstSet___dev": { + "installs": [ + "bciqbx637744bfiyvprs77xdnvdt7uuwmtlntfjpwmkda672gklkbpmi", + "bciqdoqocirh7aseu5o5hfqaj3sb3pfd3z3rlvig26xttmcmsoljuz6i", + "bciqjsjvkjm6xvoovs6y3y6me32422zr5wc5njs4kwfmmyf6nt6jzv2i", + "bciqdf7jtq3zzhn676t2dr7fyve47fj7coajtymmye353lrrluskjk7q", + "bciqeaqeduyhykw7s7gq6ney6ci7deheq3etgdwkvg55mwbzdhz2opra", + "bciqdtuhf425g6prb5fyupbcokttmkill6wyqk7bkphx3ueltl5mvu4q", + "bciqmvgsg7h3ohj3m7das4bznahgt6tyq7mamta3n2vorulqvml7mywq", + "bciqicdqw36v63cbrscwsgtu2htrmwmgtfoxexv4rx5d2y24vytxbuma", + "bciqe33uhsuaesrjk6luzxrbbimwg5ydt6x2lrieelwbr7aft4g2qwsy", + "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", + "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", + "bciqgpwvq6qf34xolzh65zr3v5dpyhahtybei46nnjmy2sitnoixwhsa", + "bciqkgncxbauys2qfguplxcz2auxrcyamj4b6htqk2fqvohfm3afd7sa", + "bciqh7qqm2bohoswnwjywael2m3f6xn4n6ceifyw4usrmaahjioldq6a", + "bciqezep4ufkgwesldlm5etyfkgdsiickfudx7cosydcz6xtgeorn2hy", + "bciqaixkkacuuligsvtjcfdfgjgl65owtyspiiljb3vmutlgymecsiwq", + "bciqlt27ioikxnpkqq37hma7ibn5e5wpzfarbvoh77zwdkarwghtvzxa", + "bciqojan3zglnfctnmqyxvnxaha46yrnlhj77j3kw4mxadvauqepqdba", + "bciqcnbruy2q6trpvia52n2yis4t27taoz4mxkeguqz5aif7ex6rp26y", + "bciqpu7gxs3zm7i4gwp3m3cfdxwz27ixvsykdnbxrl5m5mt3xbb3b4la", + "bciqjme7csfq43oenkrsakdhaha34hgy6vdwkfffki2ank3kf6mjcguq" + ], + "allowedBuildDeps": "bciqgdwpugwdrwifr6qqi3m6vs4y55edt7xkojyxk5zr5m3joowsgkki" + }, "ghjkEnvProvInstSet____ecma": { "installs": [ "bciqezep4ufkgwesldlm5etyfkgdsiickfudx7cosydcz6xtgeorn2hy", @@ -782,33 +809,7 @@ "bciqicdqw36v63cbrscwsgtu2htrmwmgtfoxexv4rx5d2y24vytxbuma", "bciqe33uhsuaesrjk6luzxrbbimwg5ydt6x2lrieelwbr7aft4g2qwsy", "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", - "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", - "bciqgpwvq6qf34xolzh65zr3v5dpyhahtybei46nnjmy2sitnoixwhsa", - "bciqkgncxbauys2qfguplxcz2auxrcyamj4b6htqk2fqvohfm3afd7sa", - "bciqh7qqm2bohoswnwjywael2m3f6xn4n6ceifyw4usrmaahjioldq6a", - "bciqezep4ufkgwesldlm5etyfkgdsiickfudx7cosydcz6xtgeorn2hy", - "bciqaixkkacuuligsvtjcfdfgjgl65owtyspiiljb3vmutlgymecsiwq", - "bciqlt27ioikxnpkqq37hma7ibn5e5wpzfarbvoh77zwdkarwghtvzxa", - "bciqojan3zglnfctnmqyxvnxaha46yrnlhj77j3kw4mxadvauqepqdba", - "bciqcnbruy2q6trpvia52n2yis4t27taoz4mxkeguqz5aif7ex6rp26y", - "bciqpu7gxs3zm7i4gwp3m3cfdxwz27ixvsykdnbxrl5m5mt3xbb3b4la", - "bciqjme7csfq43oenkrsakdhaha34hgy6vdwkfffki2ank3kf6mjcguq" - ], - "allowedBuildDeps": "bciqgdwpugwdrwifr6qqi3m6vs4y55edt7xkojyxk5zr5m3joowsgkki" - }, - "ghjkEnvProvInstSet___dev": { - "installs": [ - "bciqdoqocirh7aseu5o5hfqaj3sb3pfd3z3rlvig26xttmcmsoljuz6i", - "bciqjsjvkjm6xvoovs6y3y6me32422zr5wc5njs4kwfmmyf6nt6jzv2i", - "bciqdf7jtq3zzhn676t2dr7fyve47fj7coajtymmye353lrrluskjk7q", - "bciqeaqeduyhykw7s7gq6ney6ci7deheq3etgdwkvg55mwbzdhz2opra", - "bciqdtuhf425g6prb5fyupbcokttmkill6wyqk7bkphx3ueltl5mvu4q", - "bciqmvgsg7h3ohj3m7das4bznahgt6tyq7mamta3n2vorulqvml7mywq", - "bciqicdqw36v63cbrscwsgtu2htrmwmgtfoxexv4rx5d2y24vytxbuma", - "bciqe33uhsuaesrjk6luzxrbbimwg5ydt6x2lrieelwbr7aft4g2qwsy", - "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", "bciqgpwvq6qf34xolzh65zr3v5dpyhahtybei46nnjmy2sitnoixwhsa", "bciqkgncxbauys2qfguplxcz2auxrcyamj4b6htqk2fqvohfm3afd7sa", @@ -846,7 +847,7 @@ "ghjkEnvProvInstSet_______task_env_test-rust": { "installs": [ "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", "bciqezep4ufkgwesldlm5etyfkgdsiickfudx7cosydcz6xtgeorn2hy", "bciqaixkkacuuligsvtjcfdfgjgl65owtyspiiljb3vmutlgymecsiwq", @@ -860,7 +861,7 @@ "ghjkEnvProvInstSet_______task_env_dev-eg-tgraphs": { "installs": [ "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", "bciqezep4ufkgwesldlm5etyfkgdsiickfudx7cosydcz6xtgeorn2hy", "bciqaixkkacuuligsvtjcfdfgjgl65owtyspiiljb3vmutlgymecsiwq", @@ -874,7 +875,7 @@ "ghjkEnvProvInstSet_______task_env_build-tgraph-core": { "installs": [ "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", "bciqojan3zglnfctnmqyxvnxaha46yrnlhj77j3kw4mxadvauqepqdba", "bciqcnbruy2q6trpvia52n2yis4t27taoz4mxkeguqz5aif7ex6rp26y", @@ -886,7 +887,7 @@ "ghjkEnvProvInstSet_______task_env_build-tgraph-py": { "installs": [ "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", "bciqojan3zglnfctnmqyxvnxaha46yrnlhj77j3kw4mxadvauqepqdba", "bciqcnbruy2q6trpvia52n2yis4t27taoz4mxkeguqz5aif7ex6rp26y", @@ -901,7 +902,7 @@ "ghjkEnvProvInstSet_______task_env_build-tgraph-ts": { "installs": [ "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", "bciqojan3zglnfctnmqyxvnxaha46yrnlhj77j3kw4mxadvauqepqdba", "bciqcnbruy2q6trpvia52n2yis4t27taoz4mxkeguqz5aif7ex6rp26y", @@ -916,7 +917,7 @@ "ghjkEnvProvInstSet_______task_env_build-sys-tgraphs": { "installs": [ "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", "bciqgpwvq6qf34xolzh65zr3v5dpyhahtybei46nnjmy2sitnoixwhsa", "bciqkgncxbauys2qfguplxcz2auxrcyamj4b6htqk2fqvohfm3afd7sa", @@ -927,7 +928,7 @@ "ghjkEnvProvInstSet___oci": { "installs": [ "bciqj5xfgcxzfw3tusoy4v53dcinxqxskfwe3lylcjevxl6mbroky5za", - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui", + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y", "bciqd5yzu6oghbke7anippco6lr6sgw2mv3c2esojkr3ta6f4ljwwuya", "bciqojan3zglnfctnmqyxvnxaha46yrnlhj77j3kw4mxadvauqepqdba", "bciqcnbruy2q6trpvia52n2yis4t27taoz4mxkeguqz5aif7ex6rp26y", @@ -1036,31 +1037,31 @@ "ty": "denoFile@v1", "key": "dev-gate5", "desc": "Launch the typegate from the latests published image.", - "envKey": "bciqf6ly2orfkhcrpjkb6jx2p2igz6bvllqpuhotzlfmcjzwymmwr55q" + "envKey": "bciqikizsspknj3opc7ld3zvrfnjlv37e67vhqcz6igd4dtvffgcyokq" }, "dev-gate4": { "ty": "denoFile@v1", "key": "dev-gate4", "desc": "Launch the typegate from the locally built typegate image.", - "envKey": "bciqf6ly2orfkhcrpjkb6jx2p2igz6bvllqpuhotzlfmcjzwymmwr55q" + "envKey": "bciqikizsspknj3opc7ld3zvrfnjlv37e67vhqcz6igd4dtvffgcyokq" }, "dev-gate3": { "ty": "denoFile@v1", "key": "dev-gate3", "desc": "Launch the typegate from a locally found meta bin.", - "envKey": "bciqf6ly2orfkhcrpjkb6jx2p2igz6bvllqpuhotzlfmcjzwymmwr55q" + "envKey": "bciqikizsspknj3opc7ld3zvrfnjlv37e67vhqcz6igd4dtvffgcyokq" }, "dev-gate2": { "ty": "denoFile@v1", "key": "dev-gate2", "desc": "Launch the typegate in sync mode.", - "envKey": "bciqolirh75zgvw6slig4cdhdupwepxo7tuqo53aoqmrptr5k66ubjei" + "envKey": "bciqh7p45mporp4lmdynwq3xdlhbkauaww7ggluc2bzqgpkxyw7j76fq" }, "dev-gate1": { "ty": "denoFile@v1", "key": "dev-gate1", "desc": "Launch the typegate in single-instance mode.", - "envKey": "bciqf6ly2orfkhcrpjkb6jx2p2igz6bvllqpuhotzlfmcjzwymmwr55q" + "envKey": "bciqikizsspknj3opc7ld3zvrfnjlv37e67vhqcz6igd4dtvffgcyokq" }, "dev-eg-tgraphs": { "ty": "denoFile@v1", @@ -1396,7 +1397,7 @@ } ] }, - "bciqf6ly2orfkhcrpjkb6jx2p2igz6bvllqpuhotzlfmcjzwymmwr55q": { + "bciqikizsspknj3opc7ld3zvrfnjlv37e67vhqcz6igd4dtvffgcyokq": { "provides": [ { "ty": "posix.envVar", @@ -1433,6 +1434,16 @@ "key": "REDIS_URL", "val": "redis://:password@localhost:6379/0" }, + { + "ty": "posix.envVar", + "key": "TG_SECRET", + "val": "a4lNi0PbEItlFZbus1oeH/+wyIxi9uH6TpL8AIqIaMBNvp7SESmuUBbfUwC0prxhGhZqHw8vMDYZAGMhSZ4fLw==" + }, + { + "ty": "posix.envVar", + "key": "TG_ADMIN_PASSWORD", + "val": "password" + }, { "ty": "posix.envVar", "key": "TG_PORT", @@ -1444,7 +1455,7 @@ } ] }, - "bciqolirh75zgvw6slig4cdhdupwepxo7tuqo53aoqmrptr5k66ubjei": { + "bciqh7p45mporp4lmdynwq3xdlhbkauaww7ggluc2bzqgpkxyw7j76fq": { "provides": [ { "ty": "posix.envVar", @@ -1481,6 +1492,16 @@ "key": "REDIS_URL", "val": "redis://:password@localhost:6379/0" }, + { + "ty": "posix.envVar", + "key": "TG_SECRET", + "val": "a4lNi0PbEItlFZbus1oeH/+wyIxi9uH6TpL8AIqIaMBNvp7SESmuUBbfUwC0prxhGhZqHw8vMDYZAGMhSZ4fLw==" + }, + { + "ty": "posix.envVar", + "key": "TG_ADMIN_PASSWORD", + "val": "password" + }, { "ty": "posix.envVar", "key": "TG_PORT", @@ -2111,52 +2132,64 @@ "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/protoc.ts" } }, - "bciqbas5ccadnnnvdp2mtkt7xjmhvdtkgqeib5c2ir74cmyyrwfgfvui": { + "bciqlubbahrp4pxohyffmn5yj52atjgmn5nxepmkdev6wtmvpbx7kr7y": { + "buildDepConfigs": { + "asdf_plugin_git": { + "pluginRepo": "https://github.com/asdf-community/asdf-cmake", + "portRef": "asdf_plugin_git@0.1.0" + } + }, + "resolutionDepConfigs": { + "asdf_plugin_git": { + "pluginRepo": "https://github.com/asdf-community/asdf-cmake", + "portRef": "asdf_plugin_git@0.1.0" + } + }, "port": { "ty": "denoWorker@v1", - "name": "pipi_pypi", + "name": "asdf", "platforms": [ "x86_64-linux", "aarch64-linux", "x86_64-darwin", - "aarch64-darwin", - "x86_64-windows", - "aarch64-windows", - "x86_64-freebsd", - "aarch64-freebsd", - "x86_64-netbsd", - "aarch64-netbsd", - "x86_64-aix", - "aarch64-aix", - "x86_64-solaris", - "aarch64-solaris", - "x86_64-illumos", - "aarch64-illumos", - "x86_64-android", - "aarch64-android" + "aarch64-darwin" ], "version": "0.1.0", "buildDeps": [ { - "name": "cpy_bs_ghrel" + "name": "curl_aa" + }, + { + "name": "git_aa" + }, + { + "name": "asdf_plugin_git" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pipi.ts" + "resolutionDeps": [ + { + "name": "curl_aa" + }, + { + "name": "git_aa" + }, + { + "name": "asdf_plugin_git" + } + ], + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/asdf.ts" }, - "packageName": "cmake" + "pluginRepo": "https://github.com/asdf-community/asdf-cmake", + "installType": "version" }, - "bciqezep4ufkgwesldlm5etyfkgdsiickfudx7cosydcz6xtgeorn2hy": { - "version": "20.8.0", + "bciqbx637744bfiyvprs77xdnvdt7uuwmtlntfjpwmkda672gklkbpmi": { + "version": "v2.4.0", "port": { "ty": "denoWorker@v1", - "name": "node_org", + "name": "mold_ghrel", "platforms": [ "aarch64-linux", - "x86_64-linux", - "aarch64-darwin", - "x86_64-darwin", - "aarch64-windows", - "x86_64-windows" + "x86_64-linux" ], "version": "0.1.0", "buildDeps": [ @@ -2164,14 +2197,14 @@ "name": "tar_aa" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/node.ts" - } + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/mold.ts" + }, + "replaceLd": true }, - "bciqaixkkacuuligsvtjcfdfgjgl65owtyspiiljb3vmutlgymecsiwq": { - "version": "v9.4.0", + "bciqdoqocirh7aseu5o5hfqaj3sb3pfd3z3rlvig26xttmcmsoljuz6i": { "port": { "ty": "denoWorker@v1", - "name": "pnpm_ghrel", + "name": "act_ghrel", "platforms": [ "aarch64-linux", "x86_64-linux", @@ -2181,14 +2214,13 @@ "x86_64-windows" ], "version": "0.1.0", - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pnpm.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/act.ts" } }, - "bciqlt27ioikxnpkqq37hma7ibn5e5wpzfarbvoh77zwdkarwghtvzxa": { - "version": "10.0.1", + "bciqjsjvkjm6xvoovs6y3y6me32422zr5wc5njs4kwfmmyf6nt6jzv2i": { "port": { "ty": "denoWorker@v1", - "name": "npmi_npm", + "name": "cargobi_cratesio", "platforms": [ "x86_64-linux", "aarch64-linux", @@ -2212,44 +2244,59 @@ "version": "0.1.0", "buildDeps": [ { - "name": "node_org" + "name": "cargo_binstall_ghrel" + }, + { + "name": "rust_rustup" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/npmi.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" }, - "packageName": "node-gyp" + "crateName": "whiz", + "locked": true }, - "bciqgpwvq6qf34xolzh65zr3v5dpyhahtybei46nnjmy2sitnoixwhsa": { - "version": "3.8.18", + "bciqdf7jtq3zzhn676t2dr7fyve47fj7coajtymmye353lrrluskjk7q": { "port": { "ty": "denoWorker@v1", - "name": "cpy_bs_ghrel", + "name": "cargobi_cratesio", "platforms": [ "x86_64-linux", "aarch64-linux", "x86_64-darwin", "aarch64-darwin", "x86_64-windows", - "aarch64-windows" + "aarch64-windows", + "x86_64-freebsd", + "aarch64-freebsd", + "x86_64-netbsd", + "aarch64-netbsd", + "x86_64-aix", + "aarch64-aix", + "x86_64-solaris", + "aarch64-solaris", + "x86_64-illumos", + "aarch64-illumos", + "x86_64-android", + "aarch64-android" ], "version": "0.1.0", "buildDeps": [ { - "name": "tar_aa" + "name": "cargo_binstall_ghrel" }, { - "name": "zstd_aa" + "name": "rust_rustup" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cpy_bs.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" }, - "releaseTag": "20240224" + "crateName": "wit-deps-cli", + "locked": true }, - "bciqkgncxbauys2qfguplxcz2auxrcyamj4b6htqk2fqvohfm3afd7sa": { - "version": "0.4.7", + "bciqeaqeduyhykw7s7gq6ney6ci7deheq3etgdwkvg55mwbzdhz2opra": { "port": { "ty": "denoWorker@v1", - "name": "pipi_pypi", + "name": "cargobi_cratesio", "platforms": [ "x86_64-linux", "aarch64-linux", @@ -2273,15 +2320,19 @@ "version": "0.1.0", "buildDeps": [ { - "name": "cpy_bs_ghrel" + "name": "cargo_binstall_ghrel" + }, + { + "name": "rust_rustup" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pipi.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" }, - "packageName": "ruff" + "crateName": "git-cliff", + "locked": true }, - "bciqh7qqm2bohoswnwjywael2m3f6xn4n6ceifyw4usrmaahjioldq6a": { - "version": "1.7.0", + "bciqdtuhf425g6prb5fyupbcokttmkill6wyqk7bkphx3ueltl5mvu4q": { + "version": "3.7.1", "port": { "ty": "denoWorker@v1", "name": "pipi_pypi", @@ -2313,10 +2364,27 @@ ], "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pipi.ts" }, - "packageName": "poetry" + "packageName": "pre-commit" }, - "bciqojan3zglnfctnmqyxvnxaha46yrnlhj77j3kw4mxadvauqepqdba": { - "version": "0.116.1", + "bciqmvgsg7h3ohj3m7das4bznahgt6tyq7mamta3n2vorulqvml7mywq": { + "version": "v0.13.1", + "port": { + "ty": "denoWorker@v1", + "name": "temporal_cli_ghrel", + "platforms": [ + "aarch64-linux", + "x86_64-linux", + "aarch64-darwin", + "x86_64-darwin", + "aarch64-windows", + "x86_64-windows" + ], + "version": "0.1.0", + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/temporal_cli.ts" + } + }, + "bciqicdqw36v63cbrscwsgtu2htrmwmgtfoxexv4rx5d2y24vytxbuma": { + "version": "1.33.0", "port": { "ty": "denoWorker@v1", "name": "cargobi_cratesio", @@ -2351,11 +2419,11 @@ ], "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" }, - "crateName": "wasm-opt", + "crateName": "cargo-insta", "locked": true }, - "bciqcnbruy2q6trpvia52n2yis4t27taoz4mxkeguqz5aif7ex6rp26y": { - "version": "1.208.1", + "bciqe33uhsuaesrjk6luzxrbbimwg5ydt6x2lrieelwbr7aft4g2qwsy": { + "version": "0.2.5", "port": { "ty": "denoWorker@v1", "name": "cargobi_cratesio", @@ -2390,49 +2458,40 @@ ], "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" }, - "crateName": "wasm-tools", + "crateName": "cross", "locked": true }, - "bciqpu7gxs3zm7i4gwp3m3cfdxwz27ixvsykdnbxrl5m5mt3xbb3b4la": { - "version": "0.13.4", + "bciqgpwvq6qf34xolzh65zr3v5dpyhahtybei46nnjmy2sitnoixwhsa": { + "version": "3.8.18", "port": { "ty": "denoWorker@v1", - "name": "pipi_pypi", + "name": "cpy_bs_ghrel", "platforms": [ "x86_64-linux", "aarch64-linux", "x86_64-darwin", "aarch64-darwin", "x86_64-windows", - "aarch64-windows", - "x86_64-freebsd", - "aarch64-freebsd", - "x86_64-netbsd", - "aarch64-netbsd", - "x86_64-aix", - "aarch64-aix", - "x86_64-solaris", - "aarch64-solaris", - "x86_64-illumos", - "aarch64-illumos", - "x86_64-android", - "aarch64-android" + "aarch64-windows" ], "version": "0.1.0", "buildDeps": [ { - "name": "cpy_bs_ghrel" + "name": "tar_aa" + }, + { + "name": "zstd_aa" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pipi.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cpy_bs.ts" }, - "packageName": "componentize-py" + "releaseTag": "20240224" }, - "bciqjme7csfq43oenkrsakdhaha34hgy6vdwkfffki2ank3kf6mjcguq": { - "version": "1.3.0", + "bciqkgncxbauys2qfguplxcz2auxrcyamj4b6htqk2fqvohfm3afd7sa": { + "version": "0.4.7", "port": { "ty": "denoWorker@v1", - "name": "npmi_npm", + "name": "pipi_pypi", "platforms": [ "x86_64-linux", "aarch64-linux", @@ -2456,15 +2515,15 @@ "version": "0.1.0", "buildDeps": [ { - "name": "node_org" + "name": "cpy_bs_ghrel" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/npmi.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pipi.ts" }, - "packageName": "@bytecodealliance/jco" + "packageName": "ruff" }, - "bciqdtuhf425g6prb5fyupbcokttmkill6wyqk7bkphx3ueltl5mvu4q": { - "version": "3.7.1", + "bciqh7qqm2bohoswnwjywael2m3f6xn4n6ceifyw4usrmaahjioldq6a": { + "version": "1.7.0", "port": { "ty": "denoWorker@v1", "name": "pipi_pypi", @@ -2496,13 +2555,13 @@ ], "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pipi.ts" }, - "packageName": "pre-commit" + "packageName": "poetry" }, - "bciqmvgsg7h3ohj3m7das4bznahgt6tyq7mamta3n2vorulqvml7mywq": { - "version": "v0.13.1", + "bciqezep4ufkgwesldlm5etyfkgdsiickfudx7cosydcz6xtgeorn2hy": { + "version": "20.8.0", "port": { "ty": "denoWorker@v1", - "name": "temporal_cli_ghrel", + "name": "node_org", "platforms": [ "aarch64-linux", "x86_64-linux", @@ -2512,14 +2571,36 @@ "x86_64-windows" ], "version": "0.1.0", - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/temporal_cli.ts" + "buildDeps": [ + { + "name": "tar_aa" + } + ], + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/node.ts" } }, - "bciqicdqw36v63cbrscwsgtu2htrmwmgtfoxexv4rx5d2y24vytxbuma": { - "version": "1.33.0", + "bciqaixkkacuuligsvtjcfdfgjgl65owtyspiiljb3vmutlgymecsiwq": { + "version": "v9.4.0", "port": { "ty": "denoWorker@v1", - "name": "cargobi_cratesio", + "name": "pnpm_ghrel", + "platforms": [ + "aarch64-linux", + "x86_64-linux", + "aarch64-darwin", + "x86_64-darwin", + "aarch64-windows", + "x86_64-windows" + ], + "version": "0.1.0", + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pnpm.ts" + } + }, + "bciqlt27ioikxnpkqq37hma7ibn5e5wpzfarbvoh77zwdkarwghtvzxa": { + "version": "10.0.1", + "port": { + "ty": "denoWorker@v1", + "name": "npmi_npm", "platforms": [ "x86_64-linux", "aarch64-linux", @@ -2543,19 +2624,15 @@ "version": "0.1.0", "buildDeps": [ { - "name": "cargo_binstall_ghrel" - }, - { - "name": "rust_rustup" + "name": "node_org" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/npmi.ts" }, - "crateName": "cargo-insta", - "locked": true + "packageName": "node-gyp" }, - "bciqe33uhsuaesrjk6luzxrbbimwg5ydt6x2lrieelwbr7aft4g2qwsy": { - "version": "0.2.5", + "bciqojan3zglnfctnmqyxvnxaha46yrnlhj77j3kw4mxadvauqepqdba": { + "version": "0.116.1", "port": { "ty": "denoWorker@v1", "name": "cargobi_cratesio", @@ -2590,26 +2667,11 @@ ], "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" }, - "crateName": "cross", + "crateName": "wasm-opt", "locked": true }, - "bciqdoqocirh7aseu5o5hfqaj3sb3pfd3z3rlvig26xttmcmsoljuz6i": { - "port": { - "ty": "denoWorker@v1", - "name": "act_ghrel", - "platforms": [ - "aarch64-linux", - "x86_64-linux", - "aarch64-darwin", - "x86_64-darwin", - "aarch64-windows", - "x86_64-windows" - ], - "version": "0.1.0", - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/act.ts" - } - }, - "bciqjsjvkjm6xvoovs6y3y6me32422zr5wc5njs4kwfmmyf6nt6jzv2i": { + "bciqcnbruy2q6trpvia52n2yis4t27taoz4mxkeguqz5aif7ex6rp26y": { + "version": "1.208.1", "port": { "ty": "denoWorker@v1", "name": "cargobi_cratesio", @@ -2644,13 +2706,14 @@ ], "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" }, - "crateName": "whiz", + "crateName": "wasm-tools", "locked": true }, - "bciqdf7jtq3zzhn676t2dr7fyve47fj7coajtymmye353lrrluskjk7q": { + "bciqpu7gxs3zm7i4gwp3m3cfdxwz27ixvsykdnbxrl5m5mt3xbb3b4la": { + "version": "0.13.4", "port": { "ty": "denoWorker@v1", - "name": "cargobi_cratesio", + "name": "pipi_pypi", "platforms": [ "x86_64-linux", "aarch64-linux", @@ -2674,21 +2737,18 @@ "version": "0.1.0", "buildDeps": [ { - "name": "cargo_binstall_ghrel" - }, - { - "name": "rust_rustup" + "name": "cpy_bs_ghrel" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/pipi.ts" }, - "crateName": "wit-deps-cli", - "locked": true + "packageName": "componentize-py" }, - "bciqeaqeduyhykw7s7gq6ney6ci7deheq3etgdwkvg55mwbzdhz2opra": { + "bciqjme7csfq43oenkrsakdhaha34hgy6vdwkfffki2ank3kf6mjcguq": { + "version": "1.3.0", "port": { "ty": "denoWorker@v1", - "name": "cargobi_cratesio", + "name": "npmi_npm", "platforms": [ "x86_64-linux", "aarch64-linux", @@ -2712,16 +2772,12 @@ "version": "0.1.0", "buildDeps": [ { - "name": "cargo_binstall_ghrel" - }, - { - "name": "rust_rustup" + "name": "node_org" } ], - "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/cargobi.ts" + "moduleSpecifier": "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/npmi.ts" }, - "crateName": "git-cliff", - "locked": true + "packageName": "@bytecodealliance/jco" }, "bciqezkigmtx5tweeflmn27yofgwybmgbat7g6jg4xmxamomsdpvenay": { "version": "nightly-2024-05-26", diff --git a/Cargo.lock b/Cargo.lock index cbc7ae8ad3..ff77f94a43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1577,7 +1577,6 @@ dependencies = [ "anyhow", "async-trait", "base64 0.22.1", - "bytes", "colored", "flate2", "ignore", @@ -1585,7 +1584,6 @@ dependencies = [ "indoc", "itertools 0.11.0", "protobuf", - "protobuf-json-mapping", "protobuf-parse", "reqwest 0.11.27", "serde 1.0.204", @@ -1593,7 +1591,6 @@ dependencies = [ "serde_with 3.9.0", "tar", "thiserror", - "tonic 0.12.1", "url", ] @@ -12306,6 +12303,7 @@ version = "0.4.8" dependencies = [ "anyhow", "base64 0.22.1", + "bytes", "common", "connection-string", "convert_case 0.6.0", @@ -12318,6 +12316,8 @@ dependencies = [ "mt_deno", "once_cell", "prisma-models", + "protobuf", + "protobuf-json-mapping", "psl", "query-connector", "query-core", @@ -12334,6 +12334,7 @@ dependencies = [ "temporal-sdk-core-protos", "thiserror", "tokio", + "tonic 0.12.1", "user-facing-errors", "wasmtime", "wasmtime-wasi", diff --git a/libs/common/Cargo.toml b/libs/common/Cargo.toml index 6564ac3d39..e0e2af8956 100644 --- a/libs/common/Cargo.toml +++ b/libs/common/Cargo.toml @@ -21,8 +21,5 @@ colored = "2.0.4" indoc.workspace = true thiserror.workspace = true -tonic = "0.12.1" -bytes = "1.7.1" protobuf = "3.5.0" protobuf-parse = "3.5.1" -protobuf-json-mapping = "3.5.1" diff --git a/libs/common/src/grpc.rs b/libs/common/src/grpc.rs index cafbeb6970..808c26baf1 100644 --- a/libs/common/src/grpc.rs +++ b/libs/common/src/grpc.rs @@ -2,126 +2,14 @@ // SPDX-License-Identifier: Elastic-2.0 use anyhow::{Context, Result}; -use bytes::{Buf, BufMut}; -use tonic::{ - client::Grpc, - codec::{Codec, DecodeBuf, Decoder, EncodeBuf, Encoder}, - transport::{Channel, Endpoint}, - IntoRequest, Request, Status, -}; - -pub use tonic::codegen::http::uri::PathAndQuery; - -use std::{path::Path, str::FromStr}; -pub type Client = Grpc<Channel>; +use std::path::Path; use protobuf::{ descriptor::{FileDescriptorProto, MethodDescriptorProto}, reflect::{FieldDescriptor, FileDescriptor}, - MessageDyn, }; -type DynRequest = Box<dyn MessageDyn>; -type DynResponse = Box<dyn MessageDyn>; - -#[derive(Clone)] -pub struct DynCodec { - pub file_descriptor: FileDescriptor, - pub method_descriptor_proto: MethodDescriptorProto, -} - -impl Codec for DynCodec { - type Encode = DynRequest; - type Decode = DynResponse; - - type Encoder = DynCodec; - type Decoder = DynCodec; - - fn encoder(&mut self) -> Self::Encoder { - self.clone() - } - - fn decoder(&mut self) -> Self::Decoder { - self.clone() - } -} - -impl Encoder for DynCodec { - type Item = DynRequest; - - type Error = Status; - - fn encode( - &mut self, - item: Self::Item, - dst: &mut EncodeBuf<'_>, - ) -> std::prelude::v1::Result<(), Self::Error> { - item.write_to_bytes_dyn() - .map(|buf| dst.put(buf.as_slice())) - .map_err(|err| Status::internal(format!("{:?}", err))) - } -} - -impl Decoder for DynCodec { - type Item = DynResponse; - type Error = Status; - - fn decode(&mut self, src: &mut DecodeBuf<'_>) -> Result<Option<Self::Item>, Self::Error> { - let buf = src.chunk(); - let length = buf.len(); - - let response_message = - get_relative_message_name(self.method_descriptor_proto.output_type()).unwrap(); - - let response = buf2response(buf, response_message, self.file_descriptor.clone()) - .map(Some) - .map_err(|err| Status::internal(format!("{:?}", err))); - src.advance(length); - response - } -} - -pub fn json2request( - json: String, - input_message: String, - file_descriptor: FileDescriptor, -) -> anyhow::Result<Request<DynRequest>> { - let msg_descriptor = file_descriptor - .message_by_package_relative_name(&input_message) - .with_context(|| format!("Input message {input_message} not found"))?; - let mut msg = msg_descriptor.new_instance(); - protobuf_json_mapping::merge_from_str(&mut *msg, &json)?; - - Ok(msg.into_request()) -} - -fn buf2response( - buffer: &[u8], - output_message: String, - file_descriptor: FileDescriptor, -) -> anyhow::Result<DynResponse> { - let msg_descriptor = file_descriptor - .message_by_package_relative_name(&output_message) - .with_context(|| format!("Output message {output_message} not found"))?; - - let mut msg = msg_descriptor.new_instance(); - msg.merge_from_bytes_dyn(buffer)?; - - Ok(msg) -} - -pub async fn create_client(endpoint: &str) -> Result<Client> { - let endpoint = Endpoint::from_str(endpoint).context("Failed to parse endpoint")?; - - let channel = Channel::builder(endpoint.uri().to_owned()) - .connect() - .await - .context("Failed to connect to endpoint")?; - - Ok(Grpc::new(channel)) -} - pub fn get_file_descriptor(proto_file: &Path) -> Result<FileDescriptor> { let proto_folder = proto_file .parent() @@ -170,10 +58,6 @@ pub fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result< Ok(message.to_string()) } -pub fn response_print_to_string(response: &dyn MessageDyn) -> String { - protobuf_json_mapping::print_to_string(response).unwrap() -} - pub type Fields = Vec<FieldDescriptor>; pub fn get_message_field_descriptor( diff --git a/typegate/engine/Cargo.toml b/typegate/engine/Cargo.toml index 3ab8d5ba7a..29766b316b 100644 --- a/typegate/engine/Cargo.toml +++ b/typegate/engine/Cargo.toml @@ -47,6 +47,11 @@ wasmtime-wasi.workspace = true shadow-rs.workspace = true +tonic = "0.12.1" +bytes = "1.7.1" +protobuf = "3.5.0" +protobuf-json-mapping = "3.5.1" + [dev-dependencies] env_logger.workspace = true diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index e436ef7b3f..85cb1021d6 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -4,22 +4,130 @@ use std::{cell::RefCell, ops::Deref, path::PathBuf, rc::Rc, str::FromStr, sync::Arc}; use common::grpc::{ - create_client, get_file_descriptor, get_method_descriptor_proto, get_relative_message_name, - get_relative_method_name, json2request, response_print_to_string, Client, DynCodec, - PathAndQuery, + get_file_descriptor, get_method_descriptor_proto, get_relative_message_name, + get_relative_method_name, }; +use bytes::{Buf, BufMut}; use dashmap::DashMap; use deno_core::OpState; use anyhow::{Context, Result}; +use protobuf::{descriptor::MethodDescriptorProto, reflect::FileDescriptor, MessageDyn}; use serde::Deserialize; #[rustfmt::skip] use deno_core as deno_core; +use tonic::codegen::http::uri::PathAndQuery; +use tonic::{ + client::Grpc, + codec::{Codec, DecodeBuf, Decoder, EncodeBuf, Encoder}, + transport::{Channel, Endpoint}, + IntoRequest, Request, Status, +}; + +type DynRequest = Box<dyn MessageDyn>; +type DynResponse = Box<dyn MessageDyn>; + +#[derive(Clone)] +pub struct DynCodec { + pub file_descriptor: FileDescriptor, + pub method_descriptor_proto: MethodDescriptorProto, +} + +impl Codec for DynCodec { + type Encode = DynRequest; + type Decode = DynResponse; + + type Encoder = DynCodec; + type Decoder = DynCodec; + + fn encoder(&mut self) -> Self::Encoder { + self.clone() + } + + fn decoder(&mut self) -> Self::Decoder { + self.clone() + } +} + +impl Encoder for DynCodec { + type Item = DynRequest; + + type Error = Status; + + fn encode( + &mut self, + item: Self::Item, + dst: &mut EncodeBuf<'_>, + ) -> std::prelude::v1::Result<(), Self::Error> { + item.write_to_bytes_dyn() + .map(|buf| dst.put(buf.as_slice())) + .map_err(|err| Status::internal(format!("{:?}", err))) + } +} + +impl Decoder for DynCodec { + type Item = DynResponse; + type Error = Status; + + fn decode(&mut self, src: &mut DecodeBuf<'_>) -> Result<Option<Self::Item>, Self::Error> { + let buf = src.chunk(); + let length = buf.len(); + + let response_message = + get_relative_message_name(self.method_descriptor_proto.output_type()).unwrap(); + + let response = buf2response(buf, response_message, self.file_descriptor.clone()) + .map(Some) + .map_err(|err| Status::internal(format!("{:?}", err))); + src.advance(length); + response + } +} + +async fn create_client(endpoint: &str) -> Result<Grpc<Channel>> { + let endpoint = Endpoint::from_str(endpoint).context("Failed to parse endpoint")?; + + let channel = Channel::builder(endpoint.uri().to_owned()) + .connect() + .await + .context("Failed to connect to endpoint")?; + + Ok(Grpc::new(channel)) +} + +pub fn json2request( + json: String, + input_message: String, + file_descriptor: FileDescriptor, +) -> anyhow::Result<Request<DynRequest>> { + let msg_descriptor = file_descriptor + .message_by_package_relative_name(&input_message) + .with_context(|| format!("Input message {input_message} not found"))?; + let mut msg = msg_descriptor.new_instance(); + protobuf_json_mapping::merge_from_str(&mut *msg, &json)?; + + Ok(msg.into_request()) +} + +fn buf2response( + buffer: &[u8], + output_message: String, + file_descriptor: FileDescriptor, +) -> anyhow::Result<DynResponse> { + let msg_descriptor = file_descriptor + .message_by_package_relative_name(&output_message) + .with_context(|| format!("Output message {output_message} not found"))?; + + let mut msg = msg_descriptor.new_instance(); + msg.merge_from_bytes_dyn(buffer)?; + + Ok(msg) +} struct GrpcClient { - client: Client, + client: Grpc<Channel>, proto_file: String, } @@ -119,7 +227,7 @@ pub async fn op_call_grpc_method( let response = response.get_ref().deref(); - let json_response = response_print_to_string(response); + let json_response = protobuf_json_mapping::print_to_string(response).unwrap(); Ok(json_response) } From f79375e88859d83304ae2a224304f2cb5cb6bb07 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Fri, 30 Aug 2024 20:28:47 +0300 Subject: [PATCH 16/34] add our own proto_parse it's not yet perfect but it's good start --- Cargo.lock | 19 -- libs/common/Cargo.toml | 4 +- libs/common/src/{grpc.rs => grpc/mod.rs} | 25 +-- libs/common/src/grpc/proto_parser.rs | 167 ++++++++++++++++++ typegate/engine/Cargo.toml | 2 +- typegate/engine/src/runtimes/grpc.rs | 10 +- typegraph/core/Cargo.toml | 2 - .../core/src/runtimes/grpc/type_generation.rs | 9 +- 8 files changed, 179 insertions(+), 59 deletions(-) rename libs/common/src/{grpc.rs => grpc/mod.rs} (68%) create mode 100644 libs/common/src/grpc/proto_parser.rs diff --git a/Cargo.lock b/Cargo.lock index ff77f94a43..2b3e216b81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1584,7 +1584,6 @@ dependencies = [ "indoc", "itertools 0.11.0", "protobuf", - "protobuf-parse", "reqwest 0.11.27", "serde 1.0.204", "serde_json", @@ -8569,22 +8568,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "protobuf-parse" -version = "3.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b0e9b447d099ae2c4993c0cbb03c7a9d6c937b17f2d56cfc0b1550e6fcfdb76" -dependencies = [ - "anyhow", - "indexmap 2.2.6", - "log", - "protobuf", - "protobuf-support", - "tempfile", - "thiserror", - "which 4.4.2", -] - [[package]] name = "protobuf-support" version = "3.5.1" @@ -12358,8 +12341,6 @@ dependencies = [ "once_cell", "ordered-float 4.2.1", "paste", - "protobuf", - "protobuf-parse", "ptree", "regex", "seahash", diff --git a/libs/common/Cargo.toml b/libs/common/Cargo.toml index e0e2af8956..81cbf19504 100644 --- a/libs/common/Cargo.toml +++ b/libs/common/Cargo.toml @@ -20,6 +20,4 @@ itertools = "0.11.0" colored = "2.0.4" indoc.workspace = true thiserror.workspace = true - -protobuf = "3.5.0" -protobuf-parse = "3.5.1" +protobuf = "3.5.1" diff --git a/libs/common/src/grpc.rs b/libs/common/src/grpc/mod.rs similarity index 68% rename from libs/common/src/grpc.rs rename to libs/common/src/grpc/mod.rs index 808c26baf1..8c29f7a55c 100644 --- a/libs/common/src/grpc.rs +++ b/libs/common/src/grpc/mod.rs @@ -1,34 +1,15 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -use anyhow::{Context, Result}; +pub mod proto_parser; -use std::path::Path; +use anyhow::{Context, Result}; use protobuf::{ - descriptor::{FileDescriptorProto, MethodDescriptorProto}, + descriptor::MethodDescriptorProto, reflect::{FieldDescriptor, FileDescriptor}, }; -pub fn get_file_descriptor(proto_file: &Path) -> Result<FileDescriptor> { - let proto_folder = proto_file - .parent() - .context("Proto file is not within a folder")?; - - let mut file_descriptors_protos = protobuf_parse::Parser::new() - .include(proto_folder) - .input(proto_file) - .parse_and_typecheck() - .unwrap() - .file_descriptors; - - let file_descriptor_proto: FileDescriptorProto = file_descriptors_protos.pop().unwrap(); - - let file_descriptor = FileDescriptor::new_dynamic(file_descriptor_proto, &[])?; - - Ok(file_descriptor) -} - pub fn get_method_descriptor_proto( file_descriptor: FileDescriptor, method_name: &str, diff --git a/libs/common/src/grpc/proto_parser.rs b/libs/common/src/grpc/proto_parser.rs new file mode 100644 index 0000000000..2809ea7e5d --- /dev/null +++ b/libs/common/src/grpc/proto_parser.rs @@ -0,0 +1,167 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +use anyhow::{anyhow, Result}; +use protobuf::{ + descriptor::{ + field_descriptor_proto::Type, DescriptorProto, EnumDescriptorProto, + EnumValueDescriptorProto, FieldDescriptorProto, FileDescriptorProto, + }, + reflect::FileDescriptor, +}; + +pub struct ProtoParser; + +impl ProtoParser { + pub fn parse(content: &str) -> Result<FileDescriptorProto> { + let mut file_descriptor = FileDescriptorProto::new(); + let mut current_message: Option<DescriptorProto> = None; + let mut current_enum: Option<EnumDescriptorProto> = None; + + for line in content.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with("//") { + continue; + } + + if line.starts_with("syntax") { + Self::parse_syntax(&mut file_descriptor, line)?; + } else if line.starts_with("package") { + Self::parse_package(&mut file_descriptor, line)?; + } else if line.starts_with("message") { + if let Some(message) = current_message.take() { + file_descriptor.message_type.push(message); + } + current_message = Some(Self::parse_message_start(line)?); + } else if line.starts_with("enum") { + if let Some(enum_type) = current_enum.take() { + file_descriptor.enum_type.push(enum_type); + } + current_enum = Some(Self::parse_enum_start(line)?); + } else if line.ends_with('}') { + if let Some(message) = current_message.take() { + file_descriptor.message_type.push(message); + } + if let Some(enum_type) = current_enum.take() { + file_descriptor.enum_type.push(enum_type); + } + } else if let Some(ref mut message) = current_message { + Self::parse_message_field(message, line)?; + } else if let Some(ref mut enum_type) = current_enum { + Self::parse_enum_value(enum_type, line)?; + } + } + + if let Some(message) = current_message.take() { + file_descriptor.message_type.push(message); + } + if let Some(enum_type) = current_enum.take() { + file_descriptor.enum_type.push(enum_type); + } + + Ok(file_descriptor) + } + + fn parse_syntax(file_descriptor: &mut FileDescriptorProto, line: &str) -> Result<()> { + let syntax = line + .split('=') + .nth(1) + .ok_or_else(|| anyhow!("Invalid syntax line"))? + .trim() + .trim_matches('"'); + file_descriptor.set_syntax(syntax.to_string()); + Ok(()) + } + + fn parse_package(file_descriptor: &mut FileDescriptorProto, line: &str) -> Result<()> { + let package = line + .split_whitespace() + .nth(1) + .ok_or_else(|| anyhow!("Invalid package line"))? + .trim_matches(';'); + file_descriptor.set_package(package.to_string()); + Ok(()) + } + + fn parse_message_start(line: &str) -> Result<DescriptorProto> { + let name = line + .split_whitespace() + .nth(1) + .ok_or_else(|| anyhow!("Invalid message line"))? + .trim_matches('{'); + let mut message = DescriptorProto::new(); + message.set_name(name.to_string()); + Ok(message) + } + + fn parse_message_field(message: &mut DescriptorProto, line: &str) -> Result<()> { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 4 { + return Err(anyhow!("Invalid field line")); + } + + let mut field = FieldDescriptorProto::new(); + field.set_name(parts[1].to_string()); + field.set_number(parts[3].trim_matches(';').parse()?); + + let field_type = match parts[0] { + "double" => Type::TYPE_DOUBLE, + "float" => Type::TYPE_FLOAT, + "int64" => Type::TYPE_INT64, + "uint64" => Type::TYPE_UINT64, + "int32" => Type::TYPE_INT32, + "fixed64" => Type::TYPE_FIXED64, + "fixed32" => Type::TYPE_FIXED32, + "bool" => Type::TYPE_BOOL, + "string" => Type::TYPE_STRING, + "group" => Type::TYPE_GROUP, + "message" => Type::TYPE_MESSAGE, + "bytes" => Type::TYPE_BYTES, + "uint32" => Type::TYPE_UINT32, + "enum" => Type::TYPE_ENUM, + "sfixed32" => Type::TYPE_SFIXED32, + "sfixed64" => Type::TYPE_SFIXED64, + "sint32" => Type::TYPE_SINT32, + "sint64" => Type::TYPE_SINT64, + _ => return Err(anyhow!("Unsupported field type: {}", parts[0])), + }; + field.set_type(field_type); + + message.field.push(field); + Ok(()) + } + + fn parse_enum_start(line: &str) -> Result<EnumDescriptorProto> { + let name = line + .split_whitespace() + .nth(1) + .ok_or_else(|| anyhow!("Invalid enum line"))? + .trim_matches('{'); + let mut enum_type = EnumDescriptorProto::new(); + enum_type.set_name(name.to_string()); + Ok(enum_type) + } + + fn parse_enum_value(enum_type: &mut EnumDescriptorProto, line: &str) -> Result<()> { + let parts: Vec<&str> = line.split('=').collect(); + if parts.len() != 2 { + return Err(anyhow!("Invalid enum value line")); + } + + let name = parts[0].trim().to_string(); + let number: i32 = parts[1].trim().trim_matches(';').parse()?; + + let mut enum_value = EnumValueDescriptorProto::new(); + enum_value.set_name(name); + enum_value.set_number(number); + + enum_type.value.push(enum_value); + Ok(()) + } +} + +pub fn get_file_descriptor(content: &str) -> Result<FileDescriptor> { + let file_descriptor_proto = ProtoParser::parse(content)?; + let file_descriptor = FileDescriptor::new_dynamic(file_descriptor_proto, &[])?; + Ok(file_descriptor) +} diff --git a/typegate/engine/Cargo.toml b/typegate/engine/Cargo.toml index 29766b316b..890eaa42b5 100644 --- a/typegate/engine/Cargo.toml +++ b/typegate/engine/Cargo.toml @@ -49,7 +49,7 @@ shadow-rs.workspace = true tonic = "0.12.1" bytes = "1.7.1" -protobuf = "3.5.0" +protobuf = "3.5.1" protobuf-json-mapping = "3.5.1" [dev-dependencies] diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 85cb1021d6..18af9e6d9a 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -1,11 +1,11 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -use std::{cell::RefCell, ops::Deref, path::PathBuf, rc::Rc, str::FromStr, sync::Arc}; +use std::{cell::RefCell, ops::Deref, rc::Rc, str::FromStr, sync::Arc}; use common::grpc::{ - get_file_descriptor, get_method_descriptor_proto, get_relative_message_name, - get_relative_method_name, + get_method_descriptor_proto, get_relative_message_name, get_relative_method_name, + proto_parser::get_file_descriptor, }; use bytes::{Buf, BufMut}; @@ -197,9 +197,7 @@ pub async fn op_call_grpc_method( .get_mut(&input.client_id) .with_context(|| format!("Could not find gRPC client '{}'", &input.client_id))?; - let path = PathBuf::from_str(&grpc_client.proto_file)?; - - let file_descriptor = get_file_descriptor(&path)?; + let file_descriptor = get_file_descriptor(&grpc_client.proto_file)?; let method_name = get_relative_method_name(&input.method)?; diff --git a/typegraph/core/Cargo.toml b/typegraph/core/Cargo.toml index f92b9054ad..f20119aa32 100644 --- a/typegraph/core/Cargo.toml +++ b/typegraph/core/Cargo.toml @@ -27,8 +27,6 @@ ordered-float = "4.2.0" glob = "0.3.1" unindent = "0.2.3" -protobuf = "3.5.1" -protobuf-parse = "3.5.1" [dev-dependencies] insta = { version = "1.39.0", features = ["glob"] } diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 546c886f6a..c493a46046 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -1,12 +1,10 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -use std::path::Path; - use anyhow::Result; use common::grpc::{ - get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, - get_relative_method_name, Fields, + get_message_field_descriptor, get_method_descriptor_proto, get_relative_method_name, + proto_parser::get_file_descriptor, Fields, }; use crate::{ @@ -20,8 +18,7 @@ pub struct Type { } pub fn generate_type(proto_file: &str, method_name: &str) -> Result<Type> { - let proto_path = Path::new(proto_file); - let file_descriptor = get_file_descriptor(proto_path)?; + let file_descriptor = get_file_descriptor(proto_file)?; let method_descriptor = get_method_descriptor_proto(file_descriptor.clone(), method_name)?; From abbd113a73289b43191bc5541f31a6dba9423708 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Fri, 30 Aug 2024 21:15:18 +0300 Subject: [PATCH 17/34] refact: rename traduct function && handle error on get_grpc_data --- typegraph/core/src/runtimes/grpc/mod.rs | 8 ++++---- typegraph/core/src/runtimes/grpc/type_generation.rs | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/typegraph/core/src/runtimes/grpc/mod.rs b/typegraph/core/src/runtimes/grpc/mod.rs index 2225e8446d..9cad004fd7 100644 --- a/typegraph/core/src/runtimes/grpc/mod.rs +++ b/typegraph/core/src/runtimes/grpc/mod.rs @@ -40,15 +40,15 @@ impl MaterializerConverter for GrpcMaterializer { } } -fn get_gprc_data(runtime_id: RuntimeId) -> Rc<GrpcRuntimeData> { - match Store::get_runtime(runtime_id).unwrap() { - Runtime::Grpc(data) => data, +fn get_gprc_data(runtime_id: RuntimeId) -> Result<Rc<GrpcRuntimeData>> { + match Store::get_runtime(runtime_id)? { + Runtime::Grpc(data) => Ok(data), _ => unreachable!(), } } pub fn call_grpc_method(runtime: RuntimeId, data: GrpcData) -> Result<FuncParams> { - let grpc_runtime_data = get_gprc_data(runtime); + let grpc_runtime_data = get_gprc_data(runtime)?; let mat = GrpcMaterializer { method: data.method.clone(), diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index c493a46046..8ab23e069b 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -30,12 +30,12 @@ pub fn generate_type(proto_file: &str, method_name: &str) -> Result<Type> { let output_fields = get_message_field_descriptor(&file_descriptor, &out_message)?; Ok(Type { - input: traduct(input_fields)?, - output: traduct(output_fields)?, + input: convert_proto_fields_to_type_id(input_fields)?, + output: convert_proto_fields_to_type_id(output_fields)?, }) } -fn traduct(fields: Fields) -> Result<TypeId> { +fn convert_proto_fields_to_type_id(fields: Fields) -> Result<TypeId> { let mut r#type = t::struct_(); for field in fields { let the_type = match field.proto().type_name() { From b8426ade0fe7da93d517582dc4f9cabc6b6f9531 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Sun, 1 Sep 2024 10:15:37 +0300 Subject: [PATCH 18/34] rename proto-file to proto_file_content generate type from proto_file_content --- libs/common/src/typegraph/runtimes/grpc.rs | 2 +- typegate/engine/runtime.d.ts | 2 +- typegate/engine/src/runtimes/grpc.rs | 16 ++++++++-------- typegate/src/runtimes/grpc.ts | 4 ++-- typegraph/core/src/conversion/runtimes.rs | 2 +- typegraph/core/src/runtimes/grpc/mod.rs | 2 +- .../core/src/runtimes/grpc/type_generation.rs | 4 ++-- typegraph/core/wit/typegraph.wit | 2 +- typegraph/deno/sdk/src/runtimes/grpc.ts | 5 ++++- typegraph/python/typegraph/runtimes/grpc.py | 5 ++++- 10 files changed, 25 insertions(+), 19 deletions(-) diff --git a/libs/common/src/typegraph/runtimes/grpc.rs b/libs/common/src/typegraph/runtimes/grpc.rs index 4bc5523e3c..02826f08ff 100644 --- a/libs/common/src/typegraph/runtimes/grpc.rs +++ b/libs/common/src/typegraph/runtimes/grpc.rs @@ -5,6 +5,6 @@ use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Clone, Debug)] pub struct GrpcRuntimeData { - pub proto_file: String, + pub proto_file_content: String, pub endpoint: String, } diff --git a/typegate/engine/runtime.d.ts b/typegate/engine/runtime.d.ts index 1992daeb3c..fadc6d9ab4 100644 --- a/typegate/engine/runtime.d.ts +++ b/typegate/engine/runtime.d.ts @@ -267,7 +267,7 @@ export type WitWireHandleResponse = }; export type GrpcRegisterInput = { - protoFile: string; + protoFileContent: string; endpoint: string; client_id: string; }; diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 18af9e6d9a..06a7c65962 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -8,16 +8,16 @@ use common::grpc::{ proto_parser::get_file_descriptor, }; +use anyhow::{Context, Result}; use bytes::{Buf, BufMut}; use dashmap::DashMap; -use deno_core::OpState; - -use anyhow::{Context, Result}; use protobuf::{descriptor::MethodDescriptorProto, reflect::FileDescriptor, MessageDyn}; -use serde::Deserialize; +use deno_core::OpState; +use serde::Deserialize; #[rustfmt::skip] use deno_core as deno_core; + use tonic::codegen::http::uri::PathAndQuery; use tonic::{ client::Grpc, @@ -128,7 +128,7 @@ fn buf2response( struct GrpcClient { client: Grpc<Channel>, - proto_file: String, + proto_file_content: String, } #[derive(Default)] @@ -139,7 +139,7 @@ pub struct Ctx { #[derive(Deserialize)] #[serde(crate = "serde")] pub struct GrpcRegisterInput { - proto_file: String, + proto_file_content: String, endpoint: String, client_id: String, } @@ -156,7 +156,7 @@ pub async fn op_grpc_register( let grpc_client = GrpcClient { client, - proto_file: input.proto_file, + proto_file_content: input.proto_file_content, }; ctx.grpc_clients .insert(input.client_id.clone(), grpc_client); @@ -197,7 +197,7 @@ pub async fn op_call_grpc_method( .get_mut(&input.client_id) .with_context(|| format!("Could not find gRPC client '{}'", &input.client_id))?; - let file_descriptor = get_file_descriptor(&grpc_client.proto_file)?; + let file_descriptor = get_file_descriptor(&grpc_client.proto_file_content)?; let method_name = get_relative_method_name(&input.method)?; diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index b07c749056..84bf3da346 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -13,7 +13,7 @@ import { registerRuntime } from "@typegate/runtimes/mod.ts"; const logger = getLogger(import.meta); interface GrpcRuntimeData { - protoFile: string; + protoFileContent: string; endpoint: string; } @@ -36,7 +36,7 @@ export class GrpcRuntime extends Runtime { nativeVoid( await native.grpc_register({ - protoFile: args.protoFile, + protoFileContent: args.protoFileContent, endpoint: args.endpoint, client_id: instance.id, }), diff --git a/typegraph/core/src/conversion/runtimes.rs b/typegraph/core/src/conversion/runtimes.rs index dad19397ca..3e227f793c 100644 --- a/typegraph/core/src/conversion/runtimes.rs +++ b/typegraph/core/src/conversion/runtimes.rs @@ -507,7 +507,7 @@ pub fn convert_runtime(_c: &mut TypegraphContext, runtime: Runtime) -> Result<Co } Runtime::Kv(d) => Ok(TGRuntime::Known(Rt::Kv(KvRuntimeData { url: d.url.clone() })).into()), Runtime::Grpc(d) => Ok(TGRuntime::Known(Rt::Grpc(GrpcRuntimeData { - proto_file: d.proto_file.clone(), + proto_file_content: d.proto_file_content.clone(), endpoint: d.endpoint.clone(), })) .into()), diff --git a/typegraph/core/src/runtimes/grpc/mod.rs b/typegraph/core/src/runtimes/grpc/mod.rs index 9cad004fd7..e633a04890 100644 --- a/typegraph/core/src/runtimes/grpc/mod.rs +++ b/typegraph/core/src/runtimes/grpc/mod.rs @@ -57,7 +57,7 @@ pub fn call_grpc_method(runtime: RuntimeId, data: GrpcData) -> Result<FuncParams let mat_id = Store::register_materializer(super::Materializer::grpc(runtime, mat, WitEffect::Read)); - let t = type_generation::generate_type(&grpc_runtime_data.proto_file, &data.method) + let t = type_generation::generate_type(&grpc_runtime_data.proto_file_content, &data.method) .map_err(|err| err.to_string())?; Ok(FuncParams { diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 8ab23e069b..504d8af5c1 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -17,8 +17,8 @@ pub struct Type { pub output: TypeId, } -pub fn generate_type(proto_file: &str, method_name: &str) -> Result<Type> { - let file_descriptor = get_file_descriptor(proto_file)?; +pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<Type> { + let file_descriptor = get_file_descriptor(proto_file_content)?; let method_descriptor = get_method_descriptor_proto(file_descriptor.clone(), method_name)?; diff --git a/typegraph/core/wit/typegraph.wit b/typegraph/core/wit/typegraph.wit index 4a07ca4460..1c95f7bb72 100644 --- a/typegraph/core/wit/typegraph.wit +++ b/typegraph/core/wit/typegraph.wit @@ -537,7 +537,7 @@ interface runtimes { // Grpc record grpc-runtime-data { - proto-file: string, + proto-file-content: string, endpoint: string, } diff --git a/typegraph/deno/sdk/src/runtimes/grpc.ts b/typegraph/deno/sdk/src/runtimes/grpc.ts index db3feea641..7fc9bc4d1f 100644 --- a/typegraph/deno/sdk/src/runtimes/grpc.ts +++ b/typegraph/deno/sdk/src/runtimes/grpc.ts @@ -7,8 +7,11 @@ import { Runtime } from "./mod.ts"; export class GrpcRuntime extends Runtime { constructor(protoFile: string, endpoint: string) { + const protoFileContent = new TextDecoder().decode( + Deno.readFileSync(protoFile), + ); const id = runtimes.registerGrpcRuntime({ - protoFile, + protoFileContent, endpoint, }); super(id); diff --git a/typegraph/python/typegraph/runtimes/grpc.py b/typegraph/python/typegraph/runtimes/grpc.py index d2ba98b156..f06f25c4f2 100644 --- a/typegraph/python/typegraph/runtimes/grpc.py +++ b/typegraph/python/typegraph/runtimes/grpc.py @@ -13,7 +13,10 @@ class GrpcRuntime(Runtime): def __init__(self, proto_file: str, endpoint: str): - data = GrpcRuntimeData(proto_file, endpoint) + file = open(proto_file, "r") + proto_file_content = file.read() + file.close() + data = GrpcRuntimeData(proto_file_content, endpoint) runtime_id = runtimes.register_grpc_runtime(store, data) if isinstance(runtime_id, Err): raise Exception(runtime_id.value) From a4b5caa9b5b7ba6dbafb8cee9eb87f2e668911c6 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Sun, 1 Sep 2024 16:44:16 +0300 Subject: [PATCH 19/34] add op_grpc_* to ext.rs fix error: about missing op_grpc_* --- libs/common/src/grpc/mod.rs | 4 +++- libs/common/src/grpc/proto_parser.rs | 4 ++-- typegate/engine/src/ext.rs | 5 ++++- typegate/engine/src/runtimes/grpc.rs | 7 +++++-- typegraph/core/src/runtimes/grpc/type_generation.rs | 4 ++-- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libs/common/src/grpc/mod.rs b/libs/common/src/grpc/mod.rs index 8c29f7a55c..c7db694bb9 100644 --- a/libs/common/src/grpc/mod.rs +++ b/libs/common/src/grpc/mod.rs @@ -1,7 +1,9 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -pub mod proto_parser; +mod proto_parser; + +pub use proto_parser::get_file_descriptor; use anyhow::{Context, Result}; diff --git a/libs/common/src/grpc/proto_parser.rs b/libs/common/src/grpc/proto_parser.rs index 2809ea7e5d..824a90f393 100644 --- a/libs/common/src/grpc/proto_parser.rs +++ b/libs/common/src/grpc/proto_parser.rs @@ -1,5 +1,5 @@ -// Copyright Metatype OÜ, licensed under the Elastic License 2.0. -// SPDX-License-Identifier: Elastic-2.0 +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 use anyhow::{anyhow, Result}; use protobuf::{ diff --git a/typegate/engine/src/ext.rs b/typegate/engine/src/ext.rs index 21d9bc80a9..f8afde0c6c 100644 --- a/typegate/engine/src/ext.rs +++ b/typegate/engine/src/ext.rs @@ -3,7 +3,7 @@ use crate::interlude::*; use crate::{ - runtimes::{prisma, temporal, wasm, wit_wire}, + runtimes::{grpc, prisma, temporal, wasm, wit_wire}, typegraph, }; @@ -40,6 +40,9 @@ deno_core::extension!( wit_wire::op_wit_wire_init, wit_wire::op_wit_wire_handle, wit_wire::op_wit_wire_destroy, + grpc::op_grpc_register, + grpc::op_grpc_unregister, + grpc::op_call_grpc_method // FIXME(yohe): this test broke and has proven difficult to fix // #[cfg(test)] // tests::op_obj_go_round, diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 06a7c65962..496379d2bd 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -1,11 +1,14 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + use std::{cell::RefCell, ops::Deref, rc::Rc, str::FromStr, sync::Arc}; use common::grpc::{ - get_method_descriptor_proto, get_relative_message_name, get_relative_method_name, - proto_parser::get_file_descriptor, + get_file_descriptor, get_method_descriptor_proto, get_relative_message_name, + get_relative_method_name, }; use anyhow::{Context, Result}; diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 504d8af5c1..097e188857 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -3,8 +3,8 @@ use anyhow::Result; use common::grpc::{ - get_message_field_descriptor, get_method_descriptor_proto, get_relative_method_name, - proto_parser::get_file_descriptor, Fields, + get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, + get_relative_method_name, Fields, }; use crate::{ From e24694bb05bfea9eff2c8759f684b1dd9a754309 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Sun, 1 Sep 2024 16:52:24 +0300 Subject: [PATCH 20/34] remove duplicate license --- typegate/engine/src/runtimes/grpc.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 496379d2bd..82caa471e8 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -1,9 +1,6 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. -// SPDX-License-Identifier: MPL-2.0 - use std::{cell::RefCell, ops::Deref, rc::Rc, str::FromStr, sync::Arc}; use common::grpc::{ From 40b0e84f16ed7edc3d007aae6b7577c7cec024a5 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 5 Sep 2024 10:20:35 +0300 Subject: [PATCH 21/34] add grpc runtime test --- dev/envs/compose.grpc.yml | 13 +++++++++ dev/grpc/proto/geography.proto | 22 +++++++++++++++ dev/grpc/proto/helloworld.proto | 15 ++++++++++ dev/grpc/proto/maths.proto | 24 ++++++++++++++++ dev/grpc/stub/geography.json | 28 +++++++++++++++++++ dev/grpc/stub/helloworld.json | 14 ++++++++++ dev/grpc/stub/maths.is_prime.json | 14 ++++++++++ dev/grpc/stub/maths.sum.json | 14 ++++++++++ examples/typegraphs/execute.ts | 14 +++++----- typegate/src/runtimes/grpc.ts | 7 ++--- typegate/tests/runtimes/grpc/grpc.py | 20 +++++++++++++ typegate/tests/runtimes/grpc/grpc_test.ts | 22 +++++++++++++++ .../tests/runtimes/grpc/proto/geography.proto | 22 +++++++++++++++ .../runtimes/grpc/proto/helloworld.proto | 15 ++++++++++ .../tests/runtimes/grpc/proto/maths.proto | 24 ++++++++++++++++ .../core/src/runtimes/grpc/type_generation.rs | 16 +++++++---- 16 files changed, 266 insertions(+), 18 deletions(-) create mode 100644 dev/envs/compose.grpc.yml create mode 100644 dev/grpc/proto/geography.proto create mode 100644 dev/grpc/proto/helloworld.proto create mode 100644 dev/grpc/proto/maths.proto create mode 100644 dev/grpc/stub/geography.json create mode 100644 dev/grpc/stub/helloworld.json create mode 100644 dev/grpc/stub/maths.is_prime.json create mode 100644 dev/grpc/stub/maths.sum.json create mode 100644 typegate/tests/runtimes/grpc/grpc.py create mode 100644 typegate/tests/runtimes/grpc/grpc_test.ts create mode 100644 typegate/tests/runtimes/grpc/proto/geography.proto create mode 100644 typegate/tests/runtimes/grpc/proto/helloworld.proto create mode 100644 typegate/tests/runtimes/grpc/proto/maths.proto diff --git a/dev/envs/compose.grpc.yml b/dev/envs/compose.grpc.yml new file mode 100644 index 0000000000..2aa3737778 --- /dev/null +++ b/dev/envs/compose.grpc.yml @@ -0,0 +1,13 @@ +services: + grpc-runtime-test: + image: tkpd/gripmock + container_name: grpc-runtime-test + ports: + - "4770:4770" + - "4771:4771" + volumes: + - ./proto:/proto + - ./stub:/stub + command: > + --stub=/stub \ + ../grpc/proto/helloworld.proto diff --git a/dev/grpc/proto/geography.proto b/dev/grpc/proto/geography.proto new file mode 100644 index 0000000000..84bc259e67 --- /dev/null +++ b/dev/grpc/proto/geography.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package geography; + +service Demography { + rpc Country(CountryRequest) returns (CountryReply) {} +} + +message CountryRequest { string name = 1; } + +message Currency { + string code = 1; + string name = 2; + string symbol = 3; +} + +message CountryReply { + string name = 1; + string capital = 2; + int32 population = 3; + repeated Currency currencies = 4; +} diff --git a/dev/grpc/proto/helloworld.proto b/dev/grpc/proto/helloworld.proto new file mode 100644 index 0000000000..1d90ea750d --- /dev/null +++ b/dev/grpc/proto/helloworld.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package helloworld; + +service Greeter { + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +message HelloRequest { + string name = 1; +} + +message HelloReply { + string message = 1; +} diff --git a/dev/grpc/proto/maths.proto b/dev/grpc/proto/maths.proto new file mode 100644 index 0000000000..d02a58a687 --- /dev/null +++ b/dev/grpc/proto/maths.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package maths; + +service Calculator { + rpc Sum (SumRequest) returns (SumReply) {} + rpc IsPrime (IsPrimeRequest) returns (IsPrimeReply) {} +} + +message SumRequest { + repeated int32 list = 1; +} + +message SumReply { + int32 total = 1; +} + +message IsPrimeRequest { + int32 number = 1; +} + +message IsPrimeReply { + bool isPrime = 1; +} diff --git a/dev/grpc/stub/geography.json b/dev/grpc/stub/geography.json new file mode 100644 index 0000000000..5e3ee62173 --- /dev/null +++ b/dev/grpc/stub/geography.json @@ -0,0 +1,28 @@ +{ + "service": "Demography", + "method": "Country", + "input": { + "equals": { + "name": "France" + } + }, + "output": { + "data": { + "name": "France", + "capital": "Paris", + "population": 68035000, + "currencies": [ + { + "code": "EUR", + "name": "Euro", + "symbol": "€" + }, + { + "code": "XPF", + "name": "CFP franc", + "symbol": "F" + } + ] + } + } +} diff --git a/dev/grpc/stub/helloworld.json b/dev/grpc/stub/helloworld.json new file mode 100644 index 0000000000..8e02845dcd --- /dev/null +++ b/dev/grpc/stub/helloworld.json @@ -0,0 +1,14 @@ +{ + "service": "Greeter", + "method": "SayHello", + "input": { + "equals": { + "name": "Metatype" + } + }, + "output": { + "data": { + "message": "Hello Metatype" + } + } +} diff --git a/dev/grpc/stub/maths.is_prime.json b/dev/grpc/stub/maths.is_prime.json new file mode 100644 index 0000000000..5b41d0f8c3 --- /dev/null +++ b/dev/grpc/stub/maths.is_prime.json @@ -0,0 +1,14 @@ +{ + "service": "Calculator", + "method": "IsPrime", + "input": { + "equals": { + "number": 17 + } + }, + "output": { + "data": { + "isPrime": true + } + } +} diff --git a/dev/grpc/stub/maths.sum.json b/dev/grpc/stub/maths.sum.json new file mode 100644 index 0000000000..cbfa828b07 --- /dev/null +++ b/dev/grpc/stub/maths.sum.json @@ -0,0 +1,14 @@ +{ + "service": "Calculator", + "method": "Sum", + "input": { + "equals": { + "list": [1, 2, 3, 4, 5] + } + }, + "output": { + "data": { + "total": 15 + } + } +} diff --git a/examples/typegraphs/execute.ts b/examples/typegraphs/execute.ts index 348381c501..37b850f604 100644 --- a/examples/typegraphs/execute.ts +++ b/examples/typegraphs/execute.ts @@ -22,7 +22,7 @@ await typegraph( name: t.string(), ideas: t.list(g.ref("idea")), }, - { name: "bucket" } + { name: "bucket" }, ); const idea = t.struct( @@ -33,7 +33,7 @@ await typegraph( votes: t.list(g.ref("vote")), bucket: g.ref("bucket"), }, - { name: "idea" } + { name: "idea" }, ); const vote = t.struct( @@ -44,7 +44,7 @@ await typegraph( desc: t.string().optional(), idea: g.ref("idea"), }, - { name: "vote" } + { name: "vote" }, ); g.auth(Auth.basic(["andim"])); @@ -53,7 +53,7 @@ await typegraph( "admins", ` (_args, { context }) => !!context.username - ` + `, ); g.expose( @@ -77,7 +77,7 @@ await typegraph( vote_id: t.uuid(), importance: t.enum_(["medium", "important", "critical"]), }), - effects.update() + effects.update(true), ), get_context: deno .identity(t.struct({ username: t.string().optional() })) @@ -85,7 +85,7 @@ await typegraph( username: g.fromContext("username"), }), }, - pub + pub, ); - } + }, ); diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index 84bf3da346..31ce8c6d2d 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -48,9 +48,7 @@ export class GrpcRuntime extends Runtime { } async deinit(): Promise<void> { - nativeVoid( - await native.grpc_unregister({ client_id: this.id }), - ); + nativeVoid(await native.grpc_unregister({ client_id: this.id })); } materialize( @@ -61,11 +59,10 @@ export class GrpcRuntime extends Runtime { const { method } = stage.props.materializer?.data ?? {}; const resolver: Resolver = async (args) => { - const { payload } = args; return nativeResult( await native.call_grpc_method({ method: String(method), - payload, + payload: JSON.stringify(args), client_id: this.id, }), ); diff --git a/typegate/tests/runtimes/grpc/grpc.py b/typegate/tests/runtimes/grpc/grpc.py new file mode 100644 index 0000000000..28ec937473 --- /dev/null +++ b/typegate/tests/runtimes/grpc/grpc.py @@ -0,0 +1,20 @@ +# Copyright Metatype OÜ, licensed under the Elastic License 2.0. +# SPDX-License-Identifier: Elastic-2.0 + +from pathlib import Path + +from typegraph import Graph, Policy, typegraph +from typegraph.runtimes.grpc import GrpcRuntime + + +BASE_DIR = Path(__file__).parent + + +@typegraph() +def grpc(g: Graph): + proto_file = BASE_DIR.joinpath("proto/helloworld.proto") + grpc = GrpcRuntime(proto_file, "localhost:4770") + + g.expose( + Policy.public(), greet=grpc.call_grpc_method("/helloworld.Greeter/SayHello") + ) diff --git a/typegate/tests/runtimes/grpc/grpc_test.ts b/typegate/tests/runtimes/grpc/grpc_test.ts new file mode 100644 index 0000000000..5d2d81c79f --- /dev/null +++ b/typegate/tests/runtimes/grpc/grpc_test.ts @@ -0,0 +1,22 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { gql, Meta } from "test-utils/mod.ts"; + +Meta.test({ name: "Grpc Runtime" }, async (t) => { + const e = await t.engine("runtimes/grpc/grpc.py"); + + await t.should("", async () => { + await gql` + query { + greet(name: "Metatype") { + message + } + } + ` + .expectData({ + message: "Hello Metatype", + }) + .on(e); + }); +}); diff --git a/typegate/tests/runtimes/grpc/proto/geography.proto b/typegate/tests/runtimes/grpc/proto/geography.proto new file mode 100644 index 0000000000..84bc259e67 --- /dev/null +++ b/typegate/tests/runtimes/grpc/proto/geography.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package geography; + +service Demography { + rpc Country(CountryRequest) returns (CountryReply) {} +} + +message CountryRequest { string name = 1; } + +message Currency { + string code = 1; + string name = 2; + string symbol = 3; +} + +message CountryReply { + string name = 1; + string capital = 2; + int32 population = 3; + repeated Currency currencies = 4; +} diff --git a/typegate/tests/runtimes/grpc/proto/helloworld.proto b/typegate/tests/runtimes/grpc/proto/helloworld.proto new file mode 100644 index 0000000000..1d90ea750d --- /dev/null +++ b/typegate/tests/runtimes/grpc/proto/helloworld.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package helloworld; + +service Greeter { + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +message HelloRequest { + string name = 1; +} + +message HelloReply { + string message = 1; +} diff --git a/typegate/tests/runtimes/grpc/proto/maths.proto b/typegate/tests/runtimes/grpc/proto/maths.proto new file mode 100644 index 0000000000..d02a58a687 --- /dev/null +++ b/typegate/tests/runtimes/grpc/proto/maths.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package maths; + +service Calculator { + rpc Sum (SumRequest) returns (SumReply) {} + rpc IsPrime (IsPrimeRequest) returns (IsPrimeReply) {} +} + +message SumRequest { + repeated int32 list = 1; +} + +message SumReply { + int32 total = 1; +} + +message IsPrimeRequest { + int32 number = 1; +} + +message IsPrimeReply { + bool isPrime = 1; +} diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 097e188857..0f76f3c447 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -1,7 +1,7 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -use anyhow::Result; +use anyhow::{bail, Result}; use common::grpc::{ get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, get_relative_method_name, Fields, @@ -19,11 +19,9 @@ pub struct Type { pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<Type> { let file_descriptor = get_file_descriptor(proto_file_content)?; - let method_descriptor = get_method_descriptor_proto(file_descriptor.clone(), method_name)?; let input_message = get_relative_method_name(&method_descriptor.input_type.unwrap())?; - let out_message = get_relative_method_name(&method_descriptor.output_type.unwrap())?; let input_fields = get_message_field_descriptor(&file_descriptor, &input_message)?; @@ -38,15 +36,21 @@ pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<Type fn convert_proto_fields_to_type_id(fields: Fields) -> Result<TypeId> { let mut r#type = t::struct_(); for field in fields { - let the_type = match field.proto().type_name() { + let field_name = field.name(); + let type_name = field.proto().type_name(); + let type_id = match type_name { "string" => t::string().build()?, "int32" => t::integer().build()?, "int64" => t::integer().build()?, "bool" => t::boolean().build()?, "float" => t::float().build()?, - tt => panic!("{tt} is not type"), + _ => bail!( + "Unsupported field type '{}' for field '{}'", + type_name, + field_name + ), }; - r#type.prop(field.name(), the_type); + r#type.prop(field_name, type_id); } Ok(r#type.build()?) From 2ab04b7c2ebaa1219ce24161faff35cc0eb2e17d Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Sun, 8 Sep 2024 13:29:33 +0300 Subject: [PATCH 22/34] add crate proto_parse --- Cargo.lock | 13 ++ libs/common/Cargo.toml | 2 + libs/common/src/{grpc/mod.rs => grpc.rs} | 20 +-- libs/common/src/grpc/proto_parser.rs | 167 ------------------ typegate/engine/runtime.d.ts | 22 +-- typegate/engine/src/runtimes/grpc.rs | 9 +- typegate/src/runtimes/grpc.ts | 16 +- typegate/src/runtimes/mod.ts | 5 +- typegate/tests/runtimes/grpc/grpc.py | 6 +- typegraph/core/src/runtimes/grpc/mod.rs | 2 +- .../core/src/runtimes/grpc/type_generation.rs | 30 ++-- 11 files changed, 59 insertions(+), 233 deletions(-) rename libs/common/src/{grpc/mod.rs => grpc.rs} (76%) delete mode 100644 libs/common/src/grpc/proto_parser.rs diff --git a/Cargo.lock b/Cargo.lock index 2b3e216b81..c48e08def5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1583,6 +1583,7 @@ dependencies = [ "indexmap 2.2.6", "indoc", "itertools 0.11.0", + "proto-parser", "protobuf", "reqwest 0.11.27", "serde 1.0.204", @@ -8546,6 +8547,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "proto-parser" +version = "0.1.0" +source = "git+https://github.com/metatypedev/proto-parser?branch=main#aa47da5a8eb16f07d28f7aeece77953401cce3a2" +dependencies = [ + "anyhow", + "indexmap 2.2.6", + "protobuf", + "protobuf-support", + "thiserror", +] + [[package]] name = "protobuf" version = "3.5.1" diff --git a/libs/common/Cargo.toml b/libs/common/Cargo.toml index 81cbf19504..fa0eaf7f75 100644 --- a/libs/common/Cargo.toml +++ b/libs/common/Cargo.toml @@ -20,4 +20,6 @@ itertools = "0.11.0" colored = "2.0.4" indoc.workspace = true thiserror.workspace = true + protobuf = "3.5.1" +proto-parser = { git = "https://github.com/metatypedev/proto-parser", branch = "main" } diff --git a/libs/common/src/grpc/mod.rs b/libs/common/src/grpc.rs similarity index 76% rename from libs/common/src/grpc/mod.rs rename to libs/common/src/grpc.rs index c7db694bb9..728c9c68bf 100644 --- a/libs/common/src/grpc/mod.rs +++ b/libs/common/src/grpc.rs @@ -1,10 +1,6 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -mod proto_parser; - -pub use proto_parser::get_file_descriptor; - use anyhow::{Context, Result}; use protobuf::{ @@ -12,6 +8,15 @@ use protobuf::{ reflect::{FieldDescriptor, FileDescriptor}, }; +pub use protobuf::descriptor::field_descriptor_proto::Type; + +pub fn get_file_descriptor(content: &str) -> Result<FileDescriptor> { + let parsed = proto_parser::model::FileDescriptor::parse(content)?; + let file_descriptor_proto = proto_parser::convert::file_descriptor(&parsed)?; + let file_descriptor = FileDescriptor::new_dynamic(file_descriptor_proto, &[])?; + Ok(file_descriptor) +} + pub fn get_method_descriptor_proto( file_descriptor: FileDescriptor, method_name: &str, @@ -27,13 +32,6 @@ pub fn get_method_descriptor_proto( Ok(method.clone()) } -pub fn get_relative_method_name(absolute_method_name: &str) -> anyhow::Result<String> { - let path: Vec<&str> = absolute_method_name.split('/').collect(); - let method = path.get(2).context("Invalid path")?; - - Ok(method.to_string()) -} - pub fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result<String> { let path: Vec<&str> = absolute_message_name.split('.').collect(); let message = path.get(2).context("Invalid path")?; diff --git a/libs/common/src/grpc/proto_parser.rs b/libs/common/src/grpc/proto_parser.rs deleted file mode 100644 index 824a90f393..0000000000 --- a/libs/common/src/grpc/proto_parser.rs +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. -// SPDX-License-Identifier: MPL-2.0 - -use anyhow::{anyhow, Result}; -use protobuf::{ - descriptor::{ - field_descriptor_proto::Type, DescriptorProto, EnumDescriptorProto, - EnumValueDescriptorProto, FieldDescriptorProto, FileDescriptorProto, - }, - reflect::FileDescriptor, -}; - -pub struct ProtoParser; - -impl ProtoParser { - pub fn parse(content: &str) -> Result<FileDescriptorProto> { - let mut file_descriptor = FileDescriptorProto::new(); - let mut current_message: Option<DescriptorProto> = None; - let mut current_enum: Option<EnumDescriptorProto> = None; - - for line in content.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with("//") { - continue; - } - - if line.starts_with("syntax") { - Self::parse_syntax(&mut file_descriptor, line)?; - } else if line.starts_with("package") { - Self::parse_package(&mut file_descriptor, line)?; - } else if line.starts_with("message") { - if let Some(message) = current_message.take() { - file_descriptor.message_type.push(message); - } - current_message = Some(Self::parse_message_start(line)?); - } else if line.starts_with("enum") { - if let Some(enum_type) = current_enum.take() { - file_descriptor.enum_type.push(enum_type); - } - current_enum = Some(Self::parse_enum_start(line)?); - } else if line.ends_with('}') { - if let Some(message) = current_message.take() { - file_descriptor.message_type.push(message); - } - if let Some(enum_type) = current_enum.take() { - file_descriptor.enum_type.push(enum_type); - } - } else if let Some(ref mut message) = current_message { - Self::parse_message_field(message, line)?; - } else if let Some(ref mut enum_type) = current_enum { - Self::parse_enum_value(enum_type, line)?; - } - } - - if let Some(message) = current_message.take() { - file_descriptor.message_type.push(message); - } - if let Some(enum_type) = current_enum.take() { - file_descriptor.enum_type.push(enum_type); - } - - Ok(file_descriptor) - } - - fn parse_syntax(file_descriptor: &mut FileDescriptorProto, line: &str) -> Result<()> { - let syntax = line - .split('=') - .nth(1) - .ok_or_else(|| anyhow!("Invalid syntax line"))? - .trim() - .trim_matches('"'); - file_descriptor.set_syntax(syntax.to_string()); - Ok(()) - } - - fn parse_package(file_descriptor: &mut FileDescriptorProto, line: &str) -> Result<()> { - let package = line - .split_whitespace() - .nth(1) - .ok_or_else(|| anyhow!("Invalid package line"))? - .trim_matches(';'); - file_descriptor.set_package(package.to_string()); - Ok(()) - } - - fn parse_message_start(line: &str) -> Result<DescriptorProto> { - let name = line - .split_whitespace() - .nth(1) - .ok_or_else(|| anyhow!("Invalid message line"))? - .trim_matches('{'); - let mut message = DescriptorProto::new(); - message.set_name(name.to_string()); - Ok(message) - } - - fn parse_message_field(message: &mut DescriptorProto, line: &str) -> Result<()> { - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() < 4 { - return Err(anyhow!("Invalid field line")); - } - - let mut field = FieldDescriptorProto::new(); - field.set_name(parts[1].to_string()); - field.set_number(parts[3].trim_matches(';').parse()?); - - let field_type = match parts[0] { - "double" => Type::TYPE_DOUBLE, - "float" => Type::TYPE_FLOAT, - "int64" => Type::TYPE_INT64, - "uint64" => Type::TYPE_UINT64, - "int32" => Type::TYPE_INT32, - "fixed64" => Type::TYPE_FIXED64, - "fixed32" => Type::TYPE_FIXED32, - "bool" => Type::TYPE_BOOL, - "string" => Type::TYPE_STRING, - "group" => Type::TYPE_GROUP, - "message" => Type::TYPE_MESSAGE, - "bytes" => Type::TYPE_BYTES, - "uint32" => Type::TYPE_UINT32, - "enum" => Type::TYPE_ENUM, - "sfixed32" => Type::TYPE_SFIXED32, - "sfixed64" => Type::TYPE_SFIXED64, - "sint32" => Type::TYPE_SINT32, - "sint64" => Type::TYPE_SINT64, - _ => return Err(anyhow!("Unsupported field type: {}", parts[0])), - }; - field.set_type(field_type); - - message.field.push(field); - Ok(()) - } - - fn parse_enum_start(line: &str) -> Result<EnumDescriptorProto> { - let name = line - .split_whitespace() - .nth(1) - .ok_or_else(|| anyhow!("Invalid enum line"))? - .trim_matches('{'); - let mut enum_type = EnumDescriptorProto::new(); - enum_type.set_name(name.to_string()); - Ok(enum_type) - } - - fn parse_enum_value(enum_type: &mut EnumDescriptorProto, line: &str) -> Result<()> { - let parts: Vec<&str> = line.split('=').collect(); - if parts.len() != 2 { - return Err(anyhow!("Invalid enum value line")); - } - - let name = parts[0].trim().to_string(); - let number: i32 = parts[1].trim().trim_matches(';').parse()?; - - let mut enum_value = EnumValueDescriptorProto::new(); - enum_value.set_name(name); - enum_value.set_number(number); - - enum_type.value.push(enum_value); - Ok(()) - } -} - -pub fn get_file_descriptor(content: &str) -> Result<FileDescriptor> { - let file_descriptor_proto = ProtoParser::parse(content)?; - let file_descriptor = FileDescriptor::new_dynamic(file_descriptor_proto, &[])?; - Ok(file_descriptor) -} diff --git a/typegate/engine/runtime.d.ts b/typegate/engine/runtime.d.ts index fadc6d9ab4..3c4218e6ac 100644 --- a/typegate/engine/runtime.d.ts +++ b/typegate/engine/runtime.d.ts @@ -13,9 +13,7 @@ type MetaNS = { wasmtimeWit: (inp: WasmInput) => string; prisma: { - registerEngine: ( - inp: PrismaRegisterEngineInp, - ) => Promise<void>; + registerEngine: (inp: PrismaRegisterEngineInp) => Promise<void>; unregisterEngine: (engine_name: string) => Promise<void>; query: (inp: PrismaQueryInp) => Promise<string>; diff: ( @@ -32,15 +30,9 @@ type MetaNS = { temporal: { clientRegister: (inp: TemporalRegisterInput) => Promise<void>; clientUnregister: (client_id: string) => void; - workflowStart: ( - inp: TemporalWorkflowStartInput, - ) => Promise<string>; - workflowSignal: ( - inp: TemporalWorkflowSignalInput, - ) => Promise<void>; - workflowQuery: ( - inp: TemporalWorkflowQueryInput, - ) => Promise<Array<string>>; + workflowStart: (inp: TemporalWorkflowStartInput) => Promise<string>; + workflowSignal: (inp: TemporalWorkflowSignalInput) => Promise<void>; + workflowQuery: (inp: TemporalWorkflowQueryInput) => Promise<Array<string>>; workflowDescribe: ( inp: TemporalWorkflowDescribeInput, ) => Promise<TemporalWorkflowDescribeOutput>; @@ -53,9 +45,7 @@ type MetaNS = { args: WitWireInitArgs, cb: (op_name: string, json: string) => Promise<string>, ) => Promise<WitWireInitResponse>; - destroy: ( - instanceId: string, - ) => Promise<void>; + destroy: (instanceId: string) => Promise<void>; handle: ( instanceId: string, args: WitWireReq, @@ -267,7 +257,7 @@ export type WitWireHandleResponse = }; export type GrpcRegisterInput = { - protoFileContent: string; + proto_file_content: string; endpoint: string; client_id: string; }; diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 82caa471e8..61115019bb 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -3,10 +3,7 @@ use std::{cell::RefCell, ops::Deref, rc::Rc, str::FromStr, sync::Arc}; -use common::grpc::{ - get_file_descriptor, get_method_descriptor_proto, get_relative_message_name, - get_relative_method_name, -}; +use common::grpc::{get_file_descriptor, get_method_descriptor_proto, get_relative_message_name}; use anyhow::{Context, Result}; use bytes::{Buf, BufMut}; @@ -199,10 +196,8 @@ pub async fn op_call_grpc_method( let file_descriptor = get_file_descriptor(&grpc_client.proto_file_content)?; - let method_name = get_relative_method_name(&input.method)?; - let method_descriptor_proto = - get_method_descriptor_proto(file_descriptor.clone(), &method_name)?; + get_method_descriptor_proto(file_descriptor.clone(), &input.method)?; let request_message = get_relative_message_name(method_descriptor_proto.input_type())?; diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index 31ce8c6d2d..0b950ad9b7 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -1,19 +1,19 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -import { Runtime } from "@typegate/runtimes/Runtime.ts"; +import { Runtime } from "./Runtime.ts"; import * as native from "native"; -import { ComputeStage } from "@typegate/engine/query_engine.ts"; -import { getLogger, Logger } from "@typegate/log.ts"; -import { TypeGraph } from "@typegate/typegraph/mod.ts"; +import { ComputeStage } from "../engine/query_engine.ts"; +import { getLogger, Logger } from "../log.ts"; +import { TypeGraph } from "../typegraph/mod.ts"; import { Resolver, RuntimeInitParams } from "../types.ts"; -import { nativeResult, nativeVoid } from "@typegate/utils.ts"; -import { registerRuntime } from "@typegate/runtimes/mod.ts"; +import { nativeResult, nativeVoid } from "../utils.ts"; +import { registerRuntime } from "./mod.ts"; const logger = getLogger(import.meta); interface GrpcRuntimeData { - protoFileContent: string; + proto_file_content: string; endpoint: string; } @@ -36,7 +36,7 @@ export class GrpcRuntime extends Runtime { nativeVoid( await native.grpc_register({ - protoFileContent: args.protoFileContent, + proto_file_content: args.proto_file_content, endpoint: args.endpoint, client_id: instance.id, }), diff --git a/typegate/src/runtimes/mod.ts b/typegate/src/runtimes/mod.ts index 1c59cf149e..f57be8b391 100644 --- a/typegate/src/runtimes/mod.ts +++ b/typegate/src/runtimes/mod.ts @@ -10,9 +10,7 @@ interface RegistrableRuntime { init(params: RuntimeInitParams): Promise<Runtime> | Runtime; } -export function registerRuntime< - T extends RegistrableRuntime, ->( +export function registerRuntime<T extends RegistrableRuntime>( name: string, ): (runtime: T) => void { return (runtime: T) => { @@ -49,5 +47,6 @@ export async function init_runtimes(): Promise<void> { import("./wasm_wire.ts"), import("./wasm_reflected.ts"), import("./kv.ts"), + import("./grpc.ts"), ]); } diff --git a/typegate/tests/runtimes/grpc/grpc.py b/typegate/tests/runtimes/grpc/grpc.py index 28ec937473..7776b96c14 100644 --- a/typegate/tests/runtimes/grpc/grpc.py +++ b/typegate/tests/runtimes/grpc/grpc.py @@ -13,8 +13,6 @@ @typegraph() def grpc(g: Graph): proto_file = BASE_DIR.joinpath("proto/helloworld.proto") - grpc = GrpcRuntime(proto_file, "localhost:4770") + grpc = GrpcRuntime(proto_file, "tcp://localhost:4770") - g.expose( - Policy.public(), greet=grpc.call_grpc_method("/helloworld.Greeter/SayHello") - ) + g.expose(Policy.public(), greet=grpc.call_grpc_method("SayHello")) diff --git a/typegraph/core/src/runtimes/grpc/mod.rs b/typegraph/core/src/runtimes/grpc/mod.rs index e633a04890..6e212ff6ab 100644 --- a/typegraph/core/src/runtimes/grpc/mod.rs +++ b/typegraph/core/src/runtimes/grpc/mod.rs @@ -58,7 +58,7 @@ pub fn call_grpc_method(runtime: RuntimeId, data: GrpcData) -> Result<FuncParams Store::register_materializer(super::Materializer::grpc(runtime, mat, WitEffect::Read)); let t = type_generation::generate_type(&grpc_runtime_data.proto_file_content, &data.method) - .map_err(|err| err.to_string())?; + .map_err(|err| format!("failed generate type {err}"))?; Ok(FuncParams { inp: t.input.0, diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 0f76f3c447..6882cc4141 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -3,8 +3,7 @@ use anyhow::{bail, Result}; use common::grpc::{ - get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, - get_relative_method_name, Fields, + get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, Fields, Type, }; use crate::{ @@ -12,22 +11,22 @@ use crate::{ types::TypeId, }; -pub struct Type { +pub struct TType { pub input: TypeId, pub output: TypeId, } -pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<Type> { +pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<TType> { let file_descriptor = get_file_descriptor(proto_file_content)?; let method_descriptor = get_method_descriptor_proto(file_descriptor.clone(), method_name)?; - let input_message = get_relative_method_name(&method_descriptor.input_type.unwrap())?; - let out_message = get_relative_method_name(&method_descriptor.output_type.unwrap())?; + let input_type = method_descriptor.input_type(); + let output_type = method_descriptor.output_type(); - let input_fields = get_message_field_descriptor(&file_descriptor, &input_message)?; - let output_fields = get_message_field_descriptor(&file_descriptor, &out_message)?; + let input_fields = get_message_field_descriptor(&file_descriptor, input_type)?; + let output_fields = get_message_field_descriptor(&file_descriptor, output_type)?; - Ok(Type { + Ok(TType { input: convert_proto_fields_to_type_id(input_fields)?, output: convert_proto_fields_to_type_id(output_fields)?, }) @@ -37,15 +36,14 @@ fn convert_proto_fields_to_type_id(fields: Fields) -> Result<TypeId> { let mut r#type = t::struct_(); for field in fields { let field_name = field.name(); - let type_name = field.proto().type_name(); + let type_name = field.proto().type_(); let type_id = match type_name { - "string" => t::string().build()?, - "int32" => t::integer().build()?, - "int64" => t::integer().build()?, - "bool" => t::boolean().build()?, - "float" => t::float().build()?, + Type::TYPE_STRING => t::string().build()?, + Type::TYPE_INT32 | Type::TYPE_INT64 => t::integer().build()?, + Type::TYPE_FLOAT => t::float().build()?, + Type::TYPE_BOOL => t::boolean().build()?, _ => bail!( - "Unsupported field type '{}' for field '{}'", + "Unsupported field type '{:?}' for field '{}'", type_name, field_name ), From 2b1ff4b7095f9eebda4e6993dc921c050a9e354c Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Mon, 9 Sep 2024 17:23:40 +0300 Subject: [PATCH 23/34] type_generated done add more test --- dev/envs/compose.grpc.yml | 16 ++++---- dev/{grpc => envs}/proto/geography.proto | 0 dev/{grpc => envs}/proto/helloworld.proto | 0 dev/{grpc => envs}/proto/maths.proto | 0 dev/{grpc => envs}/stub/geography.json | 0 dev/{grpc => envs}/stub/helloworld.json | 0 dev/{grpc => envs}/stub/maths.is_prime.json | 0 dev/{grpc => envs}/stub/maths.sum.json | 0 libs/common/src/grpc.rs | 22 +++++------ typegate/engine/src/lib.rs | 1 + typegate/engine/src/runtimes/grpc.rs | 24 +++++++----- typegate/src/runtimes/grpc.ts | 24 +++++++++--- typegate/tests/runtimes/grpc/grpc.py | 4 +- typegate/tests/runtimes/grpc/grpc_test.ts | 4 +- .../core/src/runtimes/grpc/type_generation.rs | 38 +++++++++++++++---- 15 files changed, 88 insertions(+), 45 deletions(-) rename dev/{grpc => envs}/proto/geography.proto (100%) rename dev/{grpc => envs}/proto/helloworld.proto (100%) rename dev/{grpc => envs}/proto/maths.proto (100%) rename dev/{grpc => envs}/stub/geography.json (100%) rename dev/{grpc => envs}/stub/helloworld.json (100%) rename dev/{grpc => envs}/stub/maths.is_prime.json (100%) rename dev/{grpc => envs}/stub/maths.sum.json (100%) diff --git a/dev/envs/compose.grpc.yml b/dev/envs/compose.grpc.yml index 2aa3737778..9a1b2d4456 100644 --- a/dev/envs/compose.grpc.yml +++ b/dev/envs/compose.grpc.yml @@ -1,13 +1,15 @@ services: - grpc-runtime-test: + grpc-mock-server: image: tkpd/gripmock - container_name: grpc-runtime-test + container_name: grpc-runtime ports: - "4770:4770" - "4771:4771" volumes: - - ./proto:/proto - - ./stub:/stub - command: > - --stub=/stub \ - ../grpc/proto/helloworld.proto + - ./dev/envs/proto:/proto + - ./dev/envs/stub:/stub + command: + - --stub=/stub + - /proto/helloworld.proto + - /proto/maths.proto + - /proto/geography.proto diff --git a/dev/grpc/proto/geography.proto b/dev/envs/proto/geography.proto similarity index 100% rename from dev/grpc/proto/geography.proto rename to dev/envs/proto/geography.proto diff --git a/dev/grpc/proto/helloworld.proto b/dev/envs/proto/helloworld.proto similarity index 100% rename from dev/grpc/proto/helloworld.proto rename to dev/envs/proto/helloworld.proto diff --git a/dev/grpc/proto/maths.proto b/dev/envs/proto/maths.proto similarity index 100% rename from dev/grpc/proto/maths.proto rename to dev/envs/proto/maths.proto diff --git a/dev/grpc/stub/geography.json b/dev/envs/stub/geography.json similarity index 100% rename from dev/grpc/stub/geography.json rename to dev/envs/stub/geography.json diff --git a/dev/grpc/stub/helloworld.json b/dev/envs/stub/helloworld.json similarity index 100% rename from dev/grpc/stub/helloworld.json rename to dev/envs/stub/helloworld.json diff --git a/dev/grpc/stub/maths.is_prime.json b/dev/envs/stub/maths.is_prime.json similarity index 100% rename from dev/grpc/stub/maths.is_prime.json rename to dev/envs/stub/maths.is_prime.json diff --git a/dev/grpc/stub/maths.sum.json b/dev/envs/stub/maths.sum.json similarity index 100% rename from dev/grpc/stub/maths.sum.json rename to dev/envs/stub/maths.sum.json diff --git a/libs/common/src/grpc.rs b/libs/common/src/grpc.rs index 728c9c68bf..ec76f685c5 100644 --- a/libs/common/src/grpc.rs +++ b/libs/common/src/grpc.rs @@ -3,12 +3,10 @@ use anyhow::{Context, Result}; -use protobuf::{ - descriptor::MethodDescriptorProto, - reflect::{FieldDescriptor, FileDescriptor}, -}; +use protobuf::{descriptor::MethodDescriptorProto, reflect::FieldDescriptor}; pub use protobuf::descriptor::field_descriptor_proto::Type; +pub use protobuf::reflect::FileDescriptor; pub fn get_file_descriptor(content: &str) -> Result<FileDescriptor> { let parsed = proto_parser::model::FileDescriptor::parse(content)?; @@ -19,8 +17,9 @@ pub fn get_file_descriptor(content: &str) -> Result<FileDescriptor> { pub fn get_method_descriptor_proto( file_descriptor: FileDescriptor, - method_name: &str, + relative_method_name: &str, ) -> Result<MethodDescriptorProto> { + let method_name = get_relative_method_name(relative_method_name)?; let method = file_descriptor .proto() .service @@ -28,15 +27,15 @@ pub fn get_method_descriptor_proto( .flat_map(|service| &service.method) .find(|method| method.name.as_ref().is_some_and(|name| name == method_name)) .context("method descriptor not found")?; - Ok(method.clone()) } -pub fn get_relative_message_name(absolute_message_name: &str) -> anyhow::Result<String> { - let path: Vec<&str> = absolute_message_name.split('.').collect(); - let message = path.get(2).context("Invalid path")?; - - Ok(message.to_string()) +fn get_relative_method_name(method_name: &str) -> Result<&str> { + let path = method_name.split('/').collect::<Vec<&str>>(); + let method_name = path + .last() + .context("Failed to get name from absolute path")?; + Ok(method_name.to_owned()) } pub type Fields = Vec<FieldDescriptor>; @@ -48,6 +47,5 @@ pub fn get_message_field_descriptor( let message_descriptor = file_descriptor .message_by_full_name(type_name) .context(format!("Message not found: {}", type_name))?; - Ok(message_descriptor.fields().collect()) } diff --git a/typegate/engine/src/lib.rs b/typegate/engine/src/lib.rs index d4615f2556..d056987fdc 100644 --- a/typegate/engine/src/lib.rs +++ b/typegate/engine/src/lib.rs @@ -91,6 +91,7 @@ impl OpDepInjector { .expect("error initializing wit_wire state"), ); state.put(runtimes::prisma::Ctx::new(tmp_dir)); + state.put(runtimes::grpc::Ctx::default()); } } diff --git a/typegate/engine/src/runtimes/grpc.rs b/typegate/engine/src/runtimes/grpc.rs index 61115019bb..0b87a39c55 100644 --- a/typegate/engine/src/runtimes/grpc.rs +++ b/typegate/engine/src/runtimes/grpc.rs @@ -3,7 +3,7 @@ use std::{cell::RefCell, ops::Deref, rc::Rc, str::FromStr, sync::Arc}; -use common::grpc::{get_file_descriptor, get_method_descriptor_proto, get_relative_message_name}; +use common::grpc::{get_file_descriptor, get_method_descriptor_proto}; use anyhow::{Context, Result}; use bytes::{Buf, BufMut}; @@ -72,8 +72,7 @@ impl Decoder for DynCodec { let buf = src.chunk(); let length = buf.len(); - let response_message = - get_relative_message_name(self.method_descriptor_proto.output_type()).unwrap(); + let response_message = self.method_descriptor_proto.output_type().to_string(); let response = buf2response(buf, response_message, self.file_descriptor.clone()) .map(Some) @@ -100,10 +99,11 @@ pub fn json2request( file_descriptor: FileDescriptor, ) -> anyhow::Result<Request<DynRequest>> { let msg_descriptor = file_descriptor - .message_by_package_relative_name(&input_message) + .message_by_full_name(&input_message) .with_context(|| format!("Input message {input_message} not found"))?; let mut msg = msg_descriptor.new_instance(); - protobuf_json_mapping::merge_from_str(&mut *msg, &json)?; + protobuf_json_mapping::merge_from_str(&mut *msg, &json) + .context("failed to merge json to str")?; Ok(msg.into_request()) } @@ -114,7 +114,7 @@ fn buf2response( file_descriptor: FileDescriptor, ) -> anyhow::Result<DynResponse> { let msg_descriptor = file_descriptor - .message_by_package_relative_name(&output_message) + .message_by_full_name(&output_message) .with_context(|| format!("Output message {output_message} not found"))?; let mut msg = msg_descriptor.new_instance(); @@ -123,12 +123,13 @@ fn buf2response( Ok(msg) } +#[derive(Clone)] struct GrpcClient { client: Grpc<Channel>, proto_file_content: String, } -#[derive(Default)] +#[derive(Default, Clone)] pub struct Ctx { grpc_clients: Arc<DashMap<String, GrpcClient>>, } @@ -148,13 +149,16 @@ pub async fn op_grpc_register( ) -> Result<()> { let client = create_client(&input.endpoint).await?; - let state = state.borrow(); - let ctx = state.borrow::<Ctx>(); + let ctx = { + let state = state.borrow(); + state.borrow::<Ctx>().clone() + }; let grpc_client = GrpcClient { client, proto_file_content: input.proto_file_content, }; + ctx.grpc_clients .insert(input.client_id.clone(), grpc_client); @@ -199,7 +203,7 @@ pub async fn op_call_grpc_method( let method_descriptor_proto = get_method_descriptor_proto(file_descriptor.clone(), &input.method)?; - let request_message = get_relative_message_name(method_descriptor_proto.input_type())?; + let request_message = method_descriptor_proto.input_type().to_string(); let req = json2request(input.payload, request_message, file_descriptor.clone())?; diff --git a/typegate/src/runtimes/grpc.ts b/typegate/src/runtimes/grpc.ts index 0b950ad9b7..be89a0822e 100644 --- a/typegate/src/runtimes/grpc.ts +++ b/typegate/src/runtimes/grpc.ts @@ -56,15 +56,27 @@ export class GrpcRuntime extends Runtime { _waitlist: ComputeStage[], _verbose: boolean, ): ComputeStage[] | Promise<ComputeStage[]> { + if (stage.props.materializer == null) { + return [ + stage.withResolver(({ _: { parent } }) => { + const resolver = parent[stage.props.node]; + return typeof resolver === "function" ? resolver() : resolver; + }), + ]; + } + const { method } = stage.props.materializer?.data ?? {}; const resolver: Resolver = async (args) => { - return nativeResult( - await native.call_grpc_method({ - method: String(method), - payload: JSON.stringify(args), - client_id: this.id, - }), + const { _, ...payload } = args; + return JSON.parse( + nativeResult( + await native.call_grpc_method({ + method: String(method), + payload: JSON.stringify(payload), + client_id: this.id, + }), + ), ); }; diff --git a/typegate/tests/runtimes/grpc/grpc.py b/typegate/tests/runtimes/grpc/grpc.py index 7776b96c14..ef41740dcc 100644 --- a/typegate/tests/runtimes/grpc/grpc.py +++ b/typegate/tests/runtimes/grpc/grpc.py @@ -15,4 +15,6 @@ def grpc(g: Graph): proto_file = BASE_DIR.joinpath("proto/helloworld.proto") grpc = GrpcRuntime(proto_file, "tcp://localhost:4770") - g.expose(Policy.public(), greet=grpc.call_grpc_method("SayHello")) + g.expose( + Policy.public(), greet=grpc.call_grpc_method("/helloworld.Greeter/SayHello") + ) diff --git a/typegate/tests/runtimes/grpc/grpc_test.ts b/typegate/tests/runtimes/grpc/grpc_test.ts index 5d2d81c79f..d04794c1d3 100644 --- a/typegate/tests/runtimes/grpc/grpc_test.ts +++ b/typegate/tests/runtimes/grpc/grpc_test.ts @@ -15,7 +15,9 @@ Meta.test({ name: "Grpc Runtime" }, async (t) => { } ` .expectData({ - message: "Hello Metatype", + greet: { + message: "Hello Metatype", + }, }) .on(e); }); diff --git a/typegraph/core/src/runtimes/grpc/type_generation.rs b/typegraph/core/src/runtimes/grpc/type_generation.rs index 6882cc4141..7aa2ff8cdd 100644 --- a/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -3,7 +3,8 @@ use anyhow::{bail, Result}; use common::grpc::{ - get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, Fields, Type, + get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, Fields, + FileDescriptor, Type, }; use crate::{ @@ -11,12 +12,12 @@ use crate::{ types::TypeId, }; -pub struct TType { +pub struct GeneratedType { pub input: TypeId, pub output: TypeId, } -pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<TType> { +pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<GeneratedType> { let file_descriptor = get_file_descriptor(proto_file_content)?; let method_descriptor = get_method_descriptor_proto(file_descriptor.clone(), method_name)?; @@ -26,28 +27,49 @@ pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<TTyp let input_fields = get_message_field_descriptor(&file_descriptor, input_type)?; let output_fields = get_message_field_descriptor(&file_descriptor, output_type)?; - Ok(TType { - input: convert_proto_fields_to_type_id(input_fields)?, - output: convert_proto_fields_to_type_id(output_fields)?, + Ok(GeneratedType { + input: convert_proto_fields_to_type_id(&file_descriptor, input_fields)?, + output: convert_proto_fields_to_type_id(&file_descriptor, output_fields)?, }) } -fn convert_proto_fields_to_type_id(fields: Fields) -> Result<TypeId> { +fn convert_proto_fields_to_type_id( + file_descriptor: &FileDescriptor, + fields: Fields, +) -> Result<TypeId> { let mut r#type = t::struct_(); + for field in fields { let field_name = field.name(); let type_name = field.proto().type_(); - let type_id = match type_name { + + let mut type_id = match type_name { Type::TYPE_STRING => t::string().build()?, Type::TYPE_INT32 | Type::TYPE_INT64 => t::integer().build()?, Type::TYPE_FLOAT => t::float().build()?, Type::TYPE_BOOL => t::boolean().build()?, + Type::TYPE_MESSAGE => { + let nested_message_type = field.proto().name(); + convert_proto_fields_to_type_id( + file_descriptor, + get_message_field_descriptor(file_descriptor, nested_message_type)?, + )? + } _ => bail!( "Unsupported field type '{:?}' for field '{}'", type_name, field_name ), }; + + if field.is_repeated() { + type_id = t::list(type_id).build()?; + } + + if !field.is_required() { + type_id = t::optional(type_id).build()?; + } + r#type.prop(field_name, type_id); } From 615d55720524da3d031a3bfd8e29cb66c5c38638 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Mon, 9 Sep 2024 18:44:37 +0300 Subject: [PATCH 24/34] add some new tests but maths have some issue --- typegate/tests/runtimes/grpc/grpc_test.ts | 24 ++++++++++++++++--- .../runtimes/grpc/{grpc.py => helloworld.py} | 11 +++++---- typegate/tests/runtimes/grpc/maths.py | 24 +++++++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) rename typegate/tests/runtimes/grpc/{grpc.py => helloworld.py} (50%) create mode 100644 typegate/tests/runtimes/grpc/maths.py diff --git a/typegate/tests/runtimes/grpc/grpc_test.ts b/typegate/tests/runtimes/grpc/grpc_test.ts index d04794c1d3..34c6a3500a 100644 --- a/typegate/tests/runtimes/grpc/grpc_test.ts +++ b/typegate/tests/runtimes/grpc/grpc_test.ts @@ -4,9 +4,9 @@ import { gql, Meta } from "test-utils/mod.ts"; Meta.test({ name: "Grpc Runtime" }, async (t) => { - const e = await t.engine("runtimes/grpc/grpc.py"); + const hello_world = await t.engine("runtimes/grpc/helloworld.py"); - await t.should("", async () => { + await t.should("Say Hello", async () => { await gql` query { greet(name: "Metatype") { @@ -19,6 +19,24 @@ Meta.test({ name: "Grpc Runtime" }, async (t) => { message: "Hello Metatype", }, }) - .on(e); + .on(hello_world); + }); + + const maths = await t.engine("runtimes/grpc/maths.py"); + + await t.should("Sum number", async () => { + await gql` + query { + sum(list: [1, 2, 3, 4]) { + total + } + } + ` + .expectData({ + sum: { + total: 10, + }, + }) + .on(maths); }); }); diff --git a/typegate/tests/runtimes/grpc/grpc.py b/typegate/tests/runtimes/grpc/helloworld.py similarity index 50% rename from typegate/tests/runtimes/grpc/grpc.py rename to typegate/tests/runtimes/grpc/helloworld.py index ef41740dcc..afef73ad92 100644 --- a/typegate/tests/runtimes/grpc/grpc.py +++ b/typegate/tests/runtimes/grpc/helloworld.py @@ -11,10 +11,13 @@ @typegraph() -def grpc(g: Graph): - proto_file = BASE_DIR.joinpath("proto/helloworld.proto") - grpc = GrpcRuntime(proto_file, "tcp://localhost:4770") +def helloworld(g: Graph): + endpoint = "tcp://localhost:4770" + + helloworld = BASE_DIR.joinpath("proto/helloworld.proto") + helloworld_grpc = GrpcRuntime(helloworld, endpoint) g.expose( - Policy.public(), greet=grpc.call_grpc_method("/helloworld.Greeter/SayHello") + Policy.public(), + greet=helloworld_grpc.call_grpc_method("/helloworld.Greeter/SayHello"), ) diff --git a/typegate/tests/runtimes/grpc/maths.py b/typegate/tests/runtimes/grpc/maths.py new file mode 100644 index 0000000000..6f52fff509 --- /dev/null +++ b/typegate/tests/runtimes/grpc/maths.py @@ -0,0 +1,24 @@ +# Copyright Metatype OÜ, licensed under the Elastic License 2.0. +# SPDX-License-Identifier: Elastic-2.0 + +from pathlib import Path + +from typegraph import Graph, Policy, typegraph +from typegraph.runtimes.grpc import GrpcRuntime + + +BASE_DIR = Path(__file__).parent + + +@typegraph() +def maths(g: Graph): + endpoint = "tcp://localhost:4770" + + maths = BASE_DIR.joinpath("proto/maths.proto") + maths_grpc = GrpcRuntime(maths, endpoint) + + g.expose( + Policy.public(), + sum=maths_grpc.call_grpc_method("/maths.Calculator/Sum"), + isprime=maths_grpc.call_grpc_method("/maths.Calculator/IsPrime"), + ) From 11e364f6b60d2d3e7da5dc43e95f2b8a8982372f Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Mon, 9 Sep 2024 20:03:57 +0300 Subject: [PATCH 25/34] tests hello world, maths, is working --- typegate/tests/runtimes/grpc/geography.py | 23 +++++++++ typegate/tests/runtimes/grpc/grpc_test.ts | 61 ++++++++++++++++++++++- typegate/tests/runtimes/grpc/maths.py | 2 +- 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 typegate/tests/runtimes/grpc/geography.py diff --git a/typegate/tests/runtimes/grpc/geography.py b/typegate/tests/runtimes/grpc/geography.py new file mode 100644 index 0000000000..4e7ffb0ea2 --- /dev/null +++ b/typegate/tests/runtimes/grpc/geography.py @@ -0,0 +1,23 @@ +# Copyright Metatype OÜ, licensed under the Elastic License 2.0. +# SPDX-License-Identifier: Elastic-2.0 + +from pathlib import Path + +from typegraph import Graph, Policy, typegraph +from typegraph.runtimes.grpc import GrpcRuntime + + +BASE_DIR = Path(__file__).parent + + +@typegraph() +def geography(g: Graph): + endpoint = "tcp://localhost:4770" + + geography = BASE_DIR.joinpath("proto/geography.proto") + geography_grpc = GrpcRuntime(geography, endpoint) + + g.expose( + Policy.public(), + dem=geography_grpc.call_grpc_method("/geography.Demography/Country"), + ) diff --git a/typegate/tests/runtimes/grpc/grpc_test.ts b/typegate/tests/runtimes/grpc/grpc_test.ts index 34c6a3500a..b74aeb3ff3 100644 --- a/typegate/tests/runtimes/grpc/grpc_test.ts +++ b/typegate/tests/runtimes/grpc/grpc_test.ts @@ -27,16 +27,73 @@ Meta.test({ name: "Grpc Runtime" }, async (t) => { await t.should("Sum number", async () => { await gql` query { - sum(list: [1, 2, 3, 4]) { + sum(list: [1, 2, 3, 4, 5]) { total } } ` .expectData({ sum: { - total: 10, + total: 15, }, }) .on(maths); }); + + await t.should("Prime", async () => { + await gql` + query { + prime(number: 17) { + isPrime + } + } + ` + .expectData({ + prime: { + isPrime: true, + }, + }) + .on(maths); + }); + + // wasmtime._trap.Trap: error while executing at wasm backtrace + + // const geography = await t.engine("runtimes/grpc/geography.py"); + + // await t.should("show Contry Demography", async () => { + // await gql` + // query { + // dem(name: "France") { + // name + // capital + // population + // currencies { + // code + // name + // symbol + // } + // } + // } + // ` + // .expectData({ + // dem: { + // name: "France", + // capital: "Paris", + // population: 68035000, + // currencies: [ + // { + // code: "EURO", + // name: "Euro", + // symbol: "€", + // }, + // { + // code: "XPF", + // name: "CFP franc", + // symbol: "F", + // }, + // ], + // }, + // }) + // .on(geography); + // }); }); diff --git a/typegate/tests/runtimes/grpc/maths.py b/typegate/tests/runtimes/grpc/maths.py index 6f52fff509..c89df1eb0a 100644 --- a/typegate/tests/runtimes/grpc/maths.py +++ b/typegate/tests/runtimes/grpc/maths.py @@ -20,5 +20,5 @@ def maths(g: Graph): g.expose( Policy.public(), sum=maths_grpc.call_grpc_method("/maths.Calculator/Sum"), - isprime=maths_grpc.call_grpc_method("/maths.Calculator/IsPrime"), + prime=maths_grpc.call_grpc_method("/maths.Calculator/IsPrime"), ) From 004dadbe27ea47d22ed571f7c8b45ec69a7fd969 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Tue, 10 Sep 2024 07:41:50 +0300 Subject: [PATCH 26/34] re add proto file on test directory --- tests/runtimes/grpc/proto/geography.proto | 22 ++++++++++++++++++++ tests/runtimes/grpc/proto/helloworld.proto | 15 ++++++++++++++ tests/runtimes/grpc/proto/maths.proto | 24 ++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 tests/runtimes/grpc/proto/geography.proto create mode 100644 tests/runtimes/grpc/proto/helloworld.proto create mode 100644 tests/runtimes/grpc/proto/maths.proto diff --git a/tests/runtimes/grpc/proto/geography.proto b/tests/runtimes/grpc/proto/geography.proto new file mode 100644 index 0000000000..84bc259e67 --- /dev/null +++ b/tests/runtimes/grpc/proto/geography.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package geography; + +service Demography { + rpc Country(CountryRequest) returns (CountryReply) {} +} + +message CountryRequest { string name = 1; } + +message Currency { + string code = 1; + string name = 2; + string symbol = 3; +} + +message CountryReply { + string name = 1; + string capital = 2; + int32 population = 3; + repeated Currency currencies = 4; +} diff --git a/tests/runtimes/grpc/proto/helloworld.proto b/tests/runtimes/grpc/proto/helloworld.proto new file mode 100644 index 0000000000..1d90ea750d --- /dev/null +++ b/tests/runtimes/grpc/proto/helloworld.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package helloworld; + +service Greeter { + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +message HelloRequest { + string name = 1; +} + +message HelloReply { + string message = 1; +} diff --git a/tests/runtimes/grpc/proto/maths.proto b/tests/runtimes/grpc/proto/maths.proto new file mode 100644 index 0000000000..d02a58a687 --- /dev/null +++ b/tests/runtimes/grpc/proto/maths.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package maths; + +service Calculator { + rpc Sum (SumRequest) returns (SumReply) {} + rpc IsPrime (IsPrimeRequest) returns (IsPrimeReply) {} +} + +message SumRequest { + repeated int32 list = 1; +} + +message SumReply { + int32 total = 1; +} + +message IsPrimeRequest { + int32 number = 1; +} + +message IsPrimeReply { + bool isPrime = 1; +} From 8b48d1000fcfd3d6b5ffd3cd66253b56165a9fe4 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Tue, 10 Sep 2024 08:39:35 +0300 Subject: [PATCH 27/34] make grpc mock, work --- tools/compose/compose.grpc.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/compose/compose.grpc.yml b/tools/compose/compose.grpc.yml index 2a9147e9eb..34907d806a 100644 --- a/tools/compose/compose.grpc.yml +++ b/tools/compose/compose.grpc.yml @@ -6,8 +6,8 @@ services: - "4770:4770" - "4771:4771" volumes: - - ./tools/compose/proto:/proto - - ./tools/compose/stub:/stub + - ./proto:/proto + - ./stub:/stub command: - --stub=/stub - /proto/helloworld.proto From eb9d26cc63f602d0677ace7146f722de28388c49 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Tue, 10 Sep 2024 09:43:41 +0300 Subject: [PATCH 28/34] add grpc on github workflows --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9a7ade0008..2716e3432a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -259,7 +259,7 @@ jobs: WASM_OPT=1 ghjk x build-tgraph-ts # start the docker containers - ghjk x dev-compose base prisma + ghjk x dev-compose base prisma grpc WASM_OPT=1 ghjk x build-tgraph From 6b7973617010c93cee3446e7b8b10942563f6d19 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Tue, 10 Sep 2024 10:29:50 +0300 Subject: [PATCH 29/34] add caching on grpc type_generation --- .../core/src/runtimes/grpc/type_generation.rs | 32 +++++--- tests/runtimes/grpc/grpc_test.ts | 76 +++++++++---------- 2 files changed, 59 insertions(+), 49 deletions(-) diff --git a/src/typegraph/core/src/runtimes/grpc/type_generation.rs b/src/typegraph/core/src/runtimes/grpc/type_generation.rs index 7aa2ff8cdd..f8df3053f9 100644 --- a/src/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/src/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -6,6 +6,7 @@ use common::grpc::{ get_file_descriptor, get_message_field_descriptor, get_method_descriptor_proto, Fields, FileDescriptor, Type, }; +use std::collections::HashMap; use crate::{ t::{self, TypeBuilder}, @@ -17,6 +18,8 @@ pub struct GeneratedType { pub output: TypeId, } +type Cache = HashMap<String, TypeId>; + pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<GeneratedType> { let file_descriptor = get_file_descriptor(proto_file_content)?; let method_descriptor = get_method_descriptor_proto(file_descriptor.clone(), method_name)?; @@ -24,40 +27,47 @@ pub fn generate_type(proto_file_content: &str, method_name: &str) -> Result<Gene let input_type = method_descriptor.input_type(); let output_type = method_descriptor.output_type(); + let mut cache = HashMap::new(); + let input_fields = get_message_field_descriptor(&file_descriptor, input_type)?; let output_fields = get_message_field_descriptor(&file_descriptor, output_type)?; Ok(GeneratedType { - input: convert_proto_fields_to_type_id(&file_descriptor, input_fields)?, - output: convert_proto_fields_to_type_id(&file_descriptor, output_fields)?, + input: convert_proto_fields_to_type_id(&file_descriptor, input_fields, &mut cache)?, + output: convert_proto_fields_to_type_id(&file_descriptor, output_fields, &mut cache)?, }) } fn convert_proto_fields_to_type_id( file_descriptor: &FileDescriptor, fields: Fields, + cache: &mut Cache, ) -> Result<TypeId> { let mut r#type = t::struct_(); for field in fields { - let field_name = field.name(); + let field_name = field.name().to_string(); let type_name = field.proto().type_(); + if let Some(cached_type_id) = cache.get(&field_name) { + r#type.prop(&field_name, *cached_type_id); + continue; + } + let mut type_id = match type_name { Type::TYPE_STRING => t::string().build()?, Type::TYPE_INT32 | Type::TYPE_INT64 => t::integer().build()?, Type::TYPE_FLOAT => t::float().build()?, Type::TYPE_BOOL => t::boolean().build()?, Type::TYPE_MESSAGE => { - let nested_message_type = field.proto().name(); - convert_proto_fields_to_type_id( - file_descriptor, - get_message_field_descriptor(file_descriptor, nested_message_type)?, - )? + let nested_message_type = field.proto().type_name().to_string(); + let nested_fields = + get_message_field_descriptor(file_descriptor, &nested_message_type)?; + convert_proto_fields_to_type_id(file_descriptor, nested_fields, cache)? } _ => bail!( "Unsupported field type '{:?}' for field '{}'", - type_name, + field.proto().type_(), field_name ), }; @@ -70,7 +80,9 @@ fn convert_proto_fields_to_type_id( type_id = t::optional(type_id).build()?; } - r#type.prop(field_name, type_id); + cache.insert(field_name.clone(), type_id); + + r#type.prop(&field_name, type_id); } Ok(r#type.build()?) diff --git a/tests/runtimes/grpc/grpc_test.ts b/tests/runtimes/grpc/grpc_test.ts index 4077935178..c1acf4452e 100644 --- a/tests/runtimes/grpc/grpc_test.ts +++ b/tests/runtimes/grpc/grpc_test.ts @@ -56,44 +56,42 @@ Meta.test({ name: "Grpc Runtime" }, async (t) => { .on(maths); }); - // wasmtime._trap.Trap: error while executing at wasm backtrace + const geography = await t.engine("runtimes/grpc/geography.py"); - // const geography = await t.engine("runtimes/grpc/geography.py"); - - // await t.should("show Contry Demography", async () => { - // await gql` - // query { - // dem(name: "France") { - // name - // capital - // population - // currencies { - // code - // name - // symbol - // } - // } - // } - // ` - // .expectData({ - // dem: { - // name: "France", - // capital: "Paris", - // population: 68035000, - // currencies: [ - // { - // code: "EURO", - // name: "Euro", - // symbol: "€", - // }, - // { - // code: "XPF", - // name: "CFP franc", - // symbol: "F", - // }, - // ], - // }, - // }) - // .on(geography); - // }); + await t.should("show Contry Demography", async () => { + await gql` + query { + dem(name: "France") { + name + capital + population + currencies { + code + name + symbol + } + } + } + ` + .expectData({ + dem: { + name: "France", + capital: "Paris", + population: 68035000, + currencies: [ + { + code: "EUR", + name: "Euro", + symbol: "€", + }, + { + code: "XPF", + name: "CFP franc", + symbol: "F", + }, + ], + }, + }) + .on(geography); + }); }); From 5111c736ea1ab158394f2d39242e4495235a89fb Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Tue, 10 Sep 2024 18:02:17 +0300 Subject: [PATCH 30/34] add documentation && example && additional test --- .../docs/concepts/features-overview/index.mdx | 14 +- .../docs/reference/runtimes/grpc/index.mdx | 36 +++ .../docs/reference/runtimes/index.mdx | 1 + examples/typegraphs/grpc.py | 32 ++ examples/typegraphs/grpc.ts | 28 ++ examples/typegraphs/metagen/rs/mdk.rs | 2 +- examples/typegraphs/proto/helloworld.proto | 15 + tests/e2e/cli/dev_test.ts | 3 +- tests/e2e/website/website_test.ts | 1 + .../grpc/__snapshots__/grpc_test.ts.snap | 295 ++++++++++++++++++ tests/runtimes/grpc/grpc_test.ts | 13 + tests/runtimes/grpc/helloworld.ts | 21 ++ 12 files changed, 456 insertions(+), 5 deletions(-) create mode 100644 docs/metatype.dev/docs/reference/runtimes/grpc/index.mdx create mode 100644 examples/typegraphs/grpc.py create mode 100644 examples/typegraphs/grpc.ts create mode 100644 examples/typegraphs/proto/helloworld.proto create mode 100644 tests/runtimes/grpc/__snapshots__/grpc_test.ts.snap create mode 100644 tests/runtimes/grpc/helloworld.ts diff --git a/docs/metatype.dev/docs/concepts/features-overview/index.mdx b/docs/metatype.dev/docs/concepts/features-overview/index.mdx index ab5df0cb54..d6598bab32 100644 --- a/docs/metatype.dev/docs/concepts/features-overview/index.mdx +++ b/docs/metatype.dev/docs/concepts/features-overview/index.mdx @@ -59,9 +59,17 @@ We're taking any and all kinds of feature requests, suggestions and recommendati }, { title: "gRPC API", - status: "future", + status: "complete", link: "/docs/reference/typegate", - body: <>Support to expose the typegraph through a gRPC API.</>, + body: ( + <> + Expose your typegraph through a gRPC API, enabling high-performance, + bi-directional communication between services. This allows you to call + gRPC methods directly from the typegraph, providing more flexibility + and better integration with existing gRPC-based microservices. + This approach supports a broader range of real-time use cases . + </> + ), }, ], [ @@ -353,7 +361,7 @@ from typegraph.runtimes.deno import PythonRuntime ), ) -}`} </CodeBlock> ], [ { title: "S3 runtime", status: "complete", link: "/docs/reference/runtimes/s3", body: ( <> Store large blobs in S3 compatible object stores. This includes support for <Link to="/docs/guides/files-upload">GraphQL file uploads</Link> and presigned URLs making working with images and large uploads a breeze. </> ), }, { title: "Temporal runtime", status: "complete", link: "/docs/reference/runtimes/temporal", body: ( <> Kick off and query Temporal workflows from a typegraph. Temporal is a durable execution engine that can be used to implement long-running, failure-resistant business processes. </> ), }, { title: "Wasm runtime", status: "beta", link: "/docs/reference/runtimes/wasm", body: ( <> Make functions that use WASM components to implement their logic. Write core-wasm or wasi 0.2 based wasm components in languages like Rust, Python and Javascript to power all your custom functions. </> ), }, ], [ { title: "GraphQL runtime", status: "complete", link: "/docs/reference/runtimes/graphql", body: ( <> Include external GraphQL APIs as part of your typegraph. This runtime provides functions that resolve the data by querying an external GraphQL API. </> ), }, { title: "Http runtime", status: "complete", link: "/docs/reference/runtimes/http", body: ( <> Include external Http APIs as part of your typegraph. Source the data for your graph using external HTTP requests. </> ), }, ], [ { title: "Random runtime", status: "complete", link: "/docs/reference/runtimes/random", body: ( <> Functions to generate random data based on your types. Useful for quickly throwing together an API and testing. </> ), }, { title: "gRPC runtime", status: "future", link: "https://github.com/metatypedev/metatype/issues/723", body: ( <> Include external gRPC APIs as part of your typegraph. </> ), }, ], ]} /> +}`} </CodeBlock> ], [ { title: "S3 runtime", status: "complete", link: "/docs/reference/runtimes/s3", body: ( <> Store large blobs in S3 compatible object stores. This includes support for <Link to="/docs/guides/files-upload">GraphQL file uploads</Link> and presigned URLs making working with images and large uploads a breeze. </> ), }, { title: "Temporal runtime", status: "complete", link: "/docs/reference/runtimes/temporal", body: ( <> Kick off and query Temporal workflows from a typegraph. Temporal is a durable execution engine that can be used to implement long-running, failure-resistant business processes. </> ), }, { title: "Wasm runtime", status: "beta", link: "/docs/reference/runtimes/wasm", body: ( <> Make functions that use WASM components to implement their logic. Write core-wasm or wasi 0.2 based wasm components in languages like Rust, Python and Javascript to power all your custom functions. </> ), }, ], [ { title: "GraphQL runtime", status: "complete", link: "/docs/reference/runtimes/graphql", body: ( <> Include external GraphQL APIs as part of your typegraph. This runtime provides functions that resolve the data by querying an external GraphQL API. </> ), }, { title: "Http runtime", status: "complete", link: "/docs/reference/runtimes/http", body: ( <> Include external Http APIs as part of your typegraph. Source the data for your graph using external HTTP requests. </> ), }, ], [ { title: "Random runtime", status: "complete", link: "/docs/reference/runtimes/random", body: ( <> Functions to generate random data based on your types. Useful for quickly throwing together an API and testing. </> ), }, { title: "gRPC runtime", status: "complete", link: "/docs/reference/runtimes/grpc", body: ( <> Include external gRPC APIs as part of your typegraph. </> ), }, ], ]} /> ### Prisma diff --git a/docs/metatype.dev/docs/reference/runtimes/grpc/index.mdx b/docs/metatype.dev/docs/reference/runtimes/grpc/index.mdx new file mode 100644 index 0000000000..16342db7b0 --- /dev/null +++ b/docs/metatype.dev/docs/reference/runtimes/grpc/index.mdx @@ -0,0 +1,36 @@ +import TGExample from "@site/src/components/TGExample"; + +# gRPC Runtime + +## Overview + +The `GrpcRuntime` allows your Typegraphs to interact with external gRPC services. Similar to the HTTP runtime, this enables integrating gRPC services into your graph through a unified interface. + +Common use cases include: + +- Consuming a gRPC API within a Typegraph +- Calling gRPC methods programmatically via a type-safe structure + +In a `GrpcRuntime`, several key parameters help define how the graph interacts with the service, such as the gRPC **endpoint**, the **protofile** that describes the service, and the **method** to be called. + +## Variables + +### `proto_file` + +The `proto_file` is the path to the `.proto` file that defines the gRPC service. This file contains the service definitions, including message types and method signatures, that the Typegraph will use to communicate with the gRPC server. + +### `endpoint` + +The `endpoint` is the address of the gRPC server that the Typegraph will communicate with. It uses the format `tcp://<host>:<port>`, and is specified to point to the correct server and port where the gRPC service is running. + +### `call_grpc_method` + +This method is used to call a specific method on the gRPC service. It accepts the full path to the gRPC method, usually in the form `/package_name.service_name/method_name`. The **package_name** refers to the package defined in the `.proto` file, and it must be included when calling the method. In the example below, `greet` will call the `SayHello` method of the `Greeter` service within the `helloworld` package, as defined in the `helloworld.proto` file. + +## Example + +<TGExample + typegraph="grpc-runtime" + typescript={require("!!code-loader!../../../../../../examples/typegraphs/grpc.ts")} + python={require("!!code-loader!../../../../../../examples/typegraphs/grpc.py")} +/> diff --git a/docs/metatype.dev/docs/reference/runtimes/index.mdx b/docs/metatype.dev/docs/reference/runtimes/index.mdx index f51ed7e59a..25fe025e56 100644 --- a/docs/metatype.dev/docs/reference/runtimes/index.mdx +++ b/docs/metatype.dev/docs/reference/runtimes/index.mdx @@ -17,6 +17,7 @@ This includes: - [Temporal](./runtimes/temporal) - [S3](./runtimes/s3) - [KV](./runtimes/kv) +- [GRPC](./runtimes/grpc) :::tip Missing your favorite runtime? Submit your request and vote for your preferred ones [here](https://github.com/metatypedev/metatype/discussions/305). diff --git a/examples/typegraphs/grpc.py b/examples/typegraphs/grpc.py new file mode 100644 index 0000000000..f7c0cc24f6 --- /dev/null +++ b/examples/typegraphs/grpc.py @@ -0,0 +1,32 @@ +# skip:start +from typegraph.graph.params import Cors +from typegraph import Graph, Policy, typegraph + +# skip:end +# highlight-next-line +from typegraph.runtimes.grpc import GrpcRuntime + + +from pathlib import Path + + +BASE_DIR = Path(__file__).parent + + +@typegraph( + # skip:start + cors=Cors(allow_origin=["https://metatype.dev", "http://localhost:3000"]), + # skip:end +) +def grpc(g: Graph): + endpoint = "tcp://localhost:4770" + + proto_file = BASE_DIR.joinpath("proto/helloworld.proto") + + # highlight-next-line + grpc_runtime = GrpcRuntime(proto_file, endpoint) + + g.expose( + Policy.public(), + greet=grpc_runtime.call_grpc_method("/helloworld.Greeter/SayHello"), + ) diff --git a/examples/typegraphs/grpc.ts b/examples/typegraphs/grpc.ts new file mode 100644 index 0000000000..01eefd3303 --- /dev/null +++ b/examples/typegraphs/grpc.ts @@ -0,0 +1,28 @@ +// skip:start +import { Policy, typegraph } from "@typegraph/sdk/index.ts"; +import { GrpcRuntime } from "@typegraph/sdk/runtimes/grpc.ts"; + +// skip:end +const __dirname = new URL(".", import.meta.url).pathname; + +export const tg = await typegraph( + { + name: "grpc", + // skip:next-line + cors: { allowOrigin: ["https://metatype.dev", "http://localhost:3000"] }, + }, + (g) => { + const endpoint = "tcp://localhost:4770"; + + const proto_file = `${__dirname}proto/helloworld.proto`; + // highlight-next-line + const grpc_runtime = new GrpcRuntime( + proto_file, + endpoint, + ); + + g.expose({ + greet: grpc_runtime.call_grpc_method("/helloworld.Greeter/SayHello"), + }, Policy.public()); + }, +); diff --git a/examples/typegraphs/metagen/rs/mdk.rs b/examples/typegraphs/metagen/rs/mdk.rs index 610c6a0b5d..81d857ab0f 100644 --- a/examples/typegraphs/metagen/rs/mdk.rs +++ b/examples/typegraphs/metagen/rs/mdk.rs @@ -109,7 +109,7 @@ impl Router { } pub fn init(&self, args: InitArgs) -> Result<InitResponse, InitError> { - static MT_VERSION: &str = "0.4.10-rc1"; + static MT_VERSION: &str = "0.4.11-rc.0"; if args.metatype_version != MT_VERSION { return Err(InitError::VersionMismatch(MT_VERSION.into())); } diff --git a/examples/typegraphs/proto/helloworld.proto b/examples/typegraphs/proto/helloworld.proto new file mode 100644 index 0000000000..1d90ea750d --- /dev/null +++ b/examples/typegraphs/proto/helloworld.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package helloworld; + +service Greeter { + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +message HelloRequest { + string name = 1; +} + +message HelloReply { + string message = 1; +} diff --git a/tests/e2e/cli/dev_test.ts b/tests/e2e/cli/dev_test.ts index 95c4fc4d81..dbe8c00d87 100644 --- a/tests/e2e/cli/dev_test.ts +++ b/tests/e2e/cli/dev_test.ts @@ -314,7 +314,7 @@ Meta.test({ } deployed.push([match[2].slice(prefix.length), match[1]]); } - return deployed.length < 41; + return deployed.length < 42; }, 3 * 60 * 1000); await t.should("have deployed all the typegraphs", () => { @@ -336,6 +336,7 @@ Meta.test({ ["func.ts", "roadmap-func"], ["graphql-server.ts", "graphql-server"], ["graphql.ts", "graphql"], + ["grpc.ts", "grpc"], ["http-runtime.ts", "http-runtime"], ["iam-provider.ts", "iam-provider"], ["index.ts", "homepage"], diff --git a/tests/e2e/website/website_test.ts b/tests/e2e/website/website_test.ts index 9279fe63e9..372495f9c2 100644 --- a/tests/e2e/website/website_test.ts +++ b/tests/e2e/website/website_test.ts @@ -29,6 +29,7 @@ const list = [ "func", "graphql-server", "graphql", + "grpc", "http-runtime", "iam-provider", "index", diff --git a/tests/runtimes/grpc/__snapshots__/grpc_test.ts.snap b/tests/runtimes/grpc/__snapshots__/grpc_test.ts.snap new file mode 100644 index 0000000000..a0923ba324 --- /dev/null +++ b/tests/runtimes/grpc/__snapshots__/grpc_test.ts.snap @@ -0,0 +1,295 @@ +export const snapshot = {}; + +snapshot[`Typegraph using grpc 1`] = ` +'[ + { + "types": [ + { + "type": "object", + "title": "helloworld", + "runtime": 0, + "policies": [], + "config": {}, + "as_id": false, + "properties": { + "greet": 1 + }, + "required": [ + "greet" + ] + }, + { + "type": "function", + "title": "func_7", + "runtime": 1, + "policies": [ + 0 + ], + "config": {}, + "as_id": false, + "input": 2, + "output": 5, + "materializer": 0, + "rate_weight": null, + "rate_calls": false + }, + { + "type": "object", + "title": "object_2", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false, + "properties": { + "name": 3 + }, + "required": [] + }, + { + "type": "optional", + "title": "optional_1", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false, + "item": 4, + "default_value": null + }, + { + "type": "string", + "title": "string_0", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false + }, + { + "type": "object", + "title": "object_5", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false, + "properties": { + "message": 3 + }, + "required": [] + } + ], + "materializers": [ + { + "name": "grpc", + "runtime": 1, + "effect": { + "effect": "read", + "idempotent": true + }, + "data": { + "method": "/helloworld.Greeter/SayHello" + } + }, + { + "name": "predefined_function", + "runtime": 0, + "effect": { + "effect": "read", + "idempotent": true + }, + "data": { + "name": "true" + } + } + ], + "runtimes": [ + { + "name": "deno", + "data": { + "worker": "default", + "permissions": {} + } + }, + { + "name": "grpc", + "data": { + "proto_file_content": "syntax = \\\\"proto3\\\\";\\\\n\\\\npackage helloworld;\\\\n\\\\nservice Greeter {\\\\n rpc SayHello (HelloRequest) returns (HelloReply) {}\\\\n}\\\\n\\\\nmessage HelloRequest {\\\\n string name = 1;\\\\n}\\\\n\\\\nmessage HelloReply {\\\\n string message = 1;\\\\n}\\\\n", + "endpoint": "tcp://localhost:4770" + } + } + ], + "policies": [ + { + "name": "__public", + "materializer": 1 + } + ], + "meta": { + "prefix": null, + "secrets": [], + "queries": { + "dynamic": true, + "endpoints": [] + }, + "cors": { + "allow_origin": [], + "allow_headers": [], + "expose_headers": [], + "allow_methods": [], + "allow_credentials": true, + "max_age_sec": null + }, + "auths": [], + "rate": null, + "version": "0.0.3", + "randomSeed": null, + "artifacts": {} + } + } +]' +`; + +snapshot[`Typegraph using grpc 2`] = ` +'[ + { + "types": [ + { + "type": "object", + "title": "helloworld", + "runtime": 0, + "policies": [], + "config": {}, + "as_id": false, + "properties": { + "greet": 1 + }, + "required": [ + "greet" + ] + }, + { + "type": "function", + "title": "func_7", + "runtime": 1, + "policies": [ + 0 + ], + "config": {}, + "as_id": false, + "input": 2, + "output": 5, + "materializer": 0, + "rate_weight": null, + "rate_calls": false + }, + { + "type": "object", + "title": "object_2", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false, + "properties": { + "name": 3 + }, + "required": [] + }, + { + "type": "optional", + "title": "optional_1", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false, + "item": 4, + "default_value": null + }, + { + "type": "string", + "title": "string_0", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false + }, + { + "type": "object", + "title": "object_5", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false, + "properties": { + "message": 3 + }, + "required": [] + } + ], + "materializers": [ + { + "name": "grpc", + "runtime": 1, + "effect": { + "effect": "read", + "idempotent": true + }, + "data": { + "method": "/helloworld.Greeter/SayHello" + } + }, + { + "name": "predefined_function", + "runtime": 0, + "effect": { + "effect": "read", + "idempotent": true + }, + "data": { + "name": "true" + } + } + ], + "runtimes": [ + { + "name": "deno", + "data": { + "worker": "default", + "permissions": {} + } + }, + { + "name": "grpc", + "data": { + "proto_file_content": "syntax = \\\\"proto3\\\\";\\\\n\\\\npackage helloworld;\\\\n\\\\nservice Greeter {\\\\n rpc SayHello (HelloRequest) returns (HelloReply) {}\\\\n}\\\\n\\\\nmessage HelloRequest {\\\\n string name = 1;\\\\n}\\\\n\\\\nmessage HelloReply {\\\\n string message = 1;\\\\n}\\\\n", + "endpoint": "tcp://localhost:4770" + } + } + ], + "policies": [ + { + "name": "__public", + "materializer": 1 + } + ], + "meta": { + "prefix": null, + "secrets": [], + "queries": { + "dynamic": true, + "endpoints": [] + }, + "cors": { + "allow_origin": [], + "allow_headers": [], + "expose_headers": [], + "allow_methods": [], + "allow_credentials": true, + "max_age_sec": null + }, + "auths": [], + "rate": null, + "version": "0.0.3", + "randomSeed": null, + "artifacts": {} + } + } +]' +`; diff --git a/tests/runtimes/grpc/grpc_test.ts b/tests/runtimes/grpc/grpc_test.ts index c1acf4452e..a58c14e5df 100644 --- a/tests/runtimes/grpc/grpc_test.ts +++ b/tests/runtimes/grpc/grpc_test.ts @@ -1,8 +1,21 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 +import { MetaTest } from "../../utils/test.ts"; import { gql, Meta } from "../../utils/mod.ts"; +async function testSerialize(t: MetaTest, file: string) { + await t.should(`serialize typegraph ${file}`, async () => { + const { stdout: tg } = await Meta.cli("serialize", "--pretty", "-f", file); + await t.assertSnapshot(tg); + }); +} + +Meta.test({ name: "Typegraph using grpc" }, async (t) => { + await testSerialize(t, "runtimes/grpc/helloworld.ts"); + await testSerialize(t, "runtimes/grpc/helloworld.py"); +}); + Meta.test({ name: "Grpc Runtime" }, async (t) => { const hello_world = await t.engine("runtimes/grpc/helloworld.py"); diff --git a/tests/runtimes/grpc/helloworld.ts b/tests/runtimes/grpc/helloworld.ts new file mode 100644 index 0000000000..0ea7bfe75f --- /dev/null +++ b/tests/runtimes/grpc/helloworld.ts @@ -0,0 +1,21 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { Policy, typegraph } from "@typegraph/sdk/index.ts"; +import { GrpcRuntime } from "@typegraph/sdk/runtimes/grpc.ts"; + +const __dirname = new URL(".", import.meta.url).pathname; + +export const tg = await typegraph("helloworld", (g) => { + const endpoint = "tcp://localhost:4770"; + const helloworld = `${__dirname}proto/helloworld.proto`; + + const helloworld_grpc = new GrpcRuntime( + helloworld, + endpoint, + ); + + g.expose({ + greet: helloworld_grpc.call_grpc_method("/helloworld.Greeter/SayHello"), + }, Policy.public()); +}); From 4136fe58f6cfe5fc36e7eaedd4c2839ea0ce331c Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Thu, 12 Sep 2024 10:05:53 +0300 Subject: [PATCH 31/34] rename call_grpc_method to call on sdk && and improe caching --- .../docs/concepts/features-overview/index.mdx | 16 ++++++++-------- .../docs/reference/runtimes/grpc/index.mdx | 6 ++++-- examples/typegraphs/grpc.py | 2 +- examples/typegraphs/grpc.ts | 14 +++++++------- src/typegate/engine/runtime.d.ts | 2 +- .../core/src/runtimes/grpc/type_generation.rs | 16 ++++++++++------ src/typegraph/deno/src/runtimes/grpc.ts | 2 +- src/typegraph/python/typegraph/runtimes/grpc.py | 2 +- tests/runtimes/grpc/geography.py | 6 +++--- tests/runtimes/grpc/helloworld.py | 6 +++--- tests/runtimes/grpc/helloworld.ts | 16 ++++++++-------- tests/runtimes/grpc/maths.py | 8 ++++---- 12 files changed, 51 insertions(+), 45 deletions(-) diff --git a/docs/metatype.dev/docs/concepts/features-overview/index.mdx b/docs/metatype.dev/docs/concepts/features-overview/index.mdx index d6598bab32..3727559114 100644 --- a/docs/metatype.dev/docs/concepts/features-overview/index.mdx +++ b/docs/metatype.dev/docs/concepts/features-overview/index.mdx @@ -63,11 +63,11 @@ We're taking any and all kinds of feature requests, suggestions and recommendati link: "/docs/reference/typegate", body: ( <> - Expose your typegraph through a gRPC API, enabling high-performance, - bi-directional communication between services. This allows you to call - gRPC methods directly from the typegraph, providing more flexibility - and better integration with existing gRPC-based microservices. - This approach supports a broader range of real-time use cases . + Expose your gRPC API, enabling high-performance, bi-directional + communication between services. This allows you to call gRPC methods + directly from the typegraph, providing more flexibility and better + integration with existing gRPC-based microservices. This approach + supports a broader range of real-time use cases . </> ), }, @@ -314,7 +314,7 @@ We're taking any and all kinds of feature requests, suggestions and recommendati Includes easy access to ESM and libraries through standard Deno features. </> ), - }, + }, ], [ <TGExample @@ -333,7 +333,7 @@ We're taking any and all kinds of feature requests, suggestions and recommendati Implement functions that execute python functions in code snippets or on disk modules. </> ), - }, + }, ], [ <CodeBlock @@ -543,7 +543,7 @@ Access a myriad of databases directly from your typegraph. Run queries, mange it Bundles the typegate within it making the CLI all one needs to get started. (And a text editor, of course.) </> ), - }, + }, ], [ <CodeBlock diff --git a/docs/metatype.dev/docs/reference/runtimes/grpc/index.mdx b/docs/metatype.dev/docs/reference/runtimes/grpc/index.mdx index 16342db7b0..0e62623e74 100644 --- a/docs/metatype.dev/docs/reference/runtimes/grpc/index.mdx +++ b/docs/metatype.dev/docs/reference/runtimes/grpc/index.mdx @@ -23,9 +23,11 @@ The `proto_file` is the path to the `.proto` file that defines the gRPC service. The `endpoint` is the address of the gRPC server that the Typegraph will communicate with. It uses the format `tcp://<host>:<port>`, and is specified to point to the correct server and port where the gRPC service is running. -### `call_grpc_method` +## Method -This method is used to call a specific method on the gRPC service. It accepts the full path to the gRPC method, usually in the form `/package_name.service_name/method_name`. The **package_name** refers to the package defined in the `.proto` file, and it must be included when calling the method. In the example below, `greet` will call the `SayHello` method of the `Greeter` service within the `helloworld` package, as defined in the `helloworld.proto` file. +### `call` + +This method creates a typegraph function for gRPC method calls. It accepts the full path to the gRPC method, usually in the form `/package_name.service_name/method_name`. The **package_name** refers to the package defined in the `.proto` file, and it must be included when calling the method. In the example below, `greet` will call the `SayHello` method of the `Greeter` service within the `helloworld` package, as defined in the `helloworld.proto` file. ## Example diff --git a/examples/typegraphs/grpc.py b/examples/typegraphs/grpc.py index f7c0cc24f6..982d985bc0 100644 --- a/examples/typegraphs/grpc.py +++ b/examples/typegraphs/grpc.py @@ -28,5 +28,5 @@ def grpc(g: Graph): g.expose( Policy.public(), - greet=grpc_runtime.call_grpc_method("/helloworld.Greeter/SayHello"), + greet=grpc_runtime.call("/helloworld.Greeter/SayHello"), ) diff --git a/examples/typegraphs/grpc.ts b/examples/typegraphs/grpc.ts index 01eefd3303..2554517f50 100644 --- a/examples/typegraphs/grpc.ts +++ b/examples/typegraphs/grpc.ts @@ -16,13 +16,13 @@ export const tg = await typegraph( const proto_file = `${__dirname}proto/helloworld.proto`; // highlight-next-line - const grpc_runtime = new GrpcRuntime( - proto_file, - endpoint, - ); + const grpc_runtime = new GrpcRuntime(proto_file, endpoint); - g.expose({ - greet: grpc_runtime.call_grpc_method("/helloworld.Greeter/SayHello"), - }, Policy.public()); + g.expose( + { + greet: grpc_runtime.call("/helloworld.Greeter/SayHello"), + }, + Policy.public(), + ); }, ); diff --git a/src/typegate/engine/runtime.d.ts b/src/typegate/engine/runtime.d.ts index 3c4218e6ac..0099bdd8dd 100644 --- a/src/typegate/engine/runtime.d.ts +++ b/src/typegate/engine/runtime.d.ts @@ -54,7 +54,7 @@ type MetaNS = { grpc: { register: (inp: GrpcRegisterInput) => Promise<void>; - unregister: (client_id: string) => void; + unregister: (client_id: string) => Promise<void>; callGrpcMethod: (inp: CallGrpcMethodInput) => Promise<string>; }; }; diff --git a/src/typegraph/core/src/runtimes/grpc/type_generation.rs b/src/typegraph/core/src/runtimes/grpc/type_generation.rs index f8df3053f9..1a6e2d8f74 100644 --- a/src/typegraph/core/src/runtimes/grpc/type_generation.rs +++ b/src/typegraph/core/src/runtimes/grpc/type_generation.rs @@ -47,22 +47,25 @@ fn convert_proto_fields_to_type_id( for field in fields { let field_name = field.name().to_string(); - let type_name = field.proto().type_(); + let type_name = field.proto().type_name(); - if let Some(cached_type_id) = cache.get(&field_name) { + let cache_key = format!("{}/{}", type_name, field_name); + + // Check if the type is already cached + if let Some(cached_type_id) = cache.get(&cache_key) { r#type.prop(&field_name, *cached_type_id); continue; } - let mut type_id = match type_name { + let mut type_id = match field.proto().type_() { Type::TYPE_STRING => t::string().build()?, Type::TYPE_INT32 | Type::TYPE_INT64 => t::integer().build()?, Type::TYPE_FLOAT => t::float().build()?, Type::TYPE_BOOL => t::boolean().build()?, Type::TYPE_MESSAGE => { - let nested_message_type = field.proto().type_name().to_string(); + let nested_message_type = field.proto().type_name(); let nested_fields = - get_message_field_descriptor(file_descriptor, &nested_message_type)?; + get_message_field_descriptor(file_descriptor, nested_message_type)?; convert_proto_fields_to_type_id(file_descriptor, nested_fields, cache)? } _ => bail!( @@ -80,7 +83,8 @@ fn convert_proto_fields_to_type_id( type_id = t::optional(type_id).build()?; } - cache.insert(field_name.clone(), type_id); + // Cache the type_id using the unique cache key + cache.insert(cache_key, type_id); r#type.prop(&field_name, type_id); } diff --git a/src/typegraph/deno/src/runtimes/grpc.ts b/src/typegraph/deno/src/runtimes/grpc.ts index 7fc9bc4d1f..56b5e63e35 100644 --- a/src/typegraph/deno/src/runtimes/grpc.ts +++ b/src/typegraph/deno/src/runtimes/grpc.ts @@ -17,7 +17,7 @@ export class GrpcRuntime extends Runtime { super(id); } - call_grpc_method(method: string) { + call(method: string) { const funcData = runtimes.callGrpcMethod(this._id, { method: method }); return Func.fromTypeFunc(funcData); } diff --git a/src/typegraph/python/typegraph/runtimes/grpc.py b/src/typegraph/python/typegraph/runtimes/grpc.py index f06f25c4f2..5e82f9a22e 100644 --- a/src/typegraph/python/typegraph/runtimes/grpc.py +++ b/src/typegraph/python/typegraph/runtimes/grpc.py @@ -23,7 +23,7 @@ def __init__(self, proto_file: str, endpoint: str): super().__init__(runtime_id.value) - def call_grpc_method(self, method: str): + def call(self, method: str): data = GrpcData(method) func_data = runtimes.call_grpc_method(store, self.id, data) diff --git a/tests/runtimes/grpc/geography.py b/tests/runtimes/grpc/geography.py index 4e7ffb0ea2..70e359b8c9 100644 --- a/tests/runtimes/grpc/geography.py +++ b/tests/runtimes/grpc/geography.py @@ -14,10 +14,10 @@ def geography(g: Graph): endpoint = "tcp://localhost:4770" - geography = BASE_DIR.joinpath("proto/geography.proto") - geography_grpc = GrpcRuntime(geography, endpoint) + proto_file = BASE_DIR.joinpath("proto/geography.proto") + grpc_runtime = GrpcRuntime(proto_file, endpoint) g.expose( Policy.public(), - dem=geography_grpc.call_grpc_method("/geography.Demography/Country"), + dem=grpc_runtime.call("/geography.Demography/Country"), ) diff --git a/tests/runtimes/grpc/helloworld.py b/tests/runtimes/grpc/helloworld.py index afef73ad92..2aec246ad5 100644 --- a/tests/runtimes/grpc/helloworld.py +++ b/tests/runtimes/grpc/helloworld.py @@ -14,10 +14,10 @@ def helloworld(g: Graph): endpoint = "tcp://localhost:4770" - helloworld = BASE_DIR.joinpath("proto/helloworld.proto") - helloworld_grpc = GrpcRuntime(helloworld, endpoint) + proto_file = BASE_DIR.joinpath("proto/helloworld.proto") + grpc_runtime = GrpcRuntime(proto_file, endpoint) g.expose( Policy.public(), - greet=helloworld_grpc.call_grpc_method("/helloworld.Greeter/SayHello"), + greet=grpc_runtime.call("/helloworld.Greeter/SayHello"), ) diff --git a/tests/runtimes/grpc/helloworld.ts b/tests/runtimes/grpc/helloworld.ts index 0ea7bfe75f..7c9edc18bc 100644 --- a/tests/runtimes/grpc/helloworld.ts +++ b/tests/runtimes/grpc/helloworld.ts @@ -8,14 +8,14 @@ const __dirname = new URL(".", import.meta.url).pathname; export const tg = await typegraph("helloworld", (g) => { const endpoint = "tcp://localhost:4770"; - const helloworld = `${__dirname}proto/helloworld.proto`; + const proto_file = `${__dirname}proto/helloworld.proto`; - const helloworld_grpc = new GrpcRuntime( - helloworld, - endpoint, - ); + const grpc_runtime = new GrpcRuntime(proto_file, endpoint); - g.expose({ - greet: helloworld_grpc.call_grpc_method("/helloworld.Greeter/SayHello"), - }, Policy.public()); + g.expose( + { + greet: grpc_runtime.call("/helloworld.Greeter/SayHello"), + }, + Policy.public(), + ); }); diff --git a/tests/runtimes/grpc/maths.py b/tests/runtimes/grpc/maths.py index c89df1eb0a..fd7a10f226 100644 --- a/tests/runtimes/grpc/maths.py +++ b/tests/runtimes/grpc/maths.py @@ -14,11 +14,11 @@ def maths(g: Graph): endpoint = "tcp://localhost:4770" - maths = BASE_DIR.joinpath("proto/maths.proto") - maths_grpc = GrpcRuntime(maths, endpoint) + proto_file = BASE_DIR.joinpath("proto/maths.proto") + grpc_runtime = GrpcRuntime(proto_file, endpoint) g.expose( Policy.public(), - sum=maths_grpc.call_grpc_method("/maths.Calculator/Sum"), - prime=maths_grpc.call_grpc_method("/maths.Calculator/IsPrime"), + sum=grpc_runtime.call("/maths.Calculator/Sum"), + prime=grpc_runtime.call("/maths.Calculator/IsPrime"), ) From 3fa90905e363246d3ecb58377185167856072696 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Mon, 16 Sep 2024 18:40:53 +0300 Subject: [PATCH 32/34] remove BASE_DIR --- examples/typegraphs/grpc.py | 9 +-------- examples/typegraphs/grpc.ts | 3 +-- tests/runtimes/grpc/geography.py | 7 +------ tests/runtimes/grpc/helloworld.py | 7 +------ tests/runtimes/grpc/helloworld.ts | 4 +--- tests/runtimes/grpc/maths.py | 7 +------ 6 files changed, 6 insertions(+), 31 deletions(-) diff --git a/examples/typegraphs/grpc.py b/examples/typegraphs/grpc.py index 982d985bc0..4e923965e2 100644 --- a/examples/typegraphs/grpc.py +++ b/examples/typegraphs/grpc.py @@ -7,12 +7,6 @@ from typegraph.runtimes.grpc import GrpcRuntime -from pathlib import Path - - -BASE_DIR = Path(__file__).parent - - @typegraph( # skip:start cors=Cors(allow_origin=["https://metatype.dev", "http://localhost:3000"]), @@ -20,8 +14,7 @@ ) def grpc(g: Graph): endpoint = "tcp://localhost:4770" - - proto_file = BASE_DIR.joinpath("proto/helloworld.proto") + proto_file = "typegraphs/proto/helloworld.proto" # highlight-next-line grpc_runtime = GrpcRuntime(proto_file, endpoint) diff --git a/examples/typegraphs/grpc.ts b/examples/typegraphs/grpc.ts index 2554517f50..15b749a563 100644 --- a/examples/typegraphs/grpc.ts +++ b/examples/typegraphs/grpc.ts @@ -3,7 +3,6 @@ import { Policy, typegraph } from "@typegraph/sdk/index.ts"; import { GrpcRuntime } from "@typegraph/sdk/runtimes/grpc.ts"; // skip:end -const __dirname = new URL(".", import.meta.url).pathname; export const tg = await typegraph( { @@ -13,8 +12,8 @@ export const tg = await typegraph( }, (g) => { const endpoint = "tcp://localhost:4770"; + const proto_file = "typegraphs/proto/helloworld.proto"; - const proto_file = `${__dirname}proto/helloworld.proto`; // highlight-next-line const grpc_runtime = new GrpcRuntime(proto_file, endpoint); diff --git a/tests/runtimes/grpc/geography.py b/tests/runtimes/grpc/geography.py index 70e359b8c9..29fabaea81 100644 --- a/tests/runtimes/grpc/geography.py +++ b/tests/runtimes/grpc/geography.py @@ -1,20 +1,15 @@ # Copyright Metatype OÜ, licensed under the Elastic License 2.0. # SPDX-License-Identifier: Elastic-2.0 -from pathlib import Path - from typegraph import Graph, Policy, typegraph from typegraph.runtimes.grpc import GrpcRuntime -BASE_DIR = Path(__file__).parent - - @typegraph() def geography(g: Graph): endpoint = "tcp://localhost:4770" + proto_file = "runtimes/grpc/proto/geography.proto" - proto_file = BASE_DIR.joinpath("proto/geography.proto") grpc_runtime = GrpcRuntime(proto_file, endpoint) g.expose( diff --git a/tests/runtimes/grpc/helloworld.py b/tests/runtimes/grpc/helloworld.py index 2aec246ad5..9268573777 100644 --- a/tests/runtimes/grpc/helloworld.py +++ b/tests/runtimes/grpc/helloworld.py @@ -1,20 +1,15 @@ # Copyright Metatype OÜ, licensed under the Elastic License 2.0. # SPDX-License-Identifier: Elastic-2.0 -from pathlib import Path - from typegraph import Graph, Policy, typegraph from typegraph.runtimes.grpc import GrpcRuntime -BASE_DIR = Path(__file__).parent - - @typegraph() def helloworld(g: Graph): endpoint = "tcp://localhost:4770" + proto_file = "runtimes/grpc/proto/helloworld.proto" - proto_file = BASE_DIR.joinpath("proto/helloworld.proto") grpc_runtime = GrpcRuntime(proto_file, endpoint) g.expose( diff --git a/tests/runtimes/grpc/helloworld.ts b/tests/runtimes/grpc/helloworld.ts index 7c9edc18bc..dbbcadc5d9 100644 --- a/tests/runtimes/grpc/helloworld.ts +++ b/tests/runtimes/grpc/helloworld.ts @@ -4,11 +4,9 @@ import { Policy, typegraph } from "@typegraph/sdk/index.ts"; import { GrpcRuntime } from "@typegraph/sdk/runtimes/grpc.ts"; -const __dirname = new URL(".", import.meta.url).pathname; - export const tg = await typegraph("helloworld", (g) => { const endpoint = "tcp://localhost:4770"; - const proto_file = `${__dirname}proto/helloworld.proto`; + const proto_file = "runtimes/grpc/proto/helloworld.proto"; const grpc_runtime = new GrpcRuntime(proto_file, endpoint); diff --git a/tests/runtimes/grpc/maths.py b/tests/runtimes/grpc/maths.py index fd7a10f226..91b81625c6 100644 --- a/tests/runtimes/grpc/maths.py +++ b/tests/runtimes/grpc/maths.py @@ -1,20 +1,15 @@ # Copyright Metatype OÜ, licensed under the Elastic License 2.0. # SPDX-License-Identifier: Elastic-2.0 -from pathlib import Path - from typegraph import Graph, Policy, typegraph from typegraph.runtimes.grpc import GrpcRuntime -BASE_DIR = Path(__file__).parent - - @typegraph() def maths(g: Graph): endpoint = "tcp://localhost:4770" + proto_file = "runtimes/grpc/proto/maths.proto" - proto_file = BASE_DIR.joinpath("proto/maths.proto") grpc_runtime = GrpcRuntime(proto_file, endpoint) g.expose( From 165d8552b0505bb925d1967c390a2f78926ae854 Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Wed, 18 Sep 2024 09:40:10 +0300 Subject: [PATCH 33/34] access to the directory relative of typegraph --- examples/typegraphs/grpc.py | 4 ++-- examples/typegraphs/grpc.ts | 2 +- src/typegate/src/sync/typegraph.ts | 6 +++++- src/typegraph/core/src/conversion/runtimes.rs | 2 +- src/typegraph/core/src/runtimes/grpc/mod.rs | 2 +- src/typegraph/core/src/runtimes/mod.rs | 7 +++++++ src/typegraph/core/src/typegraph.rs | 13 +++++++++++++ src/typegraph/core/src/utils/mod.rs | 2 +- src/typegraph/core/wit/typegraph.wit | 2 +- src/typegraph/deno/src/runtimes/grpc.ts | 5 +---- src/typegraph/deno/src/typegraph.ts | 4 +++- src/typegraph/python/typegraph/graph/typegraph.py | 2 +- src/typegraph/python/typegraph/runtimes/grpc.py | 7 ++----- tests/runtimes/grpc/geography.py | 2 +- tests/runtimes/grpc/helloworld.py | 2 +- tests/runtimes/grpc/helloworld.ts | 2 +- tests/runtimes/grpc/maths.py | 2 +- 17 files changed, 43 insertions(+), 23 deletions(-) diff --git a/examples/typegraphs/grpc.py b/examples/typegraphs/grpc.py index 4e923965e2..a6deaea434 100644 --- a/examples/typegraphs/grpc.py +++ b/examples/typegraphs/grpc.py @@ -1,6 +1,6 @@ # skip:start -from typegraph.graph.params import Cors from typegraph import Graph, Policy, typegraph +from typegraph.graph.params import Cors # skip:end # highlight-next-line @@ -14,7 +14,7 @@ ) def grpc(g: Graph): endpoint = "tcp://localhost:4770" - proto_file = "typegraphs/proto/helloworld.proto" + proto_file = "proto/helloworld.proto" # highlight-next-line grpc_runtime = GrpcRuntime(proto_file, endpoint) diff --git a/examples/typegraphs/grpc.ts b/examples/typegraphs/grpc.ts index 15b749a563..18a97b878b 100644 --- a/examples/typegraphs/grpc.ts +++ b/examples/typegraphs/grpc.ts @@ -12,7 +12,7 @@ export const tg = await typegraph( }, (g) => { const endpoint = "tcp://localhost:4770"; - const proto_file = "typegraphs/proto/helloworld.proto"; + const proto_file = "proto/helloworld.proto"; // highlight-next-line const grpc_runtime = new GrpcRuntime(proto_file, endpoint); diff --git a/src/typegate/src/sync/typegraph.ts b/src/typegate/src/sync/typegraph.ts index 08a390558b..370596a950 100644 --- a/src/typegate/src/sync/typegraph.ts +++ b/src/typegate/src/sync/typegraph.ts @@ -1,7 +1,11 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -import { SecretManager, TypeGraph, type TypeGraphDS } from "../typegraph/mod.ts"; +import { + SecretManager, + TypeGraph, + type TypeGraphDS, +} from "../typegraph/mod.ts"; import { GetObjectCommand, PutObjectCommand, diff --git a/src/typegraph/core/src/conversion/runtimes.rs b/src/typegraph/core/src/conversion/runtimes.rs index 3e227f793c..13880c9232 100644 --- a/src/typegraph/core/src/conversion/runtimes.rs +++ b/src/typegraph/core/src/conversion/runtimes.rs @@ -507,7 +507,7 @@ pub fn convert_runtime(_c: &mut TypegraphContext, runtime: Runtime) -> Result<Co } Runtime::Kv(d) => Ok(TGRuntime::Known(Rt::Kv(KvRuntimeData { url: d.url.clone() })).into()), Runtime::Grpc(d) => Ok(TGRuntime::Known(Rt::Grpc(GrpcRuntimeData { - proto_file_content: d.proto_file_content.clone(), + proto_file_content: d.proto_file.clone(), endpoint: d.endpoint.clone(), })) .into()), diff --git a/src/typegraph/core/src/runtimes/grpc/mod.rs b/src/typegraph/core/src/runtimes/grpc/mod.rs index 6e212ff6ab..2a821c03da 100644 --- a/src/typegraph/core/src/runtimes/grpc/mod.rs +++ b/src/typegraph/core/src/runtimes/grpc/mod.rs @@ -57,7 +57,7 @@ pub fn call_grpc_method(runtime: RuntimeId, data: GrpcData) -> Result<FuncParams let mat_id = Store::register_materializer(super::Materializer::grpc(runtime, mat, WitEffect::Read)); - let t = type_generation::generate_type(&grpc_runtime_data.proto_file_content, &data.method) + let t = type_generation::generate_type(&grpc_runtime_data.proto_file, &data.method) .map_err(|err| format!("failed generate type {err}"))?; Ok(FuncParams { diff --git a/src/typegraph/core/src/runtimes/mod.rs b/src/typegraph/core/src/runtimes/mod.rs index 4afd7d4cfe..df1d74e86e 100644 --- a/src/typegraph/core/src/runtimes/mod.rs +++ b/src/typegraph/core/src/runtimes/mod.rs @@ -15,6 +15,7 @@ pub mod typegraph; pub mod wasm; use std::cell::RefCell; +use std::path::Path; use std::rc::Rc; use crate::conversion::runtimes::MaterializerConverter; @@ -24,6 +25,8 @@ use crate::runtimes::prisma::migration::{ }; use crate::runtimes::typegraph::TypegraphOperation; use crate::t::TypeBuilder; +use crate::typegraph::current_typegraph_dir; +use crate::utils::fs::FsContext; use crate::validation::types::validate_value; use crate::wit::aws::S3RuntimeData; use crate::wit::core::{FuncParams, MaterializerId, RuntimeId, TypeId as CoreTypeId}; @@ -709,6 +712,10 @@ impl crate::wit::runtimes::Guest for crate::Lib { } fn register_grpc_runtime(data: GrpcRuntimeData) -> Result<RuntimeId, wit::Error> { + let fs_ctx = FsContext::new(current_typegraph_dir()?); + let proto_file = fs_ctx.read_text_file(Path::new(&data.proto_file))?; + let data = GrpcRuntimeData { proto_file, ..data }; + Ok(Store::register_runtime(Runtime::Grpc(data.into()))) } diff --git a/src/typegraph/core/src/typegraph.rs b/src/typegraph/core/src/typegraph.rs index 32e63cc69f..f946f715d6 100644 --- a/src/typegraph/core/src/typegraph.rs +++ b/src/typegraph/core/src/typegraph.rs @@ -23,6 +23,7 @@ use std::cell::RefCell; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::hash::Hasher as _; +use std::path::{Path, PathBuf}; use std::rc::Rc; use crate::wit::core::{ @@ -47,6 +48,7 @@ struct RuntimeContexts { #[derive(Default)] pub struct TypegraphContext { name: String, + path: Option<Rc<Path>>, meta: TypeMeta, types: Vec<Option<TypeNode>>, runtimes: Vec<TGRuntime>, @@ -95,6 +97,7 @@ pub fn init(params: TypegraphInitParams) -> Result<()> { let mut ctx = TypegraphContext { name: params.name.clone(), + path: Some(PathBuf::from(¶ms.path).into()), meta: TypeMeta { version: TYPEGRAPH_VERSION.to_string(), queries: Queries { @@ -490,3 +493,13 @@ impl TypegraphContext { self.mapping.policies.get(&id).copied() } } + +pub fn current_typegraph_path() -> Result<Rc<Path>> { + with_tg(|tg| tg.path.clone().unwrap()) +} + +pub fn current_typegraph_dir() -> Result<PathBuf> { + let tg_path = current_typegraph_path()?; + // TODO error handling + Ok(tg_path.parent().unwrap().to_owned()) +} diff --git a/src/typegraph/core/src/utils/mod.rs b/src/typegraph/core/src/utils/mod.rs index 7802839047..cd2fdb15da 100644 --- a/src/typegraph/core/src/utils/mod.rs +++ b/src/typegraph/core/src/utils/mod.rs @@ -22,7 +22,7 @@ use std::path::Path; mod archive; mod artifacts; -mod fs; +pub mod fs; pub mod metagen_utils; mod oauth2; mod pathlib; diff --git a/src/typegraph/core/wit/typegraph.wit b/src/typegraph/core/wit/typegraph.wit index 1c95f7bb72..4a07ca4460 100644 --- a/src/typegraph/core/wit/typegraph.wit +++ b/src/typegraph/core/wit/typegraph.wit @@ -537,7 +537,7 @@ interface runtimes { // Grpc record grpc-runtime-data { - proto-file-content: string, + proto-file: string, endpoint: string, } diff --git a/src/typegraph/deno/src/runtimes/grpc.ts b/src/typegraph/deno/src/runtimes/grpc.ts index 56b5e63e35..d75be27f79 100644 --- a/src/typegraph/deno/src/runtimes/grpc.ts +++ b/src/typegraph/deno/src/runtimes/grpc.ts @@ -7,11 +7,8 @@ import { Runtime } from "./mod.ts"; export class GrpcRuntime extends Runtime { constructor(protoFile: string, endpoint: string) { - const protoFileContent = new TextDecoder().decode( - Deno.readFileSync(protoFile), - ); const id = runtimes.registerGrpcRuntime({ - protoFileContent, + protoFile, endpoint, }); super(id); diff --git a/src/typegraph/deno/src/typegraph.ts b/src/typegraph/deno/src/typegraph.ts index e477885234..a43ed06a94 100644 --- a/src/typegraph/deno/src/typegraph.ts +++ b/src/typegraph/deno/src/typegraph.ts @@ -165,7 +165,9 @@ export async function typegraph( } // node/deno compat tick until MET-236 is landed const simpleFile = file.replace(/:[0-9]+$/, "").replace(/^file:\/\//, ""); - const path = dirname(fromFileUrl(`file://${simpleFile}`)); + const path = fromFileUrl(`file://${simpleFile}`); + + console.log({ path }); const defaultCorsFields = { allowCredentials: true, diff --git a/src/typegraph/python/typegraph/graph/typegraph.py b/src/typegraph/python/typegraph/graph/typegraph.py index 6ed7e5ffee..5ecf16682f 100644 --- a/src/typegraph/python/typegraph/graph/typegraph.py +++ b/src/typegraph/python/typegraph/graph/typegraph.py @@ -51,7 +51,7 @@ def __init__( ): self.name = name self.dynamic = dynamic - self.path = str(Path(inspect.stack()[2].filename).resolve().parent) + self.path = str(Path(inspect.stack()[2].filename).resolve()) self.rate = rate diff --git a/src/typegraph/python/typegraph/runtimes/grpc.py b/src/typegraph/python/typegraph/runtimes/grpc.py index 5e82f9a22e..9b8fde896a 100644 --- a/src/typegraph/python/typegraph/runtimes/grpc.py +++ b/src/typegraph/python/typegraph/runtimes/grpc.py @@ -2,21 +2,18 @@ # SPDX-License-Identifier: MPL-2.0 from typegraph import t -from typegraph.runtimes.base import Runtime from typegraph.gen.exports.runtimes import ( GrpcData, GrpcRuntimeData, ) from typegraph.gen.types import Err +from typegraph.runtimes.base import Runtime from typegraph.wit import runtimes, store class GrpcRuntime(Runtime): def __init__(self, proto_file: str, endpoint: str): - file = open(proto_file, "r") - proto_file_content = file.read() - file.close() - data = GrpcRuntimeData(proto_file_content, endpoint) + data = GrpcRuntimeData(proto_file, endpoint) runtime_id = runtimes.register_grpc_runtime(store, data) if isinstance(runtime_id, Err): raise Exception(runtime_id.value) diff --git a/tests/runtimes/grpc/geography.py b/tests/runtimes/grpc/geography.py index 29fabaea81..b5d6432bec 100644 --- a/tests/runtimes/grpc/geography.py +++ b/tests/runtimes/grpc/geography.py @@ -8,7 +8,7 @@ @typegraph() def geography(g: Graph): endpoint = "tcp://localhost:4770" - proto_file = "runtimes/grpc/proto/geography.proto" + proto_file = "proto/geography.proto" grpc_runtime = GrpcRuntime(proto_file, endpoint) diff --git a/tests/runtimes/grpc/helloworld.py b/tests/runtimes/grpc/helloworld.py index 9268573777..6888a2c1c1 100644 --- a/tests/runtimes/grpc/helloworld.py +++ b/tests/runtimes/grpc/helloworld.py @@ -8,7 +8,7 @@ @typegraph() def helloworld(g: Graph): endpoint = "tcp://localhost:4770" - proto_file = "runtimes/grpc/proto/helloworld.proto" + proto_file = "proto/helloworld.proto" grpc_runtime = GrpcRuntime(proto_file, endpoint) diff --git a/tests/runtimes/grpc/helloworld.ts b/tests/runtimes/grpc/helloworld.ts index dbbcadc5d9..58f2ccd6d1 100644 --- a/tests/runtimes/grpc/helloworld.ts +++ b/tests/runtimes/grpc/helloworld.ts @@ -6,7 +6,7 @@ import { GrpcRuntime } from "@typegraph/sdk/runtimes/grpc.ts"; export const tg = await typegraph("helloworld", (g) => { const endpoint = "tcp://localhost:4770"; - const proto_file = "runtimes/grpc/proto/helloworld.proto"; + const proto_file = "proto/helloworld.proto"; const grpc_runtime = new GrpcRuntime(proto_file, endpoint); diff --git a/tests/runtimes/grpc/maths.py b/tests/runtimes/grpc/maths.py index 91b81625c6..216f185de0 100644 --- a/tests/runtimes/grpc/maths.py +++ b/tests/runtimes/grpc/maths.py @@ -8,7 +8,7 @@ @typegraph() def maths(g: Graph): endpoint = "tcp://localhost:4770" - proto_file = "runtimes/grpc/proto/maths.proto" + proto_file = "proto/maths.proto" grpc_runtime = GrpcRuntime(proto_file, endpoint) From 582afe687cb46addd862b5f770558dcd2972d30d Mon Sep 17 00:00:00 2001 From: j03-dev <24nomeniavo@gmail.com> Date: Wed, 18 Sep 2024 10:05:53 +0300 Subject: [PATCH 34/34] remove console log --- src/typegraph/deno/src/typegraph.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/typegraph/deno/src/typegraph.ts b/src/typegraph/deno/src/typegraph.ts index a43ed06a94..8f91adab2c 100644 --- a/src/typegraph/deno/src/typegraph.ts +++ b/src/typegraph/deno/src/typegraph.ts @@ -167,8 +167,6 @@ export async function typegraph( const simpleFile = file.replace(/:[0-9]+$/, "").replace(/^file:\/\//, ""); const path = fromFileUrl(`file://${simpleFile}`); - console.log({ path }); - const defaultCorsFields = { allowCredentials: true, allowHeaders: [],