From 78b0d27d58e3e2322ae53a0b7491a0ef57663c4f Mon Sep 17 00:00:00 2001 From: "joshua.ho@bytedance.com" Date: Fri, 8 Aug 2025 16:19:04 +0800 Subject: [PATCH 1/3] add extra field for custom proto files --- protobuf-parse/src/pure/mod.rs | 2 +- .../src/pure/parse_and_typecheck.rs | 44 ++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/protobuf-parse/src/pure/mod.rs b/protobuf-parse/src/pure/mod.rs index c60fb2fc0..eb803d9ed 100644 --- a/protobuf-parse/src/pure/mod.rs +++ b/protobuf-parse/src/pure/mod.rs @@ -2,7 +2,7 @@ pub(crate) mod convert; pub(crate) mod model; -pub(crate) mod parse_and_typecheck; +pub mod parse_and_typecheck; pub(crate) mod parse_dependencies; mod parser; diff --git a/protobuf-parse/src/pure/parse_and_typecheck.rs b/protobuf-parse/src/pure/parse_and_typecheck.rs index 4101c4be7..605570772 100644 --- a/protobuf-parse/src/pure/parse_and_typecheck.rs +++ b/protobuf-parse/src/pure/parse_and_typecheck.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::fmt; use std::fs; use std::io; @@ -53,6 +54,7 @@ where { parsed_files: IndexMap, resolver: R, + custom_embedded: HashMap, } impl Run @@ -144,7 +146,19 @@ where return self.add_file_content(protobuf_path, &resolved); } - let embedded = match protobuf_path.to_str() { + let protobuf_path_str = protobuf_path.to_str(); + + if let Some(content) = self.custom_embedded.get(protobuf_path_str) { + return self.add_file_content( + protobuf_path, + &ResolvedProtoFile { + path: protobuf_path_str.to_string(), + content: content.as_bytes().to_vec(), + }, + ); + } + + let embedded = match protobuf_path_str { "rustproto.proto" => Some(proto::RUSTPROTO_PROTO), "google/protobuf/any.proto" => Some(proto::ANY_PROTO), "google/protobuf/api.proto" => Some(proto::API_PROTO), @@ -250,9 +264,25 @@ fn fs_resolver(includes: &[PathBuf]) -> impl ProtoPathResolver { /// Parse `.proto` files using pure Rust implementation. pub fn parse_and_typecheck(parser: &Parser) -> anyhow::Result { + parse_and_typecheck_impl(parser, HashMap::new()) +} + +/// Parse `.proto` files with custom embedded proto files. +pub fn parse_and_typecheck_with_embedded( + parser: &Parser, + custom_embedded: HashMap, +) -> anyhow::Result { + parse_and_typecheck_impl(parser, custom_embedded) +} + +fn parse_and_typecheck_impl( + parser: &Parser, + custom_embedded: HashMap, +) -> anyhow::Result { let mut run = Run { parsed_files: IndexMap::new(), resolver: fs_resolver(&parser.includes), + custom_embedded, }; let relative_paths = parser @@ -286,14 +316,24 @@ pub fn parse_and_typecheck(parser: &Parser) -> anyhow::Result anyhow::Result> { + parse_and_typecheck_custom_with_embedded(input, resolver, HashMap::new()) +} + +/// Parse and typecheck with custom resolver and embedded files. +pub fn parse_and_typecheck_custom_with_embedded( + input: &[ProtoPathBuf], + resolver: impl ProtoPathResolver, + custom_embedded: HashMap, ) -> anyhow::Result> { let mut run = Run { parsed_files: IndexMap::new(), resolver, + custom_embedded, }; for proto_path in input { From cd56e7f451c527927c80b5f792228ca6dca635cd Mon Sep 17 00:00:00 2001 From: "joshua.ho@bytedance.com" Date: Fri, 8 Aug 2025 17:19:55 +0800 Subject: [PATCH 2/3] add custom_embedded field in Parser struct --- protobuf-parse/src/parser.rs | 15 ++++++++++ .../src/pure/parse_and_typecheck.rs | 30 ++----------------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/protobuf-parse/src/parser.rs b/protobuf-parse/src/parser.rs index e642ab4c8..34e82a92f 100644 --- a/protobuf-parse/src/parser.rs +++ b/protobuf-parse/src/parser.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::collections::HashSet; use std::ffi::OsStr; use std::ffi::OsString; @@ -21,6 +22,7 @@ pub struct Parser { pub(crate) protoc: Option, pub(crate) protoc_extra_args: Vec, pub(crate) capture_stderr: bool, + pub(crate) custom_embedded: HashMap, } impl Parser { @@ -97,6 +99,19 @@ impl Parser { self } + /// Adds an embedded `.proto` file to the parser. + /// + /// This allows the parser to use `.proto` files provided as in-memory + /// strings rather than reading them from disk. + pub fn add_custom_embedded( + &mut self, + name: impl Into, + content: impl Into, + ) -> &mut Self { + self.custom_embedded.insert(name.into(), content.into()); + self + } + /// Parse `.proto` files and typecheck them using pure Rust parser of `protoc` command. pub fn parse_and_typecheck(&self) -> anyhow::Result { match &self.which_parser { diff --git a/protobuf-parse/src/pure/parse_and_typecheck.rs b/protobuf-parse/src/pure/parse_and_typecheck.rs index 605570772..0da400945 100644 --- a/protobuf-parse/src/pure/parse_and_typecheck.rs +++ b/protobuf-parse/src/pure/parse_and_typecheck.rs @@ -264,25 +264,10 @@ fn fs_resolver(includes: &[PathBuf]) -> impl ProtoPathResolver { /// Parse `.proto` files using pure Rust implementation. pub fn parse_and_typecheck(parser: &Parser) -> anyhow::Result { - parse_and_typecheck_impl(parser, HashMap::new()) -} - -/// Parse `.proto` files with custom embedded proto files. -pub fn parse_and_typecheck_with_embedded( - parser: &Parser, - custom_embedded: HashMap, -) -> anyhow::Result { - parse_and_typecheck_impl(parser, custom_embedded) -} - -fn parse_and_typecheck_impl( - parser: &Parser, - custom_embedded: HashMap, -) -> anyhow::Result { let mut run = Run { parsed_files: IndexMap::new(), resolver: fs_resolver(&parser.includes), - custom_embedded, + custom_embedded: parser.custom_embedded.clone(), }; let relative_paths = parser @@ -316,24 +301,15 @@ fn parse_and_typecheck_impl( }) } -/// Parse and typecheck with custom resolver. +/// TODO: this API is to be refactored. pub fn parse_and_typecheck_custom( input: &[ProtoPathBuf], resolver: impl ProtoPathResolver, -) -> anyhow::Result> { - parse_and_typecheck_custom_with_embedded(input, resolver, HashMap::new()) -} - -/// Parse and typecheck with custom resolver and embedded files. -pub fn parse_and_typecheck_custom_with_embedded( - input: &[ProtoPathBuf], - resolver: impl ProtoPathResolver, - custom_embedded: HashMap, ) -> anyhow::Result> { let mut run = Run { parsed_files: IndexMap::new(), resolver, - custom_embedded, + custom_embedded: HashMap::new(), }; for proto_path in input { From 6571577e6eff517f491a003b898472c653bd0b73 Mon Sep 17 00:00:00 2001 From: gemalto Date: Fri, 8 Aug 2025 19:24:14 +0800 Subject: [PATCH 3/3] update protobuf-parse version and fix errors --- protobuf-codegen/Cargo.toml | 2 +- protobuf-parse/Cargo.toml | 2 +- protobuf-parse/src/pure/mod.rs | 2 +- protobuf/src/well_known_types/any.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/protobuf-codegen/Cargo.toml b/protobuf-codegen/Cargo.toml index ff6332b57..cf1d01e34 100644 --- a/protobuf-codegen/Cargo.toml +++ b/protobuf-codegen/Cargo.toml @@ -23,7 +23,7 @@ once_cell = "1.10.0" tempfile = "3" protobuf = { path = "../protobuf", version = "=4.0.0-alpha.0" } -protobuf-parse = { path = "../protobuf-parse", version = "=4.0.0-alpha.0" } +protobuf-parse = { path = "../protobuf-parse", version = "=4.0.0-alpha.1" } [[bin]] diff --git a/protobuf-parse/Cargo.toml b/protobuf-parse/Cargo.toml index e79976bd9..bf02b5b62 100644 --- a/protobuf-parse/Cargo.toml +++ b/protobuf-parse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "protobuf-parse" -version = "4.0.0-alpha.0" +version = "4.0.0-alpha.1" edition = "2021" authors = ["Stepan Koltsov "] license = "MIT" diff --git a/protobuf-parse/src/pure/mod.rs b/protobuf-parse/src/pure/mod.rs index eb803d9ed..c60fb2fc0 100644 --- a/protobuf-parse/src/pure/mod.rs +++ b/protobuf-parse/src/pure/mod.rs @@ -2,7 +2,7 @@ pub(crate) mod convert; pub(crate) mod model; -pub mod parse_and_typecheck; +pub(crate) mod parse_and_typecheck; pub(crate) mod parse_dependencies; mod parser; diff --git a/protobuf/src/well_known_types/any.rs b/protobuf/src/well_known_types/any.rs index 4b4b21979..34fe737d7 100644 --- a/protobuf/src/well_known_types/any.rs +++ b/protobuf/src/well_known_types/any.rs @@ -37,7 +37,7 @@ pub struct Any { /// server that maps type URLs to message definitions as follows: /// /// * If no scheme is provided, `https` is assumed. - /// * An HTTP GET on the URL must yield a [google.protobuf.Type][] + /// * An HTTP GET on the URL must yield a \[google.protobuf.Type\][] /// value in binary format, or produce an error. /// * Applications are allowed to cache lookup results based on the /// URL, or have them precompiled into a binary to avoid any