Skip to content

Commit e8e1192

Browse files
aidandjAidan Jensen
andauthored
Add generate_concrete_servicer_stubs option (#688)
Signed-off-by: Aidan Jensen <aidandj.github@gmail.com> Co-authored-by: Aidan Jensen <aidan.jensen@robust.ai>
1 parent d8f24c3 commit e8e1192

40 files changed

+2013
-35
lines changed

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ __pycache__/
88
*.pyc
99
/test/generated/**/*.py
1010
!/test/generated/**/__init__.py
11+
/test/generated-concrete/**/*.py
12+
!/test/generated-concrete/**/__init__.py
1113
.pytest_cache
1214
/build/
1315
/dist/

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
- With some more work this could be added back in a testing refactor
77
- Protobuf <6.32 still had the edition enums and field options, so it *should* still work. But is untested
88
- Add support for editions (up to 2024)
9+
- Add `generate_concrete_servicer_stubs` option to generate concrete instead of abstract servicer stubs
910

1011
## 3.7.0
1112

README.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,14 @@ and insert it into the deprecation warning. This option will instead use a stand
337337

338338
```
339339
protoc --python_out=output/location --mypy_out=use_default_deprecation_warning:output/location
340+
```
341+
342+
### `generate_concrete_servicer_stubs`
340343

344+
By default mypy-protobuf will output servicer stubs with abstract methods. To output concrete stubs, set this option
345+
346+
```
347+
protoc --python_out=output/location --mypy_grpc_out=generate_concrete_servicer_stubs:output/location
341348
```
342349

343350
### Output suppression

mypy_protobuf/main.py

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -151,13 +151,15 @@ def __init__(
151151
readable_stubs: bool,
152152
relax_strict_optional_primitives: bool,
153153
use_default_deprecation_warnings: bool,
154+
generate_concrete_servicer_stubs: bool,
154155
grpc: bool,
155156
) -> None:
156157
self.fd = fd
157158
self.descriptors = descriptors
158159
self.readable_stubs = readable_stubs
159160
self.relax_strict_optional_primitives = relax_strict_optional_primitives
160161
self.use_default_depreaction_warnings = use_default_deprecation_warnings
162+
self.generate_concrete_servicer_stubs = generate_concrete_servicer_stubs
161163
self.grpc = grpc
162164
self.lines: List[str] = []
163165
self.indent = ""
@@ -689,6 +691,7 @@ def write_services(
689691
self._import("google.protobuf.service", "Service"),
690692
self._import("abc", "ABCMeta"),
691693
)
694+
# The servicer interface
692695
with self._indent():
693696
if self._write_comments(scl):
694697
wl("")
@@ -850,7 +853,8 @@ def write_grpc_methods(self, service: d.ServiceDescriptorProto, scl_prefix: Sour
850853
for i, method in methods:
851854
scl = scl_prefix + [d.ServiceDescriptorProto.METHOD_FIELD_NUMBER, i]
852855

853-
wl("@{}", self._import("abc", "abstractmethod"))
856+
if self.generate_concrete_servicer_stubs is False:
857+
wl("@{}", self._import("abc", "abstractmethod"))
854858
wl("def {}(", method.name)
855859
with self._indent():
856860
wl("self,")
@@ -950,11 +954,17 @@ def write_grpc_services(
950954
scl + [d.ServiceDescriptorProto.OPTIONS_FIELD_NUMBER] + [d.ServiceOptions.DEPRECATED_FIELD_NUMBER],
951955
"This servicer has been marked as deprecated using proto service options.",
952956
)
953-
wl(
954-
"class {}Servicer(metaclass={}):",
955-
service.name,
956-
self._import("abc", "ABCMeta"),
957-
)
957+
if self.generate_concrete_servicer_stubs is False:
958+
wl(
959+
"class {}Servicer(metaclass={}):",
960+
service.name,
961+
self._import("abc", "ABCMeta"),
962+
)
963+
else:
964+
wl(
965+
"class {}Servicer:",
966+
service.name,
967+
)
958968
with self._indent():
959969
if self._write_comments(scl):
960970
wl("")
@@ -1126,6 +1136,7 @@ def generate_mypy_stubs(
11261136
readable_stubs: bool,
11271137
relax_strict_optional_primitives: bool,
11281138
use_default_deprecation_warnings: bool,
1139+
generate_concrete_servicer_stubs: bool,
11291140
) -> None:
11301141
for name, fd in descriptors.to_generate.items():
11311142
pkg_writer = PkgWriter(
@@ -1134,6 +1145,7 @@ def generate_mypy_stubs(
11341145
readable_stubs,
11351146
relax_strict_optional_primitives,
11361147
use_default_deprecation_warnings,
1148+
generate_concrete_servicer_stubs,
11371149
grpc=False,
11381150
)
11391151

@@ -1158,6 +1170,7 @@ def generate_mypy_grpc_stubs(
11581170
readable_stubs: bool,
11591171
relax_strict_optional_primitives: bool,
11601172
use_default_deprecation_warnings: bool,
1173+
generate_concrete_servicer_stubs: bool,
11611174
) -> None:
11621175
for name, fd in descriptors.to_generate.items():
11631176
pkg_writer = PkgWriter(
@@ -1166,6 +1179,7 @@ def generate_mypy_grpc_stubs(
11661179
readable_stubs,
11671180
relax_strict_optional_primitives,
11681181
use_default_deprecation_warnings,
1182+
generate_concrete_servicer_stubs,
11691183
grpc=True,
11701184
)
11711185
pkg_writer.write_grpc_async_hacks()
@@ -1222,6 +1236,7 @@ def main() -> None:
12221236
"readable_stubs" in request.parameter,
12231237
"relax_strict_optional_primitives" in request.parameter,
12241238
"use_default_deprecation_warnings" in request.parameter,
1239+
"generate_concrete_servicer_stubs" in request.parameter,
12251240
)
12261241

12271242

@@ -1235,6 +1250,7 @@ def grpc() -> None:
12351250
"readable_stubs" in request.parameter,
12361251
"relax_strict_optional_primitives" in request.parameter,
12371252
"use_default_deprecation_warnings" in request.parameter,
1253+
"generate_concrete_servicer_stubs" in request.parameter,
12381254
)
12391255

12401256

pyproject.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,13 @@ include = [
3232
]
3333
exclude = [
3434
"**/*_pb2.py",
35-
"**/*_pb2_grpc.py"
35+
"**/*_pb2_grpc.py",
36+
"test/test_concrete.py"
3637
]
3738

3839
executionEnvironments = [
3940
# Due to how upb is typed, we need to disable incompatible variable override checks
4041
{ root = "test/generated", extraPaths = ["./"], reportIncompatibleVariableOverride = "none" },
42+
{ root = "test/generated-concrete", extraPaths = ["./"], reportIncompatibleVariableOverride = "none" },
4143
{ root = "mypy_protobuf/extensions_pb2.pyi", reportIncompatibleVariableOverride = "none" },
4244
]

run_test.sh

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,18 +124,26 @@ MYPY_PROTOBUF_VENV=venv_$PY_VER_MYPY_PROTOBUF
124124
find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=readable_stubs:test/generated
125125
find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=relax_strict_optional_primitives:test/generated
126126
find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=use_default_deprecation_warnings:test/generated
127+
find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=generate_concrete_servicer_stubs:test/generated
127128
# Overwrite w/ run with mypy-protobuf without flags
128129
find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=test/generated
129130

130131
# Generate grpc protos
131132
find proto/testproto/grpc -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_grpc_out=test/generated
132133

134+
# Generate with concrete service stubs for testing
135+
find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=generate_concrete_servicer_stubs:test/generated-concrete
136+
find proto/testproto/grpc -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_grpc_out=generate_concrete_servicer_stubs:test/generated-concrete
137+
138+
133139
if [[ -n $VALIDATE ]] && ! diff <(echo "$SHA_BEFORE") <(find test/generated -name "*.pyi" -print0 | xargs -0 sha1sum); then
134140
echo -e "${RED}Some .pyi files did not match. Please commit those files${NC}"
135141
exit 1
136142
fi
137143
)
138144

145+
ERRORS=()
146+
139147
for PY_VER in $PY_VER_UNIT_TESTS; do
140148
UNIT_TESTS_VENV=venv_$PY_VER
141149
PY_VER_MYPY_TARGET=$(echo "$PY_VER" | cut -d. -f1-2)
@@ -149,6 +157,10 @@ for PY_VER in $PY_VER_UNIT_TESTS; do
149157
# Run mypy on unit tests / generated output
150158
(
151159
source "$MYPY_VENV"/bin/activate
160+
# Run concrete mypy
161+
CONCRETE_MODULES=( -m test.test_concrete )
162+
MYPYPATH=$MYPYPATH:test/generated-concrete mypy ${CUSTOM_TYPESHED_DIR_ARG:+"$CUSTOM_TYPESHED_DIR_ARG"} --python-executable="$UNIT_TESTS_VENV"/bin/python --python-version="$PY_VER_MYPY_TARGET" "${CONCRETE_MODULES[@]}"
163+
152164
export MYPYPATH=$MYPYPATH:test/generated
153165

154166
# Run mypy
@@ -184,13 +196,13 @@ for PY_VER in $PY_VER_UNIT_TESTS; do
184196

185197
call_mypy "$PY_VER" "${NEGATIVE_MODULES[@]}"
186198
if ! diff "$MYPY_OUTPUT/mypy_output" "test_negative/output.expected.$PY_VER_MYPY_TARGET" || ! diff "$MYPY_OUTPUT/mypy_output.omit_linenos" "test_negative/output.expected.$PY_VER_MYPY_TARGET.omit_linenos"; then
187-
echo -e "${RED}test_negative/output.expected.$PY_VER_MYPY_TARGET didnt match. Copying over for you. Now rerun${NC}"
188-
189199
# Copy over all the mypy results for the developer.
190200
call_mypy "$PY_VER" "${NEGATIVE_MODULES[@]}"
191201
cp "$MYPY_OUTPUT/mypy_output" "test_negative/output.expected.$PY_VER_MYPY_TARGET"
192202
cp "$MYPY_OUTPUT/mypy_output.omit_linenos" "test_negative/output.expected.$PY_VER_MYPY_TARGET.omit_linenos"
193-
exit 1
203+
204+
# Record error instead of echoing and exiting
205+
ERRORS+=("test_negative/output.expected.$PY_VER_MYPY_TARGET didnt match. Copying over for you.")
194206
fi
195207
)
196208

@@ -200,3 +212,13 @@ for PY_VER in $PY_VER_UNIT_TESTS; do
200212
PYTHONPATH=test/generated py.test --ignore=test/generated -v
201213
)
202214
done
215+
216+
# Report all errors at the end
217+
if [ ${#ERRORS[@]} -gt 0 ]; then
218+
echo -e "\n${RED}===============================================${NC}"
219+
for error in "${ERRORS[@]}"; do
220+
echo -e "${RED}$error${NC}"
221+
done
222+
echo -e "${RED}Now rerun${NC}"
223+
exit 1
224+
fi
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
"""
2+
@generated by mypy-protobuf. Do not edit manually!
3+
isort:skip_file
4+
Protocol Buffers - Google's data interchange format
5+
Copyright 2008 Google Inc. All rights reserved.
6+
https://developers.google.com/protocol-buffers/
7+
8+
Redistribution and use in source and binary forms, with or without
9+
modification, are permitted provided that the following conditions are
10+
met:
11+
12+
* Redistributions of source code must retain the above copyright
13+
notice, this list of conditions and the following disclaimer.
14+
* Redistributions in binary form must reproduce the above
15+
copyright notice, this list of conditions and the following disclaimer
16+
in the documentation and/or other materials provided with the
17+
distribution.
18+
* Neither the name of Google Inc. nor the names of its
19+
contributors may be used to endorse or promote products derived from
20+
this software without specific prior written permission.
21+
22+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
"""
34+
35+
import builtins
36+
import google.protobuf.descriptor
37+
import google.protobuf.internal.well_known_types
38+
import google.protobuf.message
39+
import sys
40+
import typing
41+
42+
if sys.version_info >= (3, 10):
43+
import typing as typing_extensions
44+
else:
45+
import typing_extensions
46+
47+
DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
48+
49+
@typing.final
50+
class Duration(google.protobuf.message.Message, google.protobuf.internal.well_known_types.Duration):
51+
"""A Duration represents a signed, fixed-length span of time represented
52+
as a count of seconds and fractions of seconds at nanosecond
53+
resolution. It is independent of any calendar and concepts like "day"
54+
or "month". It is related to Timestamp in that the difference between
55+
two Timestamp values is a Duration and it can be added or subtracted
56+
from a Timestamp. Range is approximately +-10,000 years.
57+
58+
# Examples
59+
60+
Example 1: Compute Duration from two Timestamps in pseudo code.
61+
62+
Timestamp start = ...;
63+
Timestamp end = ...;
64+
Duration duration = ...;
65+
66+
duration.seconds = end.seconds - start.seconds;
67+
duration.nanos = end.nanos - start.nanos;
68+
69+
if (duration.seconds < 0 && duration.nanos > 0) {
70+
duration.seconds += 1;
71+
duration.nanos -= 1000000000;
72+
} else if (duration.seconds > 0 && duration.nanos < 0) {
73+
duration.seconds -= 1;
74+
duration.nanos += 1000000000;
75+
}
76+
77+
Example 2: Compute Timestamp from Timestamp + Duration in pseudo code.
78+
79+
Timestamp start = ...;
80+
Duration duration = ...;
81+
Timestamp end = ...;
82+
83+
end.seconds = start.seconds + duration.seconds;
84+
end.nanos = start.nanos + duration.nanos;
85+
86+
if (end.nanos < 0) {
87+
end.seconds -= 1;
88+
end.nanos += 1000000000;
89+
} else if (end.nanos >= 1000000000) {
90+
end.seconds += 1;
91+
end.nanos -= 1000000000;
92+
}
93+
94+
Example 3: Compute Duration from datetime.timedelta in Python.
95+
96+
td = datetime.timedelta(days=3, minutes=10)
97+
duration = Duration()
98+
duration.FromTimedelta(td)
99+
100+
# JSON Mapping
101+
102+
In JSON format, the Duration type is encoded as a string rather than an
103+
object, where the string ends in the suffix "s" (indicating seconds) and
104+
is preceded by the number of seconds, with nanoseconds expressed as
105+
fractional seconds. For example, 3 seconds with 0 nanoseconds should be
106+
encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should
107+
be expressed in JSON format as "3.000000001s", and 3 seconds and 1
108+
microsecond should be expressed in JSON format as "3.000001s".
109+
"""
110+
111+
DESCRIPTOR: google.protobuf.descriptor.Descriptor
112+
113+
SECONDS_FIELD_NUMBER: builtins.int
114+
NANOS_FIELD_NUMBER: builtins.int
115+
seconds: builtins.int
116+
"""Signed seconds of the span of time. Must be from -315,576,000,000
117+
to +315,576,000,000 inclusive. Note: these bounds are computed from:
118+
60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years
119+
"""
120+
nanos: builtins.int
121+
"""Signed fractions of a second at nanosecond resolution of the span
122+
of time. Durations less than one second are represented with a 0
123+
`seconds` field and a positive or negative `nanos` field. For durations
124+
of one second or more, a non-zero value for the `nanos` field must be
125+
of the same sign as the `seconds` field. Must be from -999,999,999
126+
to +999,999,999 inclusive.
127+
"""
128+
def __init__(
129+
self,
130+
*,
131+
seconds: builtins.int = ...,
132+
nanos: builtins.int = ...,
133+
) -> None: ...
134+
def ClearField(self, field_name: typing.Literal["nanos", b"nanos", "seconds", b"seconds"]) -> None: ...
135+
136+
Global___Duration: typing_extensions.TypeAlias = Duration

0 commit comments

Comments
 (0)