Skip to content

Commit 7d29339

Browse files
basundhara-cbasundhara-c
committed
reverse tunnel sandbox
Signed-off-by: Basundhara Chakrabarty <basundhara.c@nutanix.com>
1 parent e7d10bf commit 7d29339

File tree

7 files changed

+534
-0
lines changed

7 files changed

+534
-0
lines changed

‎reverse_tunnel/Dockerfile.xds‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
FROM ubuntu:20.04
2+
3+
# Prevent interactive prompts during package installation
4+
ENV DEBIAN_FRONTEND=noninteractive
5+
6+
WORKDIR /app
7+
8+
# Install Python and pip
9+
RUN apt-get update && apt-get install -y \
10+
python3 \
11+
python3-pip \
12+
&& rm -rf /var/lib/apt/lists/*
13+
14+
# Install dependencies
15+
RUN pip3 install requests pyyaml
16+
17+
# Create a simple xDS server script
18+
RUN echo '#!/usr/bin/env python3\n\
19+
import json\n\
20+
import time\n\
21+
import threading\n\
22+
import http.server\n\
23+
import socketserver\n\
24+
import logging\n\
25+
\n\
26+
logging.basicConfig(level=logging.INFO)\n\
27+
logger = logging.getLogger(__name__)\n\
28+
\n\
29+
class XDSServer:\n\
30+
def __init__(self):\n\
31+
self.listeners = {}\n\
32+
self.version = 1\n\
33+
self._lock = threading.Lock()\n\
34+
self.server = None\n\
35+
\n\
36+
def start(self, port):\n\
37+
class XDSHandler(http.server.BaseHTTPRequestHandler):\n\
38+
def do_POST(self):\n\
39+
if self.path == "/v3/discovery:listeners":\n\
40+
content_length = int(self.headers["Content-Length"])\n\
41+
post_data = self.rfile.read(content_length)\n\
42+
response_data = self.server.xds_server.handle_lds_request(post_data)\n\
43+
self.send_response(200)\n\
44+
self.send_header("Content-type", "application/json")\n\
45+
self.end_headers()\n\
46+
self.wfile.write(response_data.encode())\n\
47+
elif self.path == "/add_listener":\n\
48+
content_length = int(self.headers["Content-Length"])\n\
49+
post_data = self.rfile.read(content_length)\n\
50+
data = json.loads(post_data.decode())\n\
51+
self.server.xds_server.add_listener(data["name"], data["config"])\n\
52+
self.send_response(200)\n\
53+
self.send_header("Content-type", "application/json")\n\
54+
self.end_headers()\n\
55+
self.wfile.write(json.dumps({"status": "success"}).encode())\n\
56+
elif self.path == "/remove_listener":\n\
57+
content_length = int(self.headers["Content-Length"])\n\
58+
post_data = self.rfile.read(content_length)\n\
59+
data = json.loads(post_data.decode())\n\
60+
success = self.server.xds_server.remove_listener(data["name"])\n\
61+
if success:\n\
62+
self.send_response(200)\n\
63+
self.send_header("Content-type", "application/json")\n\
64+
self.end_headers()\n\
65+
self.wfile.write(json.dumps({"status": "success"}).encode())\n\
66+
else:\n\
67+
self.send_response(404)\n\
68+
self.send_header("Content-type", "application/json")\n\
69+
self.end_headers()\n\
70+
self.wfile.write(json.dumps({"status": "not_found"}).encode())\n\
71+
elif self.path == "/state":\n\
72+
state = self.server.xds_server.get_state()\n\
73+
self.send_response(200)\n\
74+
self.send_header("Content-type", "application/json")\n\
75+
self.end_headers()\n\
76+
self.wfile.write(json.dumps(state).encode())\n\
77+
else:\n\
78+
self.send_response(404)\n\
79+
self.end_headers()\n\
80+
\n\
81+
def log_message(self, format, *args):\n\
82+
pass\n\
83+
\n\
84+
class XDSServer(socketserver.TCPServer):\n\
85+
def __init__(self, server_address, RequestHandlerClass, xds_server):\n\
86+
self.xds_server = xds_server\n\
87+
super().__init__(server_address, RequestHandlerClass)\n\
88+
\n\
89+
self.server = XDSServer(("0.0.0.0", port), XDSHandler, self)\n\
90+
self.server_thread = threading.Thread(target=self.server.serve_forever)\n\
91+
self.server_thread.daemon = True\n\
92+
self.server_thread.start()\n\
93+
logger.info(f"xDS server started on port {port}")\n\
94+
\n\
95+
def handle_lds_request(self, request_data):\n\
96+
with self._lock:\n\
97+
response = {\n\
98+
"version_info": str(self.version),\n\
99+
"resources": [],\n\
100+
"type_url": "type.googleapis.com/envoy.config.listener.v3.Listener"\n\
101+
}\n\
102+
for listener_name, listener_config in self.listeners.items():\n\
103+
# Wrap the listener config in a proper Any message\n\
104+
wrapped_config = {\n\
105+
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",\n\
106+
**listener_config\n\
107+
}\n\
108+
response["resources"].append(wrapped_config)\n\
109+
return json.dumps(response)\n\
110+
\n\
111+
def add_listener(self, listener_name, listener_config):\n\
112+
with self._lock:\n\
113+
self.listeners[listener_name] = listener_config\n\
114+
self.version += 1\n\
115+
logger.info(f"Added listener {listener_name}, version {self.version}")\n\
116+
\n\
117+
def remove_listener(self, listener_name):\n\
118+
with self._lock:\n\
119+
if listener_name in self.listeners:\n\
120+
del self.listeners[listener_name]\n\
121+
self.version += 1\n\
122+
logger.info(f"Removed listener {listener_name}, version {self.version}")\n\
123+
return True\n\
124+
return False\n\
125+
\n\
126+
def get_state(self):\n\
127+
with self._lock:\n\
128+
return {\n\
129+
"version": self.version,\n\
130+
"listeners": list(self.listeners.keys())\n\
131+
}\n\
132+
\n\
133+
if __name__ == "__main__":\n\
134+
xds_server = XDSServer()\n\
135+
xds_server.start(18000)\n\
136+
try:\n\
137+
while True:\n\
138+
time.sleep(1)\n\
139+
except KeyboardInterrupt:\n\
140+
print("Shutting down xDS server...")\n\
141+
' > /app/xds_server.py
142+
143+
# Make the script executable
144+
RUN chmod +x /app/xds_server.py
145+
146+
# Expose the xDS server port
147+
EXPOSE 18000
148+
149+
# Run the xDS server
150+
CMD ["python3", "/app/xds_server.py"]

‎reverse_tunnel/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
To learn about this sandbox and for instructions on how to run it please head over
2+
to the [Envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/reverse_tunnel.html).

‎reverse_tunnel/docker-compose.yaml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
version: '2'
2+
services:
3+
4+
downstream-envoy:
5+
image: debug/envoy:latest
6+
volumes:
7+
- ./initiator-envoy.yaml:/etc/downstream-envoy.yaml
8+
command: envoy -c /etc/downstream-envoy.yaml --concurrency 1 -l trace --drain-time-s 3
9+
ports:
10+
# Admin interface
11+
- "8888:8888"
12+
# Reverse connection API listener
13+
- "9000:9000"
14+
# Ingress HTTP listener
15+
- "6060:6060"
16+
extra_hosts:
17+
- "host.docker.internal:host-gateway"
18+
networks:
19+
- envoy-network
20+
depends_on:
21+
- downstream-service
22+
23+
downstream-service:
24+
image: nginxdemos/hello:plain-text
25+
networks:
26+
- envoy-network
27+
28+
upstream-envoy:
29+
image: debug/envoy:latest
30+
volumes:
31+
- ./responder-envoy.yaml:/etc/upstream-envoy.yaml
32+
command: envoy -c /etc/upstream-envoy.yaml --concurrency 1 -l trace --drain-time-s 3
33+
ports:
34+
# Admin interface
35+
- "8889:8888"
36+
# Reverse connection API listener
37+
- "9001:9000"
38+
# Egress listener
39+
- "8085:8085"
40+
networks:
41+
- envoy-network
42+
43+
networks:
44+
envoy-network:
45+
driver: bridge

‎reverse_tunnel/example.rst‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
.. _install_sandboxes_reverse_tunnel:
2+
3+
Reverse Tunnels
4+
===============
5+
6+
.. sidebar:: Requirements
7+
8+
.. include:: _include/docker-env-setup-link.rst
9+
10+
:ref:`curl <start_sandboxes_setup_curl>`
11+
Used to make HTTP requests.
12+
13+
This sandbox demonstrates Envoy's :ref:`reverse tunnels <config_reverse_connection>` feature, which allows establishing long-lived connections from downstream to upstream in scenarios where direct connectivity from upstream to downstream is not possible, and using these cached connection sockets to send data traffic from upstream to downstream Envoy.
14+
15+
In this example, a downstream Envoy proxy initiates reverse tunnels to an upstream Envoy using a custom address resolver. The configuration includes bootstrap extensions and reverse connection listeners with specialized address formats.
16+
17+
Step 1: Build Envoy with reverse tunnels feature
18+
************************************************
19+
20+
Build Envoy with the reverse tunnels feature enabled:
21+
22+
.. code-block:: console
23+
24+
$ ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.release.server_only'
25+
26+
Step 2: Build Envoy Docker image
27+
********************************
28+
29+
Build the Envoy Docker image:
30+
31+
.. code-block:: console
32+
33+
$ docker build -f ci/Dockerfile-envoy-image -t envoy:latest .
34+
35+
Step 3: Understanding the configuration
36+
**************************************
37+
38+
The reverse tunnel configuration is explained in the :ref:`Reverse Tunnels <config_reverse_connection>` section.
39+
40+
Step 4: Launch test containers
41+
******************************
42+
43+
Change to the ``reverse_tunnel`` directory and bring up the docker composition.
44+
45+
.. code-block:: console
46+
47+
$ pwd
48+
examples/reverse_tunnel
49+
$ docker compose up
50+
51+
.. note::
52+
The docker-compose maps the following ports:
53+
54+
- **downstream-envoy**: Host port 9000 → Container port 9000 (reverse connection API)
55+
- **upstream-envoy**: Host port 9001 → Container port 9000 (reverse connection API)
56+
57+
Verify the containers are running:
58+
59+
.. code-block:: console
60+
61+
$ docker ps
62+
63+
Expected output showing all containers are up:
64+
65+
.. code-block:: console
66+
67+
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
68+
ae15eab504f8 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 0.0.0.0:6060->6060/tcp, :::6060->6060/tcp, 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp, 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp, 10000/tcp reverse_tunnel-downstream-envoy-1
69+
58eba3678f20 nginxdemos/hello:plain-text "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 80/tcp reverse_tunnel-downstream-service-1
70+
49145cc8a9d1 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 0.0.0.0:8085->8085/tcp, :::8085->8085/tcp, 10000/tcp, 0.0.0.0:8889->8888/tcp, :::8889->8888/tcp, 0.0.0.0:9001->9000/tcp, :::9001->9000/tcp reverse_tunnel-upstream-envoy-1
71+
72+
Step 5: Validate reverse tunnel establishment
73+
*********************************************
74+
75+
Verify that reverse tunnels have been successfully established by checking the stats counters on both downstream and upstream Envoy proxies.
76+
77+
Check downstream Envoy stats (port 8888):
78+
79+
.. code-block:: console
80+
81+
$ curl http://localhost:8888/stats | grep reverse_connection
82+
83+
Expected downstream stats showing connected reverse tunnels:
84+
85+
.. code-block:: console
86+
87+
downstream_reverse_connection.cluster.upstream-cluster.connected: 1
88+
downstream_reverse_connection.cluster.upstream-cluster.connecting: 0
89+
downstream_reverse_connection.host.172.27.0.2:9000.connected: 1
90+
downstream_reverse_connection.host.172.27.0.2:9000.connecting: 0
91+
downstream_reverse_connection.worker_0.cluster.upstream-cluster.connected: 1
92+
downstream_reverse_connection.worker_0.cluster.upstream-cluster.connecting: 0
93+
downstream_reverse_connection.worker_0.host.172.27.0.2:9000.connected: 1
94+
downstream_reverse_connection.worker_0.host.172.27.0.2:9000.connecting: 0
95+
96+
Check upstream Envoy stats (port 8889):
97+
98+
.. code-block:: console
99+
100+
$ curl http://localhost:8889/stats | grep reverse_connections
101+
102+
Expected upstream stats showing received reverse connections:
103+
104+
.. code-block:: console
105+
106+
reverse_connections.clusters.downstream-cluster: 1
107+
reverse_connections.nodes.downstream-node: 1
108+
reverse_connections.worker_0.cluster.downstream-cluster: 1
109+
reverse_connections.worker_0.node.downstream-node: 1
110+
111+
The stats confirm that:
112+
113+
- **Downstream Envoy**: Has successfully connected (``connected: 1``) to the upstream cluster with no pending connections (``connecting: 0``)
114+
- **Upstream Envoy**: Has received reverse connections from the downstream node and cluster, as indicated by the reverse connection counters
115+
116+
Step 6: Test reverse tunnel
117+
***************************
118+
119+
Perform an HTTP request for the service behind downstream Envoy, to upstream Envoy. This request will be sent over a reverse tunnel.
120+
121+
.. code-block:: console
122+
123+
$ curl -H "x-remote-node-id: downstream-node" -H "x-dst-cluster-uuid: downstream-cluster" http://localhost:8085/downstream_service -v
124+
125+
Expected response:
126+
127+
.. code-block:: console
128+
129+
* Trying ::1...
130+
* TCP_NODELAY set
131+
* Connected to localhost (::1) port 8085 (#0)
132+
> GET /downstream_service HTTP/1.1
133+
> Host: localhost:8085
134+
> User-Agent: curl/7.61.1
135+
> Accept: */*
136+
> x-remote-node-id: downstream-node
137+
> x-dst-cluster-uuid: downstream-cluster
138+
>
139+
< HTTP/1.1 200 OK
140+
< server: envoy
141+
< date: Thu, 25 Sep 2025 21:25:38 GMT
142+
< content-type: text/plain
143+
< content-length: 159
144+
< expires: Thu, 25 Sep 2025 21:25:37 GMT
145+
< cache-control: no-cache
146+
< x-envoy-upstream-service-time: 13
147+
<
148+
Server address: 172.27.0.3:80
149+
Server name: b490f264caf9
150+
Date: 25/Sep/2025:21:25:38 +0000
151+
URI: /downstream_service
152+
Request ID: 41807e3cd1f6a0b601597b80f7e51513
153+
* Connection #0 to host localhost left intact
154+
155+
.. seealso::
156+
157+
:ref:`Reverse Tunnels architecture overview <config_reverse_connection>`
158+
Learn more about Envoy's reverse tunnel functionality.

0 commit comments

Comments
 (0)