Work can also be TcpServerConnection, not just TcpClientConnection (#906)

* Work can also be TcpServerConnection, not just TcpClientConnection.  More over, it can be any generic work type

* Add py_class_role and py_obj

* Port internal integration tests into public repo

* Fix proxy py addr

* Use cross-platform compat shasum

Author: abhinavsingh

docs/conf.py

@@ -293,6 +293,7 @@
     (_py_class_role, 'proxy.core.pool.AcceptorPool'),
     (_py_class_role, 'proxy.core.executors.ThreadlessPool'),
     (_py_class_role, 'proxy.core.acceptor.threadless.T'),
+    (_py_class_role, 'proxy.core.acceptor.work.T'),
     (_py_class_role, 'queue.Queue[Any]'),
     (_py_class_role, 'TcpClientConnection'),
     (_py_class_role, 'TcpServerConnection'),
@@ -303,4 +304,5 @@
     (_py_class_role, 'WebsocketFrame'),
     (_py_class_role, 'Work'),
     (_py_obj_role, 'proxy.core.acceptor.threadless.T'),
+    (_py_obj_role, 'proxy.core.acceptor.work.T'),
 ]

examples/web_scraper.py

@@ -14,10 +14,11 @@
 
 from proxy import Proxy
 from proxy.core.acceptor import Work
+from proxy.core.connection import TcpClientConnection
 from proxy.common.types import Readables, Writables
 
 
-class WebScraper(Work):
+class WebScraper(Work[TcpClientConnection]):
     """Demonstrates how to orchestrate a generic work acceptors and executors
     workflow using proxy.py core.
 

proxy/core/acceptor/executors.py

@@ -125,7 +125,7 @@ def start_threaded_work(
             addr: Optional[Tuple[str, int]],
             event_queue: Optional[EventQueue] = None,
             publisher_id: Optional[str] = None,
-    ) -> Tuple[Work, threading.Thread]:
+    ) -> Tuple[Work[TcpClientConnection], threading.Thread]:
         """Utility method to start a work in a new thread."""
         work = flags.work_klass(
             TcpClientConnection(conn, addr),

proxy/core/acceptor/threadless.py

@@ -18,7 +18,7 @@
 import multiprocessing
 
 from abc import abstractmethod, ABC
-from typing import Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union
+from typing import Any, Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union
 
 from ...common.logger import Logger
 from ...common.types import Readables, Writables
@@ -71,7 +71,7 @@ def __init__(
         self.event_queue = event_queue
 
         self.running = multiprocessing.Event()
-        self.works: Dict[int, Work] = {}
+        self.works: Dict[int, Work[Any]] = {}
         self.selector: Optional[selectors.DefaultSelector] = None
         # If we remove single quotes for typing hint below,
         # runtime exceptions will occur for < Python 3.9.

proxy/core/acceptor/work.py

@@ -16,19 +16,20 @@
 
 from abc import ABC, abstractmethod
 from uuid import uuid4
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, TypeVar, Generic
 
 from ..event import eventNames, EventQueue
-from ..connection import TcpClientConnection
 from ...common.types import Readables, Writables
 
+T = TypeVar('T')
 
-class Work(ABC):
+
+class Work(ABC, Generic[T]):
     """Implement Work to hook into the event loop provided by Threadless process."""
 
     def __init__(
             self,
-            work: TcpClientConnection,
+            work: T,
             flags: argparse.Namespace,
             event_queue: Optional[EventQueue] = None,
             uid: Optional[str] = None,

proxy/core/base/tcp_server.py

@@ -19,12 +19,13 @@
 from typing import Dict, Any, Optional
 
 from ...core.acceptor import Work
+from ...core.connection import TcpClientConnection
 from ...common.types import Readables, Writables
 
 logger = logging.getLogger(__name__)
 
 
-class BaseTcpServerHandler(Work):
+class BaseTcpServerHandler(Work[TcpClientConnection]):
     """BaseTcpServerHandler implements Work interface.
 
     BaseTcpServerHandler lifecycle is controlled by Threadless core

tests/integration/test_integration.sh

@@ -15,6 +15,8 @@ if [[ -z "$PROXY_PY_PORT" ]]; then
   exit 1
 fi
 
+PROXY_URL="127.0.0.1:$PROXY_PY_PORT"
+
 # Wait for server to come up
 WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '"
 while true; do
@@ -31,44 +33,89 @@ while true; do
     curl -v \
         --max-time 1 \
         --connect-timeout 1 \
-        -x 127.0.0.1:$PROXY_PY_PORT \
-        http://127.0.0.1:$PROXY_PY_PORT/ 2>/dev/null
+        -x $PROXY_URL \
+        http://$PROXY_URL/ 2>/dev/null
     if [[ $? == 0 ]]; then
         break
     fi
     echo "Waiting for web server to start accepting requests..."
     sleep 1
 done
 
+verify_response() {
+    if [ "$1" == "" ];
+    then
+        echo "Empty response";
+        return 1;
+    else
+        if [ "$1" == "$2" ];
+        then
+            echo "Ok";
+            return 0;
+        else
+            echo "Invalid response: '$1', expected: '$2'";
+            return 1;
+        fi
+    fi;
+}
+
 # Check if proxy was started with integration
 # testing web server plugin.  If detected, use
 # internal web server for integration testing.
 
 # If integration testing plugin is not found,
 # detect if we have internet access.  If we do,
 # then use httpbin.org for integration testing.
-curl -v \
-    -x 127.0.0.1:$PROXY_PY_PORT \
-    http://httpbin.org/get
-if [[ $? != 0 ]]; then
-    echo "http request failed"
-    exit 1
-fi
+read -r -d '' ROBOTS_RESPONSE << EOM
+User-agent: *
+Disallow: /deny
+EOM
 
-curl -v \
-    -x 127.0.0.1:$PROXY_PY_PORT \
-    https://httpbin.org/get
-if [[ $? != 0 ]]; then
-    echo "https request failed"
-    exit 1
-fi
+echo "[Test HTTP Request via Proxy]"
+CMD="curl -v -x $PROXY_URL http://httpbin.org/robots.txt"
+RESPONSE=$($CMD 2> /dev/null)
+verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
+VERIFIED1=$?
+
+echo "[Test HTTPS Request via Proxy]"
+CMD="curl -v -x $PROXY_URL https://httpbin.org/robots.txt"
+RESPONSE=$($CMD 2> /dev/null)
+verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
+VERIFIED2=$?
 
+echo "[Test Internal Web Server via Proxy]"
 curl -v \
-    -x 127.0.0.1:$PROXY_PY_PORT \
-    http://127.0.0.1:$PROXY_PY_PORT/
-if [[ $? != 0 ]]; then
-    echo "http request to built in webserver failed"
-    exit 1
+    -x $PROXY_URL \
+    http://$PROXY_URL/
+VERIFIED3=$?
+
+SHASUM=sha256sum
+if [ "$(uname)" = "Darwin" ];
+then
+    SHASUM="shasum -a 256"
 fi
 
-exit 0
+echo "[Test Download File Hash Verifies 1]"
+touch downloaded.hash
+echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc  -" > downloaded.hash
+curl -vL \
+    -o downloaded.whl \
+    -x $PROXY_URL \
+    https://files.pythonhosted.org/packages/88/78/e642316313b1cd6396e4b85471a316e003eff968f29773e95ea191ea1d08/proxy.py-2.4.0rc4-py3-none-any.whl#sha256=3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc
+cat downloaded.whl | $SHASUM -c downloaded.hash
+VERIFIED4=$?
+rm downloaded.whl downloaded.hash
+
+echo "[Test Download File Hash Verifies 2]"
+touch downloaded.hash
+echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8  -" > downloaded.hash
+curl -vL \
+    -o downloaded.whl \
+    -x $PROXY_URL \
+    https://files.pythonhosted.org/packages/20/9a/e5d9ec41927401e41aea8af6d16e78b5e612bca4699d417f646a9610a076/Jinja2-3.0.3-py3-none-any.whl#sha256=077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8
+cat downloaded.whl | $SHASUM -c downloaded.hash
+VERIFIED5=$?
+rm downloaded.whl downloaded.hash
+
+EXIT_CODE=$(( $VERIFIED1 || $VERIFIED2 || $VERIFIED3 || $VERIFIED4 || $VERIFIED5 ))
+exit $EXIT_CODE