feat: Remove requests and all its dependencies. (#547)

Author: purple4reina

Dockerfile

@@ -22,8 +22,8 @@ RUN rm -rf ./python/lib/$runtime/site-packages/botocore*
 RUN rm -rf ./python/lib/$runtime/site-packages/setuptools
 RUN rm -rf ./python/lib/$runtime/site-packages/jsonschema/tests
 RUN find . -name 'libddwaf.so' -delete
-RUN rm -rf ./python/lib/$runtime/site-packages/urllib3*
-RUN rm ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_taint_tracking/*.so
+# Comment this line out for now since ddtrace now tries to import it
+#RUN rm ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_taint_tracking/*.so
 RUN rm ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_stacktrace*.so
 RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/libdd_wrapper*.so
 RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/ddup/_ddup.*.so
@@ -32,6 +32,14 @@ RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/st
 RUN find ./python/lib/$runtime/site-packages/*.dist-info -not -name "entry_points.txt" -type f -delete
 RUN find ./python/lib/$runtime/site-packages -type d -empty -delete
 
+# Remove requests and dependencies
+RUN rm -rf \
+        ./python/lib/$runtime/site-packages/requests* \
+        ./python/lib/$runtime/site-packages/urllib3* \
+        ./python/lib/$runtime/site-packages/certifi* \
+        ./python/lib/$runtime/site-packages/idna* \
+        ./python/lib/$runtime/site-packages/charset_normalizer*
+
 # Precompile all .pyc files and remove .py files. This speeds up load time.
 # Compile with optimization level 2 (-OO) and PYTHONNODEBUGRANGES=1 to redtce
 # size of .pyc files.

ci/input_files/build.yaml.tpl

@@ -98,6 +98,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
     - *install-node
     - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
     - yarn global add serverless@^3.38.0 --prefix /usr/local
+    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
     - cd integration_tests && yarn install && cd ..
   script:
     - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh

poetry.lock

@@ -25,7 +25,7 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = ">=3.8.0,<4"
-datadog = ">=0.41.0,<1.0.0"
+datadog = ">=0.51.0,<1.0.0"
 wrapt = "^1.11.2"
 ddtrace = ">=2.17.0"
 ujson = ">=5.9.0"

pyproject.toml

@@ -1,5 +1,6 @@
 {
   "devDependencies": {
-    "serverless-plugin-datadog": "^2.18.0"
+    "serverless-plugin-datadog": "^2.18.0",
+    "serverless-python-requirements": "^6.1.1"
   }
 }

tests/integration/package.json

@@ -0,0 +1,4 @@
+certifi==2024.12.14
+charset-normalizer==3.4.1
+idna==3.10
+requests==2.32.3

tests/integration/requirements.txt

@@ -20,6 +20,21 @@ provider:
   iam:
     # IAM permissions require that all functions are deployed with this role
     role: "arn:aws:iam::425362996713:role/serverless-integration-test-lambda-role"
+  layers:
+    - { Ref: PythonLambdaLayer }
+    - { Ref: PythonRequirementsLambdaLayer }
+
+plugins:
+  - serverless-python-requirements
+
+custom:
+  pythonRequirements:
+    pythonBin: python3
+    pipCmdExtraArgs:
+      - --no-deps  # install just requests
+    layer:
+      compatibleRuntimes:
+        - ${env:SERVERLESS_RUNTIME}
 
 layers:
   python:
@@ -32,8 +47,6 @@ functions:
     name: integration-tests-python-${sls:stage}-async-metrics_${env:RUNTIME}
     handler: handle.handle
     runtime: ${env:SERVERLESS_RUNTIME}
-    layers:
-      - { Ref: PythonLambdaLayer }
     environment:
       DD_FLUSH_TO_LOG: true
 
@@ -42,5 +55,3 @@ functions:
     name: integration-tests-python-${sls:stage}-sync-metrics_${env:RUNTIME}
     handler: handle.handle
     runtime: ${env:SERVERLESS_RUNTIME}
-    layers:
-      - { Ref: PythonLambdaLayer }