Skip to content

Commit e26dc96

Browse files
ShaneHarveyShaneHarvey
committed
PYTHON-2034 Support MONGODB-AWS authentication mechanism
Use botocore to perform the manual Signature Version 4 Signing Process. Test MONGODB-AWS in Evergreen. Properly unquote URI option values in authMechanismProperties and readPreferenceTags.
1 parent a43e73d commit e26dc96

19 files changed

+860
-28
lines changed

‎.evergreen/config.yml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,172 @@ functions:
429429
# DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
430430
PYTHON_BINARY=${PYTHON_BINARY} ATLAS_REPL='${atlas_repl}' ATLAS_SHRD='${atlas_shrd}' ATLAS_FREE='${atlas_free}' ATLAS_TLS11='${atlas_tls11}' ATLAS_TLS12='${atlas_tls12}' sh ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh
431431
432+
"add aws auth variables to file":
433+
- command: shell.exec
434+
type: test
435+
params:
436+
working_dir: "src"
437+
silent: true
438+
script: |
439+
cat <<EOF > ${DRIVERS_TOOLS}/.evergreen/auth_aws/aws_e2e_setup.json
440+
{
441+
"iam_auth_ecs_account" : "${iam_auth_ecs_account}",
442+
"iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}",
443+
"iam_auth_ecs_account_arn": "arn:aws:iam::557821124784:user/authtest_fargate_user",
444+
"iam_auth_ecs_cluster": "${iam_auth_ecs_cluster}",
445+
"iam_auth_ecs_task_definition": "${iam_auth_ecs_task_definition}",
446+
"iam_auth_ecs_subnet_a": "${iam_auth_ecs_subnet_a}",
447+
"iam_auth_ecs_subnet_b": "${iam_auth_ecs_subnet_b}",
448+
"iam_auth_ecs_security_group": "${iam_auth_ecs_security_group}",
449+
450+
"iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}",
451+
"iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}",
452+
"iam_auth_assume_role_name" : "${iam_auth_assume_role_name}",
453+
454+
"iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}",
455+
"iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}",
456+
"iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}"
457+
}
458+
EOF
459+
460+
"run aws auth test with regular aws credentials":
461+
- command: shell.exec
462+
type: test
463+
params:
464+
working_dir: "src"
465+
script: |
466+
${PREPARE_SHELL}
467+
cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
468+
mongo aws_e2e_regular_aws.js
469+
- command: shell.exec
470+
type: test
471+
params:
472+
working_dir: "src"
473+
silent: true
474+
script: |
475+
cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
476+
alias urlencode='python -c "import sys, urllib as ul; print ul.quote_plus(sys.argv[1])"'
477+
USER=$(urlencode ${iam_auth_ecs_account})
478+
PASS=$(urlencode ${iam_auth_ecs_secret_access_key})
479+
MONGODB_URI="mongodb://$USER:$PASS@localhost"
480+
EOF
481+
- command: shell.exec
482+
type: test
483+
params:
484+
working_dir: "src"
485+
script: |
486+
${PREPARE_SHELL}
487+
.evergreen/run-mongodb-aws-test.sh
488+
489+
"run aws auth test with assume role credentials":
490+
- command: shell.exec
491+
type: test
492+
params:
493+
working_dir: "src"
494+
script: |
495+
${PREPARE_SHELL}
496+
cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
497+
mongo aws_e2e_assume_role.js
498+
- command: shell.exec
499+
type: test
500+
params:
501+
working_dir: "src"
502+
silent: true
503+
script: |
504+
# DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
505+
cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
506+
alias urlencode='python -c "import sys, urllib as ul; print ul.quote_plus(sys.argv[1])"'
507+
USER=$(jq -r '.AccessKeyId' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
508+
USER=$(urlencode $USER)
509+
PASS=$(jq -r '.SecretAccessKey' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
510+
PASS=$(urlencode $PASS)
511+
SESSION_TOKEN=$(jq -r '.SessionToken' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
512+
SESSION_TOKEN=$(urlencode $SESSION_TOKEN)
513+
MONGODB_URI="mongodb://$USER:$PASS@localhost"
514+
EOF
515+
- command: shell.exec
516+
type: test
517+
params:
518+
working_dir: "src"
519+
script: |
520+
${PREPARE_SHELL}
521+
.evergreen/run-mongodb-aws-test.sh
522+
523+
"run aws auth test with aws EC2 credentials":
524+
- command: shell.exec
525+
type: test
526+
params:
527+
working_dir: "src"
528+
script: |
529+
${PREPARE_SHELL}
530+
cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
531+
mongo aws_e2e_ec2.js
532+
- command: shell.exec
533+
type: test
534+
params:
535+
working_dir: "src"
536+
script: |
537+
${PREPARE_SHELL}
538+
.evergreen/run-mongodb-aws-test.sh
539+
540+
"run aws auth test with aws credentials as environment variables":
541+
- command: shell.exec
542+
type: test
543+
params:
544+
working_dir: "src"
545+
silent: true
546+
script: |
547+
# DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
548+
cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
549+
export AWS_ACCESS_KEY_ID=${iam_auth_ecs_account}
550+
export AWS_SECRET_ACCESS_KEY=${iam_auth_ecs_secret_access_key}
551+
EOF
552+
- command: shell.exec
553+
type: test
554+
params:
555+
working_dir: "src"
556+
script: |
557+
${PREPARE_SHELL}
558+
PROJECT_DIRECTORY=${PROJECT_DIRECTORY} .evergreen/run-mongodb-aws-test.sh
559+
560+
"run aws auth test with aws credentials and session token as environment variables":
561+
- command: shell.exec
562+
type: test
563+
params:
564+
working_dir: "src"
565+
silent: true
566+
script: |
567+
# DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
568+
cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
569+
export AWS_ACCESS_KEY_ID=$(jq -r '.AccessKeyId' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
570+
export AWS_SECRET_ACCESS_KEY=$(jq -r '.SecretAccessKey' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
571+
export AWS_SESSION_TOKEN=$(jq -r '.SessionToken' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
572+
EOF
573+
- command: shell.exec
574+
type: test
575+
params:
576+
working_dir: "src"
577+
script: |
578+
${PREPARE_SHELL}
579+
.evergreen/run-mongodb-aws-test.sh
580+
581+
"run aws ECS auth test":
582+
- command: shell.exec
583+
type: test
584+
params:
585+
working_dir: "src"
586+
script: |
587+
${PREPARE_SHELL}
588+
cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
589+
590+
cat <<EOF > setup.js
591+
const mongo_binaries = "$MONGODB_BINARIES";
592+
const project_dir = "$PROJECT_DIRECTORY";
593+
EOF
594+
595+
mongo --nodb setup.js aws_e2e_ecs.js
596+
cd -
597+
432598
"cleanup":
433599
- command: shell.exec
434600
params:
@@ -968,6 +1134,22 @@ tasks:
9681134
vars:
9691135
OCSP_TLS_SHOULD_SUCCEED: "0"
9701136

1137+
- name: "aws-auth-test"
1138+
commands:
1139+
- func: "bootstrap mongo-orchestration"
1140+
vars:
1141+
AUTH: "auth"
1142+
# TODO: SSL??
1143+
ORCHESTRATION_FILE: "auth-aws.json"
1144+
TOPOLOGY: "server"
1145+
- func: "add aws auth variables to file"
1146+
- func: "run aws auth test with regular aws credentials"
1147+
- func: "run aws auth test with assume role credentials"
1148+
- func: "run aws auth test with aws credentials as environment variables"
1149+
- func: "run aws auth test with aws credentials and session token as environment variables"
1150+
- func: "run aws auth test with aws EC2 credentials"
1151+
- func: "run aws ECS auth test"
1152+
9711153
# }}}
9721154
- name: "coverage-report"
9731155
tags: ["coverage"]
@@ -1064,6 +1246,10 @@ axes:
10641246
batchtime: 10080 # 7 days
10651247
variables:
10661248
libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/ubuntu1604/master/latest/libmongocrypt.tar.gz
1249+
- id: ubuntu-18.04
1250+
display_name: "Ubuntu 18.04"
1251+
run_on: ubuntu1804-test
1252+
batchtime: 10080 # 7 days
10671253
- id: ubuntu1604-arm64-small
10681254
display_name: "Ubuntu 16.04 (ARM64)"
10691255
run_on: ubuntu1604-arm64-small
@@ -1920,6 +2106,14 @@ buildvariants:
19202106
tasks:
19212107
- name: ".ocsp"
19222108

2109+
- matrix_name: "aws-auth-test"
2110+
matrix_spec:
2111+
platform: ubuntu-18.04
2112+
display_name: "MONGODB-AWS Auth test"
2113+
run_on: ubuntu1804-test
2114+
tasks:
2115+
- name: "aws-auth-test"
2116+
19232117
# Platform notes
19242118
# i386 builds of OpenSSL or Cyrus SASL are not available
19252119
# Ubuntu16.04 ppc64le is only supported by MongoDB 3.4+

‎.evergreen/run-mongodb-aws-ecs-test.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
3+
# Don't trace since the URI contains a password that shouldn't show up in the logs
4+
set -o errexit # Exit the script with error if any of the commands fail
5+
6+
############################################
7+
# Main Program #
8+
############################################
9+
10+
if [[ -z "$1" ]]; then
11+
echo "usage: $0 <MONGODB_URI>"
12+
exit 1
13+
fi
14+
export MONGODB_URI="$1"
15+
16+
if echo "$MONGODB_URI" | grep -q "@"; then
17+
echo "MONGODB_URI unexpectedly contains user credentials in ECS test!";
18+
exit 1
19+
fi
20+
# Now we can safely enable xtrace
21+
set -o xtrace
22+
23+
if command -v virtualenv ; then
24+
VIRTUALENV=$(command -v virtualenv)
25+
else
26+
echo "Installing virtualenv..."
27+
apt install python3-pip -y
28+
pip3 install --user virtualenv
29+
VIRTUALENV='python3 -m virtualenv'
30+
fi
31+
32+
authtest () {
33+
echo "Running MONGODB-AWS ECS authentication tests with $PYTHON"
34+
$PYTHON --version
35+
36+
$VIRTUALENV -p $PYTHON --system-site-packages --never-download venvaws
37+
. venvaws/bin/activate
38+
pip install requests botocore
39+
40+
cd src
41+
python test/auth_aws/test_auth_aws.py
42+
cd -
43+
deactivate
44+
rm -rf venvaws
45+
}
46+
47+
PYTHON=$(command -v python) authtest
48+
PYTHON=$(command -v python3) authtest

‎.evergreen/run-mongodb-aws-test.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
3+
set -o xtrace
4+
set -o errexit # Exit the script with error if any of the commands fail
5+
6+
############################################
7+
# Main Program #
8+
############################################
9+
10+
echo "Running MONGODB-AWS authentication tests"
11+
# ensure no secrets are printed in log files
12+
set +x
13+
14+
# load the script
15+
shopt -s expand_aliases # needed for `urlencode` alias
16+
[ -s "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" ] && source "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
17+
18+
MONGODB_URI=${MONGODB_URI:-"mongodb://localhost"}
19+
MONGODB_URI="${MONGODB_URI}/aws?authMechanism=MONGODB-AWS"
20+
if [[ -n ${SESSION_TOKEN} ]]; then
21+
MONGODB_URI="${MONGODB_URI}&authMechanismProperties=AWS_SESSION_TOKEN:${SESSION_TOKEN}"
22+
fi
23+
24+
export MONGODB_URI="$MONGODB_URI"
25+
26+
# show test output
27+
set -x
28+
29+
VIRTUALENV=$(command -v virtualenv)
30+
31+
authtest () {
32+
echo "Running MONGODB-AWS authentication tests with $PYTHON"
33+
$PYTHON --version
34+
35+
$VIRTUALENV -p $PYTHON --system-site-packages --never-download venvaws
36+
. venvaws/bin/activate
37+
pip install requests botocore
38+
39+
python test/auth_aws/test_auth_aws.py
40+
deactivate
41+
rm -rf venvaws
42+
}
43+
44+
PYTHON=$(command -v python) authtest
45+
PYTHON=$(command -v python3) authtest

‎README.rst‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,12 @@ dependency can be installed automatically along with PyMongo::
9999

100100
$ python -m pip install pymongo[gssapi]
101101

102+
MONGODB-AWS authentication requires `botocore
103+
<https://pypi.org/project/botocore/>`_ and `requests
104+
<https://pypi.org/project/requests/>`_::
105+
106+
$ python -m pip install pymongo[aws]
107+
102108
Support for mongodb+srv:// URIs requires `dnspython
103109
<https://pypi.python.org/pypi/dnspython>`_::
104110

@@ -116,7 +122,7 @@ PyMongo::
116122
.. note:: Users of Python versions older than 2.7.9 will also
117123
receive the dependencies for OCSP when using the tls extra.
118124

119-
:ref:`OCSP` requires `PyOpenSSL
125+
OCSP (Online Certificate Status Protocol) requires `PyOpenSSL
120126
<https://pypi.org/project/pyOpenSSL/>`_, `requests
121127
<https://pypi.org/project/requests/>`_ and `service_identity
122128
<https://pypi.org/project/service_identity/>`_::
@@ -133,10 +139,15 @@ Wire protocol compression with zstandard requires `zstandard
133139

134140
$ python -m pip install pymongo[zstd]
135141

142+
Client-Side Field Level Encryption requires `pymongocrypt
143+
<https://pypi.org/project/pymongocrypt/>`_::
144+
145+
$ python -m pip install pymongo[encryption]
146+
136147
You can install all dependencies automatically with the following
137148
command::
138149

139-
$ python -m pip install pymongo[gssapi,ocsp,snappy,srv,tls,zstd]
150+
$ python -m pip install pymongo[gssapi,aws,ocsp,snappy,srv,tls,zstd,encryption]
140151

141152
Other optional packages:
142153

‎doc/changelog.rst‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,17 @@ Changes in Version 3.11.0
66

77
Version 3.11 adds support for MongoDB 4.4. Highlights include:
88

9-
- Added the ``allow_disk_use`` parameters to
10-
:meth:`pymongo.collection.Collection.find`.
119
- Support for :ref:`OCSP` (Online Certificate Status Protocol)
1210
- Support for `PyOpenSSL <https://pypi.org/project/pyOpenSSL/>`_ as an
1311
alternative TLS implementation. PyOpenSSL is required for :ref:`OCSP`
1412
support. It will also be installed when using the "tls" extra if the
1513
version of Python in use is older than 2.7.9.
14+
- Support for the :ref:`MONGODB-AWS` authentication mechanism.
1615
- Added the ``background`` parameter to
1716
:meth:`pymongo.database.Database.validate_collection`. For a description
1817
of this parameter see the MongoDB documentation for the `validate command`_.
18+
- Added the ``allow_disk_use`` parameters to
19+
:meth:`pymongo.collection.Collection.find`.
1920

2021
.. _validate command: https://docs.mongodb.com/manual/reference/command/validate/
2122

0 commit comments

Comments
 (0)