Copy evergreen folder from master, excluding local-kerberos

Author: Emily Giurleo

.evergreen/.gitignore

@@ -1 +1 @@
-Dockerfile
+/Dockerfile

.evergreen/Dockerfile.erb

@@ -12,38 +12,25 @@ server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#
 server_archive_basename = File.basename(server_url)
 server_extracted_dir = server_archive_basename.sub(/\.(tar\.gz|tgz)$/, '')
 
-toolchain_upper='f11598d091441ffc8d746aacfdc6c26741a3e629' 
-toolchain_lower='5e46f2793e8e866f36eda2c5_20_02_14_19_18_18' 
+toolchain_upper='717e3e0a26debdc100fecee0d093e488ee7a0219'
 
-ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{toolchain_upper}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{toolchain_upper}_#{toolchain_lower}.tar.gz"
+ruby_toolchain_url = "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/ruby-toolchain-#{distro}-#{toolchain_upper}.tar.xz"
+#ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{toolchain_upper}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{toolchain_upper}_#{toolchain_lower}.tar.gz"
 
 %>
 
 FROM <%= base_image %>
 
-# increment to force apt-get update to run
-RUN echo 2
-
 <% if debian? %>
 
   ENV DEBIAN_FRONTEND=noninteractive
 
-  RUN apt-get update
-
-  RUN apt-get install -y curl
-
-  <% if preload? && distro =~ /ubuntu1204/ %>
-  
-    # ubuntu1204 comes with python 2.7.3.
-    # Install a more recent one from deadsnakes ppa so that pip works.
-    RUN apt-get install -y python-software-properties
-    # https://github.com/deadsnakes/issues/issues/53
-    RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5BB92C09DB82666C
-    RUN add-apt-repository ppa:fkrull/deadsnakes-python2.7
-    RUN apt-get update
-    RUN apt-get install -y python2.7-dev
-
-  <% end %>
+  # increment the counter to force apt-get update to run.
+  # zsh is not required for any scripts but it is a better interactive shell
+  # than bash.
+  RUN echo 2 && \
+    apt-get update && \
+    apt-get install -y curl zsh
 
 <% end %>
 
@@ -68,10 +55,12 @@ RUN echo 2
 
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
+  # Compiling pyhton packages: python2.7-dev
   # JRuby: openjdk-8-jre
-  # Server dependencies: libsnmp30 libcurl3
+  # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
   # Kerberos testing: krb5-user
+  # Local Kerberos server: krb5-kdc krb5-admin-server
   # Installing mlaunch from git: git
   # ruby-head archive: bzip2
   # nio4r on JRuby: libgmp-dev
@@ -81,71 +70,96 @@ RUN echo 2
   # distros have pip installed (but none as of this writing have pip3)
   # therefore install python-pip in all configurations here.
 
-  # ubuntu1204, ubuntu1404 only have openjdk-7-jre
-  <% unless %w(ubuntu1204 ubuntu1404).include?(distro) %>
+  # ubuntu1404 only has openjdk-7-jre
+  <% if distro !~ /ubuntu1404/ %>
     RUN apt-get install -y openjdk-8-jre
   <% end %>
-
-  <% if %w(ubuntu1204).include?(distro) %>
-    RUN apt-get install -y libsnmp15
+  
+  # ubuntu1404, ubuntu1604: libcurl3
+  # ubuntu1804: libcurl4
+  <% if distro =~ /ubuntu1804/ %>
+    RUN apt-get install -y libcurl4
   <% else %>
-    RUN apt-get install -y libsnmp30
+    RUN apt-get install -y libcurl3
   <% end %>
 
-  # ubuntu1204, ubuntu1404, ubuntu1604: libcurl3
-  RUN apt-get install -y libyaml-0-2 gcc make git lsb-release libcurl3 \
-    krb5-user bzip2 libgmp-dev python-pip
+  RUN apt-get install -y libsnmp30 libyaml-0-2 gcc make git lsb-release \
+    krb5-user krb5-kdc krb5-admin-server bzip2 libgmp-dev python-pip python2.7-dev
 
 <% else %>
 
+  # Enterprise server: net-snmp
   # lsb_release: redhat-lsb-core
   # our runner scripts: which
   # Ruby dependency: libyaml
   # compiling python packages: gcc python-devel
   # Kerberos tests: krb5-workstation + cyrus-sasl-devel to build the
   # mongo_kerberos gem + cyrus-sasl-gssapi for authentication to work
+  # Local Kerberos server: krb5-server
+  # JRuby: java-1.8.0-openjdk
   #
   # Note: lacking cyrus-sasl-gssapi produces a cryptic message
   # "SASL(-4): no mechanism available: No worthy mechs found"
   # https://github.com/farorm/python-ad/issues/10
 
-  RUN yum install -y redhat-lsb-core which git gcc python-devel libyaml \
-    krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi
+  RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \
+    krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \
+    net-snmp
+    
+  <% if distro =~ /rhel6/ %>
+    
+    # RHEL 6 ships with Python 2.6.
+    
+    RUN yum install -y centos-release-scl && \
+      yum install -y python27-python python27-python-devel
+    ENV PATH=/opt/rh/python27/root/usr/bin:$PATH \
+      LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
+  
+  <% else %>
+    
+    RUN yum install -y python-devel
+  
+  <% end %>
 
 <% end %>
 
 <% if preload? %>
 
   WORKDIR /app
 
-  RUN curl --retry 3 -fL <%= server_download_url %> -o <%= File.basename(server_download_url) %>
-  RUN tar xfz <%= File.basename(server_download_url) %>
-  RUN mv mongo*/ /opt/mongodb
+  RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
+    mv mongo*/ /opt/mongodb
   ENV USE_OPT_MONGODB=1
 
   <% unless ruby_head? %>
 
-    RUN curl --retry 3 -fL <%= ruby_toolchain_url %> -o ruby-toolchain.tar.gz
-    RUN tar -xC /opt -zf ruby-toolchain.tar.gz
-    ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH
+    RUN curl --retry 3 -fL <%= ruby_toolchain_url %> |tar -xC /opt -Jf -
+    ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
+      USE_OPT_TOOLCHAIN=1
     #ENV PATH=/opt/rubies/python/3/bin:$PATH
-    ENV USE_OPT_TOOLCHAIN=1
 
   <% end %>
 
-  <% if distro =~ /ubuntu1204|rhel/ %>
+  <% if distro =~ /rhel|ubuntu1604/ %>
 
     # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
+    #
+    # Ubuntu 16.04 apparently also ships a pip that does not work:
+    # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry
+    # Potentially this only affects environments with less than ideal
+    # connectivity (or, perhaps, when python package registry is experiencing
+    # availability issues) when pip must retry to install packages.
+    #
     # rhel apparently does not package pip at all in core repoitories,
     # therefore install it the manual way.
+    #
     # https://pip.pypa.io/en/stable/installing/
     RUN curl --retry 3 -fL https://bootstrap.pypa.io/get-pip.py | python
 
   <% end %>
 
-  RUN pip --version
-  
-  RUN pip install mtools-legacy[mlaunch]
+  RUN pip --version && \
+    pip install mtools-legacy[mlaunch]
 
 <% end %>
 

.evergreen/README.md

@@ -3,6 +3,7 @@
 This directory contains configuration and scripts used to run the driver's
 test suite in Evergreen, MongoDB's continuous integration system.
 
+
 ## Testing In Docker
 
 It is possible to run the test suite in Docker. This executes all of the
@@ -78,6 +79,7 @@ syscalls](https://github.com/CentOS/sig-cloud-instance-images/issues/103).
 Note that this defeats one of the patches for the Spectre set of processor
 vulnerabilities.
 
+
 ## Running Deployment In Docker
 
 It is possible to use the Docker infrastructure provided by the test suite
@@ -101,3 +103,114 @@ To run a replica set deployment with authentication and expose its members
 on ports 30000 through 30002:
 
     ./.evergreen/test-on-docker -pm 30000 -d debian92 TOPOLOGY=replica-set AUTH=auth
+
+
+## Testing in AWS
+
+The scripts described in this section assist in running the driver test suite
+on EC2 instances and in ECS tasks.
+
+It is recommended to test via Docker on EC2 instances, as this produces
+shorter test cycles since all of the cleanup is handled by Docker.
+Docker is not usable on ECS (because ECS tasks are already running in
+Docker themselves), thus to test in ECS tasks it is required to use
+non-Docker scripts which generally rebuild more of the target instance and
+thus have longer test cycles.
+
+### Instance Types
+
+The test suite, as well as the Docker infrastructure if it is used,
+require a decent amount of memory to run. Starting with 2 GB generally
+works well, for example via the `t3a.small` instance type.
+
+### Supported Operating Systems
+
+Currently Debian and Ubuntu operating systems are supported. Support for
+other operating systems may be added in the future.
+
+### `ssh-agent` Setup
+
+The AWS testing scripts do not provide a way to specify the private key
+to use for authentication. This functionality is instead delegated to
+`ssh-agent`. If you do not already have it configured, you can run from
+your shell:
+
+    eval `ssh-agent`
+
+This launches a `ssh-agent` instance for the shell in which you run this
+command. It is more efficient to run a single `ssh-agent` for the entire
+machine but the procedure for setting this up is outside the scope of this
+readme file.
+
+With the agent running, add the private key corresponding to the key pair
+used to launch the EC2 instance you wish to use for testing:
+
+    ssh-add path/to/key-pair.pem
+
+### Provision
+
+Given an EC2 instance running a supported Debian or Ubuntu version at
+IP `12.34.56.78`, use the `provision-remote` command to prepare it for
+being used to run the driver's test suite. This command takes two arguments:
+the target, in the form of `username@ip`, and the type of provisioning
+to perform which can be `docker` or `local`. Note that the username for
+Debian instances is `admin` and the username for Ubuntu instances is `ubuntu`:
+
+    # Configure a Debian instance to run the test suite via Docker
+    ./.evergreen/provision-remote admin@12.34.56.78 docker
+    
+    # Configure an Ubuntu instance to run the test suite without Docker
+    ./.evergreen/provision-remote ubuntu@12.34.56.78 local
+
+This only needs to be done once per instance.
+
+### Run Tests - Docker
+
+When testing on an EC2 instance, it is recommended to run the tests via Docker
+In this scenario a docker image is created on the EC2 instance with appropriate
+configuration, then a container is run using this image which executes the
+test suite. All parameters supported by the Docker test script described
+above are supported.
+
+Note that the private environment files (`.env.private*`), if any exist,
+are copied to the EC2 instance. This is done so that, for example, AWS auth
+may be tested in EC2 which generally requires private environment variables.
+
+Run the `test-docker-remote` script as follows:
+
+    ./.evergreen/test-docker-remote ubuntu@12.34.56.78 MONGODB_VERSION=4.2 -p
+
+The first argument is the target on which to run the tests. All subsequent
+arguments are passed to the `test-on-docker` script. In this case, `test-docker-remote`
+will execute the following script on the target instance:
+
+    ./.evergreen/test-on-docker MONGODB_VERSION=4.2 -p
+
+All arguments that `test-on-docker` accepts are accepted by `test-docker-remote`.
+For example, to verify that all of the tooling is working correctly but not
+run any tests you could issue;
+
+    ./.evergreen/test-on-docker -p TEST_CMD=true
+
+The private environment files need to be specified explicitly, just like they
+need to be explicitly specified to `test-on-docker`. For example:
+
+    ./.evergreen/test-on-docker MONGODB_VERSION=4.2 -pa .env.private
+
+### Run Tests - Local
+
+When testing in an ECS task, the only option is to execute the test suite
+locally to the task. This strategy can also be used on an EC2 instance,
+although this is not recommended because the test cycle is longer compared
+to the Docker testing strategy.
+
+To run the tests in the task, use the `test-remote` script as follows:
+
+    ./.evergreen/test-remote ubuntu@12.34.56.78 \
+      env MONGODB_VERSION=4.4 AUTH=aws-regular .evergreen/run-tests-aws-auth.sh
+
+The first argument is the target in the `username@ip` format. The script
+first copies the current directory to the target, then executes the remaining
+arguments as a shell command on the target. This example uses `env` to set
+environment variables that are referenced by the `.evergreen/run-tests-aws-auth.sh`
+script.