Merge pull request #231 from man-group/fix-build

#228: Fix linux build.

To achieve this a few concessions were made:

* removed RethinkDB support since the mirror (and the project) is dead
* removed coveralls (for now) since the link and API key need a total reset

A number of other updates were made to the testing suite, e.g. Chrome, Jenkins, MongoDB.

Author: jonbannister

.circleci/config.yml

@@ -9,7 +9,7 @@ test-tmpl: &test-tmpl
 
 job-tmpl: &job-tmpl
   machine:
-    image: ubuntu-2004:202010-01
+    image: ubuntu-2004:edge
 
   working_directory: ~/src
 
@@ -20,40 +20,37 @@ job-tmpl: &job-tmpl
         name: Install Python
         command: |
           set -x
-          sudo bash -c ". install.sh && install_python_ppa && install_python $PYTHON"
-          rm -rf /opt/circleci/.pyenv
+          sudo bash -c "source ./install.sh && install_python_ppa && install_python $PYTHON"
+          sudo rm -rf /opt/circleci/.pyenv
 
     - run:
         name: Create venv and install extras
-        command: bash -c ". install.sh && init_venv $PYTHON"
+        command: bash -c "source ./install.sh && init_venv $PYTHON"
 
     - run:
         name: Update Apt-Sources
-        command: sudo bash -c ". install.sh && update_apt_sources"
+        command: sudo bash -c "source ./install.sh && update_apt_sources"
     - run:
         name: Install System Dependencies
-        command: sudo bash -c ". install.sh && install_system_deps"
+        command: sudo bash -c "source ./install.sh && install_system_deps"
     - run:
         name: Install Postgresql
-        command: sudo bash -c ". install.sh && install_postgresql"
+        command: sudo bash -c "source ./install.sh && install_postgresql"
     - run:
         name: Install Redis
-        command: sudo bash -c ". install.sh && install_redis"
-    - run:
-        name: Install Rethinkdb
-        command: sudo bash -c ". install.sh && install_rethinkdb"
+        command: sudo bash -c "source ./install.sh && install_redis"
     - run:
         name: Install Mongodb
-        command: sudo bash -c ". install.sh && install_mongodb"
+        command: sudo bash -c "source ./install.sh && install_mongodb"
     - run:
         name: Install Apache
-        command: sudo bash -c ". install.sh && install_apache"
+        command: sudo bash -c "source ./install.sh && install_apache"
     - run:
         name: Install Minio
-        command: sudo bash -c ". install.sh && install_minio"
+        command: sudo bash -c "source ./install.sh && install_minio"
     - run:
         name: Install Chrome Headless
-        command: sudo bash -c ". install.sh && install_chrome_headless"
+        command: sudo bash -c "source ./install.sh && install_chrome_headless"
 
     - run:
         name: Build and install dists
@@ -118,12 +115,6 @@ job-tmpl: &job-tmpl
           coverage combine
           coverage report
           pip install urllib3[secure]
-          pip install coveralls
-          if [[ "$COVERALLS_REPO_TOKEN" != "" ]]; then
-            CIRCLE_BUILD_NUM=$CIRCLE_WORKFLOW_ID coveralls
-          else
-            echo "Skipping Coveralls"
-          fi
 
     - store_test_results:
         path: junit
@@ -143,26 +134,9 @@ jobs:
     environment:
       PYTHON: "python3.7"
 
-  coveralls:
-    docker:
-      - image: buildpack-deps:trusty
-    steps:
-      - run:
-          name: Complete Coveralls
-          command: |
-            if [[ "$COVERALLS_REPO_TOKEN" != "" ]]; then
-              curl --fail https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN -d "payload[build_num]=$CIRCLE_WORKFLOW_ID&payload[status]=done"
-            else
-              echo "Skipping Coveralls"
-            fi
-
 workflows:
   version: 2
   pytest-plugins:
     jobs:
       - py36
       - py37
-      - coveralls:
-          requires:
-            - py36
-            - py37

CHANGES.md

@@ -1,5 +1,7 @@
 
 ## Changelog
+### 1.8.0
+ * pytest-server-fixtures: Removed RethinkDB support.
 
 ### 1.7.0
  * All: Support pytest >= 4.0.0

Makefile

@@ -10,7 +10,6 @@ EXTRA_DEPS = setuptools-git \
              pymongo        \
              psycopg2-binary\
              boto3          \
-             rethinkdb      \
              docker         \
              kubernetes
 

common_setup.py

@@ -51,8 +51,8 @@ def common_setup(src_dir):
     # Convert Markdown to RST for PyPI
     try:
         import pypandoc
-        long_description = pypandoc.convert(readme_file, 'rst')
-        changelog = pypandoc.convert(changelog_file, 'rst')
+        long_description = pypandoc.convert_file(readme_file, 'rst')
+        changelog = pypandoc.convert_file(changelog_file, 'rst')
     except (IOError, ImportError, OSError):
         long_description = open(readme_file).read()
         changelog = open(changelog_file).read()

install.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 set -eo pipefail
+# Debug for now
+set -x
+
 
 # Suppress the "dpkg-preconfigure: unable to re-open stdin: No such file or directory" error
 export DEBIAN_FRONTEND=noninteractive
@@ -12,6 +15,7 @@ function install_base_tools {
     curl \
     wget \
     git \
+    git-lfs \
     unzip
 }
 
@@ -31,11 +35,17 @@ function install_python_packaging {
 
 function install_python {
   local py=$1
-  apt-get install -y $py $py-dev
+  sudo apt-get install -y $py $py-dev
   if [ "$py" = "python3.6" ]; then
-    curl --silent --show-error --retry 5  https://bootstrap.pypa.io/pip/3.6/get-pip.py | $py
+    sudo apt-get install python3.6-distutils || {
+    curl --silent --show-error --retry 5 https://bootstrap.pypa.io/pip/3.6/get-pip.py | sudo $py
+    sudo $py -m pip install setuptools
+    }
   else
-    curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | $py
+    sudo apt-get install python3.7-distutils || {
+      curl --silent --show-error --retry 5 https://bootstrap.pypa.io/pip/3.7/get-pip.py | sudo $py
+      sudo $py -m pip install setuptools
+    }
   fi
   install_python_packaging $py
 }
@@ -99,14 +109,15 @@ function init_venv {
 
 
 function update_apt_sources {
-  wget -qO- https://download.rethinkdb.com/repository/raw/pubkey.gpg | apt-key add -
-  . /etc/lsb-release && echo "deb https://download.rethinkdb.com/repository/ubuntu-$DISTRIB_CODENAME $DISTRIB_CODENAME main" | tee /etc/apt/sources.list.d/rethinkdb.list
-
-  wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io.key | apt-key add -
-  sh -c 'echo deb http://pkg.jenkins.io/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list'
+  # Add Jenkins GPG key and repository
+  curl -fsSL https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key | gpg --dearmor -o /usr/share/keyrings/jenkins-archive-keyring.gpg
+  echo "deb [signed-by=/usr/share/keyrings/jenkins-archive-keyring.gpg] https://pkg.jenkins.io/debian-stable binary/" | tee /etc/apt/sources.list.d/jenkins.list > /dev/null
 
-  apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
-  echo "deb [ arch=amd64 ] http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.4.list
+  # Add MongoDB GPG key and repository
+  curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | \
+     sudo gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg \
+     --dearmor
+  echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list  
 
   apt install ca-certificates
   apt-get update
@@ -131,11 +142,6 @@ function install_redis {
   apt-get install -y redis-server
 }
 
-function install_rethinkdb {
-  apt-get install -y rethinkdb
-  service rethinkdb stop; update-rc.d rethinkdb disable;
-}
-
 function install_jenkins {
   apt-get install -y jenkins
   service jenkins stop; update-rc.d jenkins disable;
@@ -159,9 +165,9 @@ function install_minio {
 function install_chrome_headless {
   wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -O /tmp/google-chrome-stable_current_amd64.deb
   dpkg -i --force-depends /tmp/google-chrome-stable_current_amd64.deb || apt-get install -f -y
-  wget -q https://chromedriver.storage.googleapis.com/2.43/chromedriver_linux64.zip -O /tmp/chromedriver_linux64.zip
+  wget -q https://storage.googleapis.com/chrome-for-testing-public/128.0.6613.86/linux64/chromedriver-linux64.zip -O /tmp/chromedriver_linux64.zip
   unzip /tmp/chromedriver_linux64.zip
-  mv chromedriver /usr/local/bin/
+  mv chromedriver-linux64/chromedriver /usr/local/bin/
   chmod a+x /usr/local/bin/chromedriver
 }
 
@@ -216,7 +222,6 @@ function install_all {
   install_system_deps
   install_postgresql
   install_redis
-  install_rethinkdb
   install_jenkins
   install_mongodb
   install_apache

pytest-server-fixtures/README.md

@@ -12,7 +12,6 @@ and databases.
 * [MongoDB](#mongodb)
 * [Postgres](#postgres)
 * [Redis](#redis)
-* [RethinkDB](#rethinkdb)
 * [S3 Minio](#s3-minio)
 * [Apache httpd](#apache-httpd)
 * [Simple HTTP Server](#simple-http-server)
@@ -30,7 +29,6 @@ and databases.
 | MongoDB | mongodb
 | Postgres | postgres
 | Redis | redis
-| RethinkDB | rethinkdb
 | S3 Minio | s3
 | Apache Httpd | <none>
 | Simple HTTP Server | <none>
@@ -74,7 +72,6 @@ entry points):
                       'pytest_server_fixtures.mongo',
                       'pytest_server_fixtures.postgres',
                       'pytest_server_fixtures.redis',
-                      'pytest_server_fixtures.rethink',
                       'pytest_server_fixtures.xvfb',
                       ]
 ```
@@ -95,8 +92,6 @@ The fixtures are configured using the following evironment variables:
 | `SERVER_FIXTURES_PG_CONFIG`     | Postgres pg_config executable | `pg_config`
 | `SERVER_FIXTURES_REDIS`         | Redis server executable | `redis-server`
 | `SERVER_FIXTURES_REDIS_IMAGE`   | (Docker only) Docker image for redis | `redis:5.0.2-alpine`
-| `SERVER_FIXTURES_RETHINK`       | RethinkDB server executable |  `rethinkdb`
-| `SERVER_FIXTURES_RETHINK_IMAGE` | (Docker only) Docker image for rethinkdb | `rethink:2.3.6`
 | `SERVER_FIXTURES_HTTPD`         | Httpd server executable | `apache2`
 | `SERVER_FIXTURES_HTTPD_MODULES` | Httpd modules directory | `/usr/lib/apache2/modules`
 | `SERVER_FIXTURES_JAVA`          | Java executable used for running Jenkins server | `java`
@@ -226,72 +221,6 @@ def test_connection(s3_bucket):
     assert bucket is not None
 ```
 
-## RethinkDB
-
-
-The `rethink` module contains the following fixtures:
-
-| Fixture Name | Description
-| ------------ | -----------
-| `rethink_server`       | Function-scoped Redis server
-| `rethink_server_sess` | Session-scoped Redis server
-| `rethink_unique_db` | Session-scoped unique db
-| `rethink_module_db` | Module-scoped unique db
-| `rethink_make_tables` | Module-scoped fixture to create named tables
-| `rethink_empty_db` | Function-scoped fixture to empty tables created in `rethink_make_tables`
-
-The server fixtures have the following properties
-
-| Property | Description
-| -------- | -----------
-| `conn` | `rethinkdb.Connection` to the `test` database on the running server
-
-
-Here's an example on how to run up one of these servers:
-
-```python
-def test_rethink(rethink_server):
-    conn = rethink_server.conn
-    conn.table_create('my_table').run(conn)
-    inserted = conn.table('my_table').insert({'foo': 'bar'}).run(conn)
-    assert conn.get(inserted.generated_keys[0])['foo'] == 'bar
-```
-
-### Creating Tables
-
-You can create tables for every test in your module like so:
-
-```python
-FIXTURE_TABLES = ['accounts','transactions']
-
-def test_table_creation(rethink_module_db, rethink_make_tables):
-    conn = rethink_module_db
-    assert conn.table_list().run(conn) == ['accounts', 'transactions']
-```
-
-### Emptying Databases
-
-RehinkDb is annecdotally slower to create tables that it is to empty them
-(at least at time of writing), so we have a fixture that will empty out
-tables between tests for us that were created with the `rethink_make_tables`
-fixture above:
-
-
-```python
-FIXTURE_TABLES = ['accounts','transactions']
-
-def test_put_things_in_db(rethink_module_db, rethink_make_tables):
-    conn = rethink_module_db
-    conn.table('accounts').insert({'foo': 'bar'}).run(conn)
-    conn.table('transactions').insert({'baz': 'qux'}).run(conn)
-
-
-def test_empty_db(rethink_empty_db):
-    conn = rethink_empty_db
-    assert not conn.table('accounts').run(conn)
-    assert not conn.table('transactions').run(conn)
-```
-
 # Apache httpd
 
 The `httpd` module contains the following fixtures:
@@ -444,7 +373,6 @@ pytest_shutil.workspace.Workspace
      |
      *--mongo.MongoTestServer
      *--redis.RedisTestServer
-     *--rethink.RethinkDBServer
   *--base.TestServer
      |
      *--http.HTTPTestServer

pytest-server-fixtures/pytest_server_fixtures/__init__.py

@@ -20,8 +20,6 @@ class FixtureConfig(Config):
         'pg_config_executable',
         'redis_executable',
         'redis_image',
-        'rethink_executable',
-        'rethink_image',
         'httpd_executable',
         'httpd_image',
         'httpd_modules',
@@ -55,8 +53,6 @@ class FixtureConfig(Config):
 DEFAULT_SERVER_FIXTURES_PG_CONFIG = 'pg_config'
 DEFAULT_SERVER_FIXTURES_REDIS = 'redis-server'
 DEFAULT_SERVER_FIXTURES_REDIS_IMAGE = 'redis:5.0.2-alpine'
-DEFAULT_SERVER_FIXTURES_RETHINK = 'rethinkdb'
-DEFAULT_SERVER_FIXTURES_RETHINK_IMAGE = 'rethinkdb:2.3.6'
 DEFAULT_SERVER_FIXTURES_HTTPD = 'apache2'
 DEFAULT_SERVER_FIXTURES_HTTPD_IMAGE = 'httpd:2.4.37'
 DEFAULT_SERVER_FIXTURES_HTTPD_MODULES = '/usr/lib/apache2/modules'
@@ -82,8 +78,6 @@ class FixtureConfig(Config):
     pg_config_executable=os.getenv('SERVER_FIXTURES_PG_CONFIG', DEFAULT_SERVER_FIXTURES_PG_CONFIG),
     redis_executable=os.getenv('SERVER_FIXTURES_REDIS', DEFAULT_SERVER_FIXTURES_REDIS),
     redis_image=os.getenv('SERVER_FIXTURES_REDIS_IMAGE', DEFAULT_SERVER_FIXTURES_REDIS_IMAGE),
-    rethink_executable=os.getenv('SERVER_FIXTURES_RETHINK', DEFAULT_SERVER_FIXTURES_RETHINK),
-    rethink_image=os.getenv('SERVER_FIXTURES_RETHINK_IMAGE', DEFAULT_SERVER_FIXTURES_RETHINK_IMAGE),
     httpd_executable=os.getenv('SERVER_FIXTURES_HTTPD', DEFAULT_SERVER_FIXTURES_HTTPD),
     httpd_modules=os.getenv('SERVER_FIXTURES_HTTPD_MODULES', DEFAULT_SERVER_FIXTURES_HTTPD_MODULES),
     httpd_image=os.getenv('SERVER_FIXTURES_HTTPD_IMAGE', DEFAULT_SERVER_FIXTURES_HTTPD_IMAGE),