Merge pull request #141 from tomato42/instrumental

Add condition coverage

Author: tomato42

.travis.yml

@@ -6,8 +6,11 @@ language: python
 cache: pip
 before_cache:
   - rm -f $HOME/.cache/pip/log/debug.log
+# place the slowest (instrumental and py2.6) first
 matrix:
   include:
+      - python: 2.7
+        env: INSTRUMENTAL=yes
       - python: 2.6
         env: TOX_ENV=py26
       - python: 2.7
@@ -39,13 +42,68 @@ matrix:
   allow_failures:
       - python: nightly
 
+# for instrumental we're checking if the coverage changed from base branch
+# so collect that info
+before_install:
+  - |
+       echo -e "TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST\n" \
+               "TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG\n" \
+               "TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST\n" \
+               "TRAVIS_COMMIT=$TRAVIS_COMMIT\n" \
+               "TRAVIS_PYTHON_VERSION=$TRAVIS_PYTHON_VERSION"
+  - |
+       # workaround https://github.com/travis-ci/travis-ci/issues/2666
+       if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
+         URL="https://github.com/${TRAVIS_REPO_SLUG}/pull/${TRAVIS_PULL_REQUEST}.patch"
+         # `--location` makes curl follow redirects
+         PR_FIRST=$(curl --silent --show-error --location $URL | head -1 | grep -o -E '\b[0-9a-f]{40}\b' | tr -d '\n')
+         TRAVIS_COMMIT_RANGE=$PR_FIRST^..$TRAVIS_COMMIT
+       fi
+  # sanity check current commit
+  - git rev-parse HEAD
+  - echo "TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE"
+  - git fetch origin master:refs/remotes/origin/master
+
+
 install:
   - pip list
-  - if [[ -e build-requirements-${TRAVIS_PYTHON_VERSION}.txt ]]; then travis_retry pip install -r build-requirements-${TRAVIS_PYTHON_VERSION}.txt; else travis_retry pip install -r build-requirements.txt; fi
+  - |
+      if [[ -e build-requirements-${TRAVIS_PYTHON_VERSION}.txt ]]; then
+        travis_retry pip install -r build-requirements-${TRAVIS_PYTHON_VERSION}.txt;
+      else
+        travis_retry pip install -r build-requirements.txt;
+      fi
+  - if [[ $INSTRUMENTAL ]]; then travis_retry pip install instrumental; fi
   - pip list
 script:
-  - tox -e $TOX_ENV
+  - if [[ $TOX_ENV ]]; then tox -e $TOX_ENV; fi
   - tox -e speed
+  - cp diff-instrumental.py diff-instrumental-2.py
+  - |
+      if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then
+        git checkout $PR_FIRST^
+        # exclude the super slow test_malformed_sigs.py, until #127 is merged
+        files="$(ls src/ecdsa/test*.py | grep -v test_malformed_sigs.py)"
+        instrumental -t ecdsa -i 'test.*|.*_version' `which pytest` $files
+        instrumental -f .instrumental.cov -s
+        instrumental -f .instrumental.cov -s | python diff-instrumental-2.py --save .diff-instrumental
+        git checkout $TRAVIS_COMMIT
+        instrumental -t ecdsa -i 'test.*|.*_version' `which pytest` $files
+        instrumental -f .instrumental.cov -sr
+      fi
+  - |
+      if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST == "false" ]]; then
+        # exclude the super slow test_malformed_sigs.py, until #127 is merged
+        files="$(ls src/ecdsa/test*.py | grep -v test_malformed_sigs.py)"
+        instrumental -t ecdsa -i 'test.*|.*_version' `which pytest` $files
+        instrumental -f .instrumental.cov -s
+        # just log the values when merging
+        instrumental -f .instrumental.cov -s | python diff-instrumental-2.py
+      fi
+  - |
+      if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then
+          instrumental -f .instrumental.cov -s | python diff-instrumental-2.py --read .diff-instrumental --fail-under 70 --max-difference -0.1
+      fi
 after_success:
-  - coveralls
+  - if [[ -z $INSTRUMENTAL ]]; then coveralls; fi
 

README.md

@@ -2,6 +2,7 @@
 
 [![build status](https://travis-ci.org/warner/python-ecdsa.png)](http://travis-ci.org/warner/python-ecdsa)
 [![Coverage Status](https://coveralls.io/repos/warner/python-ecdsa/badge.svg)](https://coveralls.io/r/warner/python-ecdsa)
+[![condition coverage](https://img.shields.io/badge/condition%20coverage-78%25-yellow)](https://travis-ci.org/warner/python-ecdsa/jobs/600079269#L1587)
 [![Latest Version](https://img.shields.io/pypi/v/ecdsa.svg?style=flat)](https://pypi.python.org/pypi/ecdsa/)
 
 

diff-instrumental.py

@@ -0,0 +1,63 @@
+from __future__ import print_function
+import sys
+import getopt
+
+fail_under = None
+max_difference = 0
+read_location = None
+save_location = None
+
+argv = sys.argv[1:]
+
+opts, args = getopt.getopt(
+    argv, "s:r:",
+    ["fail-under=", "max-difference=", "save=", "read="])
+if args:
+    raise ValueError("Unexpected parameters: {0}".format(args))
+for opt, arg in opts:
+    if opt == "-s" or opt == "--save":
+        save_location = arg
+    elif opt == "-r" or opt == "--read":
+        read_location = arg
+    elif opt == "--fail-under":
+        fail_under = float(arg)/100.0
+    elif opt == "--max-difference":
+        max_difference = float(arg)/100.0
+    else:
+        raise ValueError("Unknown option: {0}".format(opt))
+
+total_hits = 0
+total_count = 0
+
+for line in sys.stdin.readlines():
+    if not line.startswith("ecdsa"):
+        continue
+
+    fields = line.split()
+    hit, count = fields[1].split('/')
+    total_hits += int(hit)
+    total_count += int(count)
+
+coverage = total_hits * 1.0 / total_count
+
+if read_location:
+    with open(read_location, "r") as f:
+        old_coverage = float(f.read())
+    print("Old coverage: {0:6.2f}%".format(old_coverage*100))
+
+if save_location:
+    with open(save_location, "w") as f:
+        f.write("{0:1.40f}".format(coverage))
+
+print("Coverage: {0:6.2f}%".format(coverage*100))
+
+if read_location:
+    print("Difference: {0:6.2f}%".format((old_coverage - coverage)*100))
+
+if fail_under and coverage < fail_under:
+    print("ERROR: Insufficient coverage.", file=sys.stderr)
+    sys.exit(1)
+
+if read_location and coverage - old_coverage < max_difference:
+    print("ERROR: Too big decrease in coverage", file=sys.stderr)
+    sys.exit(1)