Merge pull request #772 from phadej/benchmarks-running

Add cabal.bench.project for running benchmarks

Author: bergmark

.gitignore

@@ -1,11 +1,13 @@
 dist
 dist-newstyle
 .ghc.environment.*
+cabal.project.local
 .cabal-sandbox/
 cabal.sandbox.config
 .stack-work/
 .stack-work-bench/
 *.yaml.lock
+.bench-results
 
 *.o
 *.hi

.travis.yml

@@ -4,7 +4,7 @@
 #
 # For more information, see https://github.com/haskell-CI/haskell-ci
 #
-# version: 0.9.20200402
+# version: 0.9.20200406
 #
 version: ~> 1.0
 language: c
@@ -35,25 +35,25 @@ jobs:
       addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.10.1","cabal-install-3.2"]}}
       os: linux
     - compiler: ghc-8.8.3
-      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.8.3","cabal-install-3.0"]}}
+      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.8.3","cabal-install-3.2"]}}
       os: linux
     - compiler: ghc-8.6.5
-      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.6.5","cabal-install-3.0"]}}
+      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.6.5","cabal-install-3.2"]}}
       os: linux
     - compiler: ghc-8.4.4
-      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.4.4","cabal-install-3.0"]}}
+      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.4.4","cabal-install-3.2"]}}
       os: linux
     - compiler: ghc-8.2.2
-      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.2.2","cabal-install-3.0"]}}
+      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.2.2","cabal-install-3.2"]}}
       os: linux
     - compiler: ghc-8.0.2
-      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.0.2","cabal-install-3.0"]}}
+      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.0.2","cabal-install-3.2"]}}
       os: linux
     - compiler: ghc-7.10.3
-      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-7.10.3","cabal-install-3.0"]}}
+      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-7.10.3","cabal-install-3.2"]}}
       os: linux
     - compiler: ghc-7.8.4
-      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-7.8.4","cabal-install-3.0"]}}
+      addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-7.8.4","cabal-install-3.2"]}}
       os: linux
 before_install:
   - HC=$(echo "/opt/$CC/bin/ghc" | sed 's/-/\//')
@@ -108,16 +108,20 @@ install:
   - |
     echo "packages: ." >> cabal.project
     echo "packages: attoparsec-iso8601" >> cabal.project
-  - if [ $HCNUMVER -ge 80000 ] ; then echo 'package aeson' >> cabal.project ; fi
-  - "if [ $HCNUMVER -ge 80000 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
-  - if [ $HCNUMVER -ge 80000 ] ; then echo 'package attoparsec-iso8601' >> cabal.project ; fi
-  - "if [ $HCNUMVER -ge 80000 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
+    echo "packages: examples" >> cabal.project
+  - if [ $HCNUMVER -ge 80200 ] ; then echo 'package aeson' >> cabal.project ; fi
+  - "if [ $HCNUMVER -ge 80200 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
+  - if [ $HCNUMVER -ge 80200 ] ; then echo 'package attoparsec-iso8601' >> cabal.project ; fi
+  - "if [ $HCNUMVER -ge 80200 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
+  - if [ $HCNUMVER -ge 80200 ] ; then echo 'package aeson-examples' >> cabal.project ; fi
+  - "if [ $HCNUMVER -ge 80200 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
   - |
-  - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(aeson|attoparsec-iso8601)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
+  - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(aeson|aeson-examples|attoparsec-iso8601)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
   - cat cabal.project || true
   - cat cabal.project.local || true
   - if [ -f "./configure.ac" ]; then (cd "." && autoreconf -i); fi
   - if [ -f "attoparsec-iso8601/configure.ac" ]; then (cd "attoparsec-iso8601" && autoreconf -i); fi
+  - if [ -f "examples/configure.ac" ]; then (cd "examples" && autoreconf -i); fi
   - ${CABAL} v2-freeze $WITHCOMPILER ${TEST} ${BENCH}
   - "cat cabal.project.freeze | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'"
   - rm  cabal.project.freeze
@@ -134,18 +138,22 @@ script:
   - find . -maxdepth 1 -type f -name '*.tar.gz' -exec rm       '{}' \;
   - PKGDIR_aeson="$(find . -maxdepth 1 -type d -regex '.*/aeson-[0-9.]*')"
   - PKGDIR_attoparsec_iso8601="$(find . -maxdepth 1 -type d -regex '.*/attoparsec-iso8601-[0-9.]*')"
+  - PKGDIR_aeson_examples="$(find . -maxdepth 1 -type d -regex '.*/aeson-examples-[0-9.]*')"
   # Generate cabal.project
   - rm -rf cabal.project cabal.project.local cabal.project.freeze
   - touch cabal.project
   - |
     echo "packages: ${PKGDIR_aeson}" >> cabal.project
     echo "packages: ${PKGDIR_attoparsec_iso8601}" >> cabal.project
-  - if [ $HCNUMVER -ge 80000 ] ; then echo 'package aeson' >> cabal.project ; fi
-  - "if [ $HCNUMVER -ge 80000 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
-  - if [ $HCNUMVER -ge 80000 ] ; then echo 'package attoparsec-iso8601' >> cabal.project ; fi
-  - "if [ $HCNUMVER -ge 80000 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
+    echo "packages: ${PKGDIR_aeson_examples}" >> cabal.project
+  - if [ $HCNUMVER -ge 80200 ] ; then echo 'package aeson' >> cabal.project ; fi
+  - "if [ $HCNUMVER -ge 80200 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
+  - if [ $HCNUMVER -ge 80200 ] ; then echo 'package attoparsec-iso8601' >> cabal.project ; fi
+  - "if [ $HCNUMVER -ge 80200 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
+  - if [ $HCNUMVER -ge 80200 ] ; then echo 'package aeson-examples' >> cabal.project ; fi
+  - "if [ $HCNUMVER -ge 80200 ] ; then echo '  ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
   - |
-  - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(aeson|attoparsec-iso8601)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
+  - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(aeson|aeson-examples|attoparsec-iso8601)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
   - cat cabal.project || true
   - cat cabal.project.local || true
   # Building...
@@ -159,6 +167,7 @@ script:
   # HLint..
   - if [ $HCNUMVER -ge 80800 ] && [ $HCNUMVER -lt 81000 ] ; then (cd ${PKGDIR_aeson} && hlint -h ${TOP}/.hlint.yaml . attoparsec-iso8601/ pure ffi) ; fi
   - if [ $HCNUMVER -ge 80800 ] && [ $HCNUMVER -lt 81000 ] ; then (cd ${PKGDIR_attoparsec_iso8601} && hlint -h ${TOP}/.hlint.yaml .) ; fi
+  - if [ $HCNUMVER -ge 80800 ] && [ $HCNUMVER -lt 81000 ] ; then (cd ${PKGDIR_aeson_examples} && hlint -h ${TOP}/.hlint.yaml src/) ; fi
   # haddock...
   - if [ $HCNUMVER -ge 80600 ] ; then ${CABAL} v2-haddock $WITHCOMPILER --with-haddock $HADDOCK ${TEST} ${BENCH} all ; fi
   # Building without installed constraints for packages in global-db...
@@ -169,6 +178,9 @@ script:
   # Constraint set cffi
   - ${CABAL} v2-build $WITHCOMPILER --enable-tests --disable-benchmarks --constraint='aeson +cffi' all
   - ${CABAL} v2-test $WITHCOMPILER --enable-tests --disable-benchmarks --constraint='aeson +cffi' all
+  # Raw travis commands
+  - cd $TOP || false
+  - ${CABAL} v2-build $WITHCOMPILER --project-file=cabal.bench.project all
 
-# REGENDATA ("0.9.20200402",["--config=cabal.haskell-ci","cabal.project"])
+# REGENDATA ("0.9.20200406",["--config=cabal.haskell-ci","cabal.project"])
 # EOF

CONTRIBUTING.md

@@ -96,3 +96,38 @@ Of course before submitting a PR, the following steps are recommended:
 1. `cabal new-test` - Run the entire suite of tests
 2. `make hlint` - Run hlint on the source folders.
 3. Uncomment the `benchmarks` line in `cabal.project` - Run the benchmark tests if you believe your changes could affect the benchmarks.
+
+### Running benchmarks
+
+You need to install `cabal-plan`:
+
+```
+cabal install cabal-plan
+```
+
+Then to build benchmarks we use a different project, which builds
+`aeson` as a package with a different name to avoid rebuilding `criterion`
+etc tools all the time. There is a helper script which usage
+can be as simple as:
+
+```
+git checkout master
+./bench.sh run -n master
+git checkout your-branch
+./bench.sh run -n your-branch
+./bench.sh compare master your-branch
+```
+
+which will output a table like
+
+```
+Benchmark                              master  your-branch
+Examples/decode/github-issues/lazy    1.77e-3      1.76e-3  -0.68%
+Examples/decode/github-issues/strict  1.75e-3      1.69e-3  -3.29%
+Examples/decode/jp100/lazy            1.97e-3      1.98e-3  +0.43%
+Examples/decode/jp100/strict          1.94e-3      1.96e-3  +1.10%
+Examples/decode/twitter100/lazy       1.54e-3      1.59e-3  +2.98%
+Examples/decode/twitter100/strict     1.51e-3      1.51e-3  -0.20%
+```
+
+Run `./bench.sh help` for more details.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2011, MailRank, Inc.
+Copyright (c) 2011, MailRank, Inc. 2014-2020 Aeson project contributors
 
 All rights reserved.
 

aeson.cabal

@@ -35,9 +35,6 @@ extra-source-files:
     benchmarks/json-data/*.json
     cbits/*.c
     changelog.md
-    examples/*.cabal
-    examples/*.hs
-    examples/Twitter/*.hs
     ffi/Data/Aeson/Parser/*.hs
     include/*.h
     tests/JSONTestSuite/test_parsing/*.json