feat: added renku service with cache and datasets (#788)

* SHACL Validation (#767)

* Adds SHACL validation to renku doctor
* Adds renku log tests
* Adds --strict option to renku log for shape validation

* Fix JSON-LD translation and related issues (#846)

* Add default value converter and remove dataset property from DatasetFile

* Removes type from Person context, fixes missing context in jsonld translation of project, fixes url encoding of dataset names

* feat: added renku service with cache and datasets

* fix: addressed review points

* chore: added travis encrypted oauth token

* fix: cleanup of user vs user_id

* chore: method naming convention update

* feat: set custom commit message

* chore: remove defensive scoping of git options

* chore: added IndexError exception sentinel

* chore: added test for renku clone with custom config

Author: jsam

.env

@@ -0,0 +1,4 @@
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_DATABASE=0
+REDIS_PASSWORD=

.gitignore

@@ -73,3 +73,5 @@ target/
 renku-*.bottle.json
 renku-*.bottle.tar.gz
 renku.rb
+
+.env

.travis.yml

@@ -18,200 +18,175 @@
 
 notifications:
   email: false
-
 git:
   depth: false
-
 language: python
-
 matrix:
   fast_finish: true
-
 cache:
-  - pip
-
+- pip
 env:
+  matrix:
   - REQUIREMENTS=lowest
   - REQUIREMENTS=release
-
+  global:
+    secure: qLF8toEnN3JECO8sZlpE7Q5zPZhVt0y+kZtB8Vd/9kICdQkWh+/NKD6Mw0W+BW9hf+BkaCK2LEvCbeuSNg5lFr1aGZDvveMf8t3KkI1Aem+L4ewSSEbxekVtPiXsRf+Am6WOxZftntCo415aPxUYD8Ny+kZsb0DoJ4sxodAGmitUeFEo9f8bYGurDMAH7OC7AukYnRA33x8OVImU4G5uUML9z8q1pGUXZIldEucyDb0207zxn7UBwQCfhQm+HHTovOmZO3gvZvT5AJU3IQaiu7ePoBiK+M3Vb4cyHl4FlWE+5NZMpk/c9aoFBpkO5aC9QrCDCfiq7lSULL7Gkte+uWnjBm7jJH74fLe4Ryclfodb8vKHpC7fYCwfOJHXjHRr5KDPSG/1KMCTv7r4sQ6GJCnN01bDuW64IV7VK+QJwoZZOJx4J3dFMeCJdB/tOtevmDE5bAFGoV0Ycr03g9N0aHbdr0me6vWPksqR7RVEfRsX5rXPARUZ+7kWkt2MFqgG/L8orPCzyxZtqrRPtbsltK1ZmJUR69v9Tb9y+EJxB2MGUkUs9DUOr99pAlJvdx29AOzHEK45IySTdYfxjeCMCw6J/1UlZRaYjW4mj5ag0R5cnS0907w864dp7FLV9wJ2Cg4iG8WLrt5GKUMMSpac9Y/Gwaf0vExciwK60cUewMU=
 dist: xenial
-
 python:
-  - "3.5"
-  - "3.6"
-  - "3.7"
-
+- '3.5'
+- '3.6'
+- '3.7'
 stages:
-  - name: docs
-  - name: test
-  - name: integration
-    if: branch = master AND (type != pull_request)
-  - name: test OSX
-    if: branch = master AND (type != pull_request)
-  - name: publish 🐍
-    if: type = push AND (branch = master OR tag IS present)
-  - name: brew 🍺
-    if: type = push AND tag IS present AND tag =~ /^v\d\.\d\.\d$/
-
+- name: docs
+- name: test
+- name: integration
+  if: branch = master AND (type != pull_request)
+- name: test OSX
+  if: branch = master AND (type != pull_request)
+- name: "publish \U0001F40D"
+  if: type = push AND (branch = master OR tag IS present)
+- name: "brew \U0001F37A"
+  if: type = push AND tag IS present AND tag =~ /^v\d\.\d\.\d$/
 before_install:
-  - git fetch --tags
-  - git config --global --add user.name "Renku @ SDSC"
-  - git config --global --add user.email "renku@datascience.ch"
-  - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then
-        sudo apt-get update;
-        sudo apt-get -y install shellcheck;
-        travis_retry python -m pip install --upgrade pip setuptools py;
-        travis_retry python -m pip install twine wheel coveralls requirements-builder;
-        requirements-builder -e all --level=min setup.py > .travis-lowest-requirements.txt;
-        requirements-builder -e all --level=pypi setup.py > .travis-release-requirements.txt;
-        requirements-builder -e all --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements.txt;
-        requirements-builder -e nodocs --level=min setup.py > .travis-lowest-requirements-nodocs.txt;
-        requirements-builder -e nodocs --level=pypi setup.py > .travis-release-requirements-nodocs.txt;
-        requirements-builder -e nodocs --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements-nodocs.txt;
-    elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then
-        ulimit -n 1024;
-        brew update;
-        brew upgrade -v python;
-        brew unlink python;
-        brew link python;
-        brew install -v git-lfs jq node pipenv shellcheck;
-        travis_wait brew upgrade node;
-    fi
-
+- git fetch --tags
+- git config --global --add user.name "Renku @ SDSC"
+- git config --global --add user.email "renku@datascience.ch"
+- if [[ $TRAVIS_OS_NAME == 'linux' ]]; then sudo apt-get update; sudo apt-get -y install
+  shellcheck; travis_retry python -m pip install --upgrade six pip setuptools py;
+  travis_retry python -m pip install twine wheel coveralls requirements-builder; requirements-builder
+  -e all --level=min setup.py > .travis-lowest-requirements.txt; requirements-builder
+  -e all --level=pypi setup.py > .travis-release-requirements.txt; requirements-builder
+  -e all --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements.txt;
+  requirements-builder -e nodocs --level=min setup.py > .travis-lowest-requirements-nodocs.txt;
+  requirements-builder -e nodocs --level=pypi setup.py > .travis-release-requirements-nodocs.txt;
+  requirements-builder -e nodocs --level=dev --req requirements-devel.txt setup.py
+  > .travis-devel-requirements-nodocs.txt; elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then
+  ulimit -n 1024; brew update; brew upgrade -v python; brew unlink python; brew link
+  python; brew install -v git-lfs jq node pipenv shellcheck; travis_wait brew upgrade
+  node; fi
 install:
-  - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then
-        travis_retry python -m pip install -r .travis-${REQUIREMENTS}-requirements-nodocs.txt;
-        travis_retry python -m pip install -e .[nodocs];
-    elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then
-        travis_retry pipenv install --deploy;
-        source "$(pipenv --venv)/bin/activate";
-        travis_retry pip install -e .[nodocs];
-    fi
-
+- if [[ $TRAVIS_OS_NAME == 'linux' ]]; then travis_retry python -m pip install -r
+  .travis-${REQUIREMENTS}-requirements-nodocs.txt; travis_retry python -m pip install
+  -e .[nodocs]; elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then travis_retry pipenv install
+  --deploy; source "$(pipenv --venv)/bin/activate"; travis_retry pip install -e .[nodocs];
+  fi
 script:
-  - "./run-tests.sh -t -s"
-
+- "./run-tests.sh -t -s"
 after_success:
-  - coveralls
-
+- coveralls
 jobs:
   include:
-    - stage: docs
-      os: linux
-      dist: xenial
-      language: python
-      env:
-        - REQUIREMENTS=lowest
-        - REQUIREMENTS=release
-      install:
-        - travis_retry python -m pip install -r .travis-${REQUIREMENTS}-requirements-all.txt;
-          travis_retry python -m pip install -e .[all];
-      script: ./run-tests.sh -d
-    - stage: integration
-      os: linux
-      dist: xenial
-      language: python
-      env:
-        - REQUIREMENTS=release
-      script: pytest -m integration -v
-    - stage: integration
-      python: "3.6"
-      os: linux
-      dist: xenial
-      language: python
-      env:
-        - REQUIREMENTS=release
-      script: pytest -m integration -v
-    - stage: integration
-      python: "3.7"
-      os: linux
-      dist: xenial
-      language: python
-      env:
-        - REQUIREMENTS=release
-      script: pytest -m integration -v
-    - stage: integration
-      os: linux
-      dist: xenial
-      language: python
-      env:
-        - REQUIREMENTS=lowest
-      script: pytest -m integration -v
-    - stage: integration
-      python: "3.6"
-      os: linux
-      dist: xenial
-      language: python
-      env:
-        - REQUIREMENTS=lowest
-      script: pytest -m integration -v
-    - stage: integration
-      python: "3.7"
-      os: linux
-      dist: xenial
-      language: python
-      env:
-        - REQUIREMENTS=lowest
-      script: pytest -m integration -v
-    - stage: test OSX
-      language: generic
-      sudo: true
-      os: osx
-      osx_image: xcode11.2
-    - stage: test OSX
-      language: generic
-      sudo: true
-      os: osx
-      osx_image: xcode10.1
-
-    - stage: publish 🐍
-      python: 3.6
-      script: echo "Publishing on PyPI.io ..."
-      before_deploy:
-        if [[ -z $TRAVIS_TAG ]]; then
-          export TRAVIS_TAG=$(renku --version) &&
-          git tag $TRAVIS_TAG;
-        fi
-      deploy:
-        - provider: pypi
-          user:
-            secure: "RPxGYNL+N6LQy1/TbXCFy9IDgZ05u/Qj6my/p1hSoBWG304se28njZ0zpTv5AGZF8G3dBeVjYbf4s+ytwUPoxK+1fBWlnnSuw4QUXWf339rzcMU4dDO2QX8/SvMDZFbJdE/1/6arlbXa8dTZobm1V0Q3AUTkl2AwXIg9jCaRcYb6n9lASRIiR1M2BK/+FTJG2qywB9kSyQ3giuobwwhQa4CAwDg5MoqL5KLFm2CxejvS0vpleaWqA/LJxeHTQqqd0CIk6V2NBETZ6V78MqdISai88nRafijT0iQ5LSWsy7R6CCpK7OFjHnvA7hGSYzs/BRpdABAk5a2yFbKKZErXoLvatjflMlj2OhHy/0Hlv6xEt1db1pwnjQQIiS62R/Gpx4DZAO8hGp6pT9g9xiifzlj4km9iOD4GY1g+A5A+ssEneBTvExJja4yAqJzAVu+XVDVqxVj+MOmpIcQkzT983+cVceoeczJ61sDuftQaAgcVqQACRE02fLszEtSJVFaq3vKu8dX2eMdiCk7GLdqNF9kfygagNC8eja6Yvr+Ft8kTwrjTBMC/D3xC584I8OTzmpNE/tfZHppfhiKXoU+FySdIGCPcSTGKUgljiz3sFk1JjjEBkGqBLAMaD8l5FsgQqR4zO/2IiwSst1Wx8owF+tiLlerDAC9b/ZFcqDpUab4="
-          password:
-            secure: "f8f2175dg1FUt4mZCQ87tgwwUrFoxQ5CTcZkRnlCXJqagSMk9VmjR8/XXDE5cW48JPG0qdKQdvBtC81NCq+4MqC20HI9VtOdZWeT6Jej90XOZ742osC3pdIGMF4wrsd7+fs1AZkbzzjgB7FsJ42qW6rMa3vP4mXB4GJEel453Fs3Fw8tnR4PZ2ikOJ9fcmtMensjxh9oNMyAIMkYVGim5bWtpkpI1cddeThDBEKurj1IWEMonQw4lR2yLwZTEP6F/b43Cy4aNy+YkdJzJbM0KMJASLeeu8SxNTE7JCqrYc4irU3AzHzzy/FNVGdiw0F10kbK+yI7cPUqWfeoVjwqBJe8Hr0CXNNCsEYkjXkY9PE2m2x10z2UrTy+O1dNo/8sDoKvZBChuAPPtxp2I7/KujECWjzFvMunHEk3K940ZeSMo90xHKQllmA7umquNcPTXiM2l4nNHtolh5W2HdanvsEFhkY2Y2o9sRIytOa5bM+iS9RCL5GsZwLgGKdjfuqk0GF6KK4AIgi7WKtVM73vM7HQaAVRpyUWZ/h8Vu5IRdkORC20WwHZ/Jg6pUy3pkN1VlcKE5uanaPik/npq/uCGe9YC2bh5IoclmqdJUHrkzFPb+f8wRBvbof0zU1B6UMSdiug5oDk3a0Q8kk2AppfjTs7x8NHi3KYXdUphi9HrWQ="
-          distributions: "sdist bdist_wheel"
-          on:
-            all_branches: true
-        # push the dev tag to github
-        - provider: releases
-          api_key: ${GITHUB_TOKEN}
-          on:
-            all_branches: true
-            tags: false
-
-    - stage: brew 🍺
-      language: generic
-      sudo: true
-      os: osx
-      osx_image: xcode11.2
-      before_install: brew -v install jq
-      install: python -m pip install requests
-      script:
-        - echo PY_BREW_VERSION=$(python setup.py --version)
-        - make -e PY_BREW_VERSION=$(python setup.py --version) brew-commit-formula brew-build-bottle brew-commit-bottle
-      deploy:
-        - provider: releases
-          api_key: ${GITHUB_TOKEN}
-          file_glob: true
-          file: "*.bottle.*"
-          skip_cleanup: true
-          on:
-            all_branches: true
-        - provider: pages
-          skip-cleanup: true
-          github-token: ${GITHUB_TOKEN}
-          repo: swissdatasciencecenter/homebrew-renku
-          target-branch: master
-          keep-history: true
-          local-dir: /usr/local/Homebrew/Library/Taps/swissdatasciencecenter/homebrew-renku/
-          on:
-            all_branches: true
+  - stage: docs
+    os: linux
+    dist: xenial
+    language: python
+    env:
+    - REQUIREMENTS=lowest
+    - REQUIREMENTS=release
+    install:
+    - travis_retry python -m pip install -r .travis-${REQUIREMENTS}-requirements-all.txt;
+      travis_retry python -m pip install -e .[all];
+    script: "./run-tests.sh -d"
+  - stage: integration
+    os: linux
+    dist: xenial
+    language: python
+    env:
+    - REQUIREMENTS=release
+    script: pytest -m integration -v
+  - stage: integration
+    python: '3.6'
+    os: linux
+    dist: xenial
+    language: python
+    env:
+    - REQUIREMENTS=release
+    script: pytest -m integration -v
+  - stage: integration
+    python: '3.7'
+    os: linux
+    dist: xenial
+    language: python
+    env:
+    - REQUIREMENTS=release
+    script: pytest -m integration -v
+  - stage: integration
+    os: linux
+    dist: xenial
+    language: python
+    env:
+    - REQUIREMENTS=lowest
+    script: pytest -m integration -v
+  - stage: integration
+    python: '3.6'
+    os: linux
+    dist: xenial
+    language: python
+    env:
+    - REQUIREMENTS=lowest
+    script: pytest -m integration -v
+  - stage: integration
+    python: '3.7'
+    os: linux
+    dist: xenial
+    language: python
+    env:
+    - REQUIREMENTS=lowest
+    script: pytest -m integration -v
+  - stage: test OSX
+    language: generic
+    sudo: true
+    os: osx
+    osx_image: xcode11.2
+  - stage: test OSX
+    language: generic
+    sudo: true
+    os: osx
+    osx_image: xcode10.1
+  - stage: "publish \U0001F40D"
+    python: 3.6
+    script: echo "Publishing on PyPI.io ..."
+    before_deploy: if [[ -z $TRAVIS_TAG ]]; then export TRAVIS_TAG=$(renku --version)
+      && git tag $TRAVIS_TAG; fi
+    deploy:
+    - provider: pypi
+      user:
+        secure: RPxGYNL+N6LQy1/TbXCFy9IDgZ05u/Qj6my/p1hSoBWG304se28njZ0zpTv5AGZF8G3dBeVjYbf4s+ytwUPoxK+1fBWlnnSuw4QUXWf339rzcMU4dDO2QX8/SvMDZFbJdE/1/6arlbXa8dTZobm1V0Q3AUTkl2AwXIg9jCaRcYb6n9lASRIiR1M2BK/+FTJG2qywB9kSyQ3giuobwwhQa4CAwDg5MoqL5KLFm2CxejvS0vpleaWqA/LJxeHTQqqd0CIk6V2NBETZ6V78MqdISai88nRafijT0iQ5LSWsy7R6CCpK7OFjHnvA7hGSYzs/BRpdABAk5a2yFbKKZErXoLvatjflMlj2OhHy/0Hlv6xEt1db1pwnjQQIiS62R/Gpx4DZAO8hGp6pT9g9xiifzlj4km9iOD4GY1g+A5A+ssEneBTvExJja4yAqJzAVu+XVDVqxVj+MOmpIcQkzT983+cVceoeczJ61sDuftQaAgcVqQACRE02fLszEtSJVFaq3vKu8dX2eMdiCk7GLdqNF9kfygagNC8eja6Yvr+Ft8kTwrjTBMC/D3xC584I8OTzmpNE/tfZHppfhiKXoU+FySdIGCPcSTGKUgljiz3sFk1JjjEBkGqBLAMaD8l5FsgQqR4zO/2IiwSst1Wx8owF+tiLlerDAC9b/ZFcqDpUab4=
+      password:
+        secure: f8f2175dg1FUt4mZCQ87tgwwUrFoxQ5CTcZkRnlCXJqagSMk9VmjR8/XXDE5cW48JPG0qdKQdvBtC81NCq+4MqC20HI9VtOdZWeT6Jej90XOZ742osC3pdIGMF4wrsd7+fs1AZkbzzjgB7FsJ42qW6rMa3vP4mXB4GJEel453Fs3Fw8tnR4PZ2ikOJ9fcmtMensjxh9oNMyAIMkYVGim5bWtpkpI1cddeThDBEKurj1IWEMonQw4lR2yLwZTEP6F/b43Cy4aNy+YkdJzJbM0KMJASLeeu8SxNTE7JCqrYc4irU3AzHzzy/FNVGdiw0F10kbK+yI7cPUqWfeoVjwqBJe8Hr0CXNNCsEYkjXkY9PE2m2x10z2UrTy+O1dNo/8sDoKvZBChuAPPtxp2I7/KujECWjzFvMunHEk3K940ZeSMo90xHKQllmA7umquNcPTXiM2l4nNHtolh5W2HdanvsEFhkY2Y2o9sRIytOa5bM+iS9RCL5GsZwLgGKdjfuqk0GF6KK4AIgi7WKtVM73vM7HQaAVRpyUWZ/h8Vu5IRdkORC20WwHZ/Jg6pUy3pkN1VlcKE5uanaPik/npq/uCGe9YC2bh5IoclmqdJUHrkzFPb+f8wRBvbof0zU1B6UMSdiug5oDk3a0Q8kk2AppfjTs7x8NHi3KYXdUphi9HrWQ=
+      distributions: sdist bdist_wheel
+      on:
+        all_branches: true
+    - provider: releases
+      api_key: "${GITHUB_TOKEN}"
+      on:
+        all_branches: true
+        tags: false
+  - stage: "brew \U0001F37A"
+    language: generic
+    sudo: true
+    os: osx
+    osx_image: xcode11.2
+    before_install: brew -v install jq
+    install: python -m pip install requests
+    script:
+    - echo PY_BREW_VERSION=$(python setup.py --version)
+    - make -e PY_BREW_VERSION=$(python setup.py --version) brew-commit-formula brew-build-bottle
+      brew-commit-bottle
+    deploy:
+    - provider: releases
+      api_key: "${GITHUB_TOKEN}"
+      file_glob: true
+      file: "*.bottle.*"
+      skip_cleanup: true
+      on:
+        all_branches: true
+    - provider: pages
+      skip-cleanup: true
+      github-token: "${GITHUB_TOKEN}"
+      repo: swissdatasciencecenter/homebrew-renku
+      target-branch: master
+      keep-history: true
+      local-dir: "/usr/local/Homebrew/Library/Taps/swissdatasciencecenter/homebrew-renku/"
+      on:
+        all_branches: true

Dockerfile renamed to Dockerfile.cli

@@ -1,4 +1,4 @@
-FROM python:3.6-alpine as base
+FROM python:3.7-alpine as base
 
 RUN apk add --no-cache git && \
     pip install --no-cache --upgrade pip

Dockerfile.svc

@@ -0,0 +1,18 @@
+FROM python:3.7-alpine
+
+RUN apk add --update --no-cache alpine-sdk g++ gcc linux-headers libxslt-dev python3-dev build-base openssl-dev libffi-dev git bash && \
+    pip install --no-cache --upgrade pip setuptools pipenv requirements-builder
+
+RUN apk add --no-cache --allow-untrusted \
+    --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/community \
+    --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \
+    --repository http://nl.alpinelinux.org/alpine/edge/community \
+    git-lfs && \
+    git lfs install
+
+COPY . /code/renku
+WORKDIR /code/renku
+RUN requirements-builder -e all --level=pypi setup.py > requirements.txt && pip install -r requirements.txt && pip install -e . && pip install gunicorn
+
+
+ENTRYPOINT ["gunicorn", "renku.service.entrypoint:app", "-b", "0.0.0.0:8080"]