Rendering courses from forks and content fragments caching (#362)

Merges #362

Author: mikicz

.gitignore

@@ -10,3 +10,4 @@
 
 venv
 _build
+.arca

.travis.yml

@@ -2,12 +2,23 @@ language: python
 python:
 - '3.6'
 cache:
-- pip
+  directories:
+  - $HOME/.cache/pip
+  - .arca/cache
 install:
 - pip install -r test_requirements.txt
+before_script:  # encrypted variables aren't accessible in PRs
+- 'if [ -n "$DOCKER_HUB_PASSWORD" ]; then docker login -u "$DOCKER_HUB_USERNAME" -p "$DOCKER_HUB_PASSWORD"; fi'
+- 'if [ -z "$DOCKER_HUB_PASSWORD" ]; then export ARCA_BACKEND_REGISTRY_PULL_ONLY=true; fi'
 script:
 - python -m pytest test_naucse
+- python -m naucse list_courses  # don't remove, there might be timeouts otherwise, see naucse/cli.py for details
 - python -m naucse freeze
+after_script:
+- echo "Suplementary logs:"
+- cat .arca/naucse.log
+# so Travis CI stores the whole log in its build log (https://github.com/travis-ci/travis-ci/issues/6018)
+- sleep 1
 deploy:
   provider: script
   skip_cleanup: true
@@ -18,3 +29,10 @@ deploy:
 env:
   global:
   - secure: pEZZeBdTSg0WUQYfwKZCiM+VMmNf2GyCCDa5Qcw2xGaNi5Z3rb3IYntY5ACKtDazgjzul+AqHra7iEx41yictXu4wYYt1nSY7g+ilBq/SlefiTi92ZRLH9elEChOa4A2njeuFIzwLBRr4Hdnn+21Vd/uxbC0wGhRC5c11ixBVC7Kn6zatT2TXUN+zF5a63NgUDrY/SPMV0jX3nozFHOxAQmZqlV1VL7O5ihct+mLYxB6MhS1NO/v30cMwyQm9nO0wBGJ1YH5/SKk9ThvFKH15d0xLVjZMfhJ/sgqpRnuRw4awH7SV7p2BoF7/07P8VWnCNvJ+wCipd1PqAk3COgqriWCBx80auaevIDxzky4CUk/efd3WYOA4MnzSAvcjbcQjojjsi64WWwVAcWuweTA8AEYMhVdqWlMbo8E3Oa0dZcyscBo+L26OdAdKNUqeNVQ84oIcPN/iUCitLB5GVOc0cIkn9wFer66jSuwK9bRfOTmV6NaRdLt78Edo+e1wXx0sdTVasL/rtIRTJAKKUtF8D/8RFfCNopOcceAyPr/Ay3JjVEFk2rTWeZls8E8os3TN6evVawx0t2xA0qXYb6IIrb0CFxv6sz/pU7vkuzS8g+pdQnfMmNtQ01sGGyedTT4YQRcgGKmWG+kolOXQlxJCGOndgz3VGN3DCppA5Ahbfo=
+  - ARCA_BACKEND=arca.backend.DockerBackend
+  - FORKS_ENABLED=true
+  - DOCKER_HUB_USERNAME=naucse
+  - secure: eg0Lj+IvZnNwf6rK7ZNsrhZs0V2ap9TylSOh3B+xthmwLBGB2fxtVQrjJEAimwoUEEzO6XAyGWuKa2RMtPUYx0758diEZCBT2Ct9aprhz2m9girLyUynAAfBumVv/0iMYqh9QShzfGLkF/f3Z/bkIheVhKV7jBCy5iHBLGzlSr2i4lZjFUE/lYNnQh+KCVx9HGKmSm4111x57ID+N73poe5mshWE6QYw4gSjjwdMW4SYcABwkARFHLNc1cuZYJKwXtMeN1MHlp2bjdJB2wUtoyb7b8QSoGYu87egVsAHby5+ccClecu9LC6DWgFvXs/pCBriqiHPk72TC0Zfx3pWpAhQcGZ85FqN4ZNL3q2hahodHbML38FjcbgQFNE9s1eT2q+y16R8Hy34HhvDCEx13+4xwjPHNkkjm/kQOzQ1cBwmiP52PgmsxOtNVu9sEOVmSUShf9oz15E0bpX8U+GG4cjJN15vGgfvg5lkpuVdRnoz+voi+9i3yDZ4XBD9E79LZxmHkZ5gEXdRWXnI6YmE9Ejq2csyLr4RPmicslLd24QWWSYmJ2MWzSaJmSFBiw2OZn880EqRlaVKSUS0kzvT5YYo6eb36cslSVK9QVBCAgzRuPH0GHKhnV5cPOQjgNEGISmLbTgMubxILNT077dcXL7cbB29YCN+zIyAoFH9gCs=
+sudo: required
+services:
+- docker

README.md

@@ -12,7 +12,7 @@ Chceš-li server spustit na svém počítači, např. proto, že se chceš zapoj
 nebo abys ho měl/a k dispozici i bez připojení k Intenetu, je potřeba ho
 nejdřív nainstalovat:
 
-* Vytvoř a aktivuj si [virtuální prostředí](http://naucse.python.cz/lessons/beginners/install/) v Pythonu 3.6. (Verze 3.5 by teoreticky měla taky fungovat, ale cokoli staršího fungovat nebude.)
+* Vytvoř a aktivuj si [virtuální prostředí](http://naucse.python.cz/lessons/beginners/install/) v Pythonu 3.6.
 * Přepni se do adresáře s kódem projektu.
 * Nainstaluj závislosti:
    ```console
@@ -37,7 +37,6 @@ Nainstalovanou aplikaci spustíš následovně:
   ```
 * Program vypíše adresu (např. `http://0.0.0.0:8003/`); tu navštiv v prohlížeči.
 
-
 ## Testy
 
 Chceš-li pustit testy, nainstaluj si závislosti:

naucse/__init__.py

@@ -1,10 +1,43 @@
+import logging
 import sys
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+from naucse.freezer import NaucseFreezer
+
 if sys.version_info[0] <3 :
     raise RuntimeError('We love Python 3.')
 
-from elsa import cli
-from naucse.routes import app
+from naucse.cli import cli
+from naucse.routes import app, lesson_static_generator
 
 
 def main():
-    cli(app, base_url='http://naucse.python.cz')
+    arca_log_path = Path(".arca/arca.log")
+    arca_log_path.parent.mkdir(exist_ok=True)
+    arca_log_path.touch()
+
+    naucse_log_path = Path(".arca/naucse.log")
+    naucse_log_path.touch()
+
+    def get_handler(path, **kwargs):
+        handler = RotatingFileHandler(path, **kwargs)
+        formatter = logging.Formatter("[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s")
+
+        handler.setLevel(logging.INFO)
+        handler.setFormatter(formatter)
+
+        return handler
+
+    logger = logging.getLogger("arca")
+    logger.addHandler(get_handler(arca_log_path, maxBytes=10000, backupCount=0))
+
+    logger = logging.getLogger("naucse")
+    logger.addHandler(get_handler(naucse_log_path))
+
+    freezer = NaucseFreezer(app)
+
+    # see the generator for details
+    freezer.register_generator(lesson_static_generator)
+
+    cli(app, base_url='http://naucse.python.cz', freezer=freezer)

naucse/cli.py

@@ -0,0 +1,72 @@
+import click
+import elsa
+
+from naucse.utils.routes import forks_enabled, does_course_return_info
+
+
+def cli(app, *, base_url=None, freezer=None):
+    """ Extends the elsa command line interface with a new command which prints all courses and runs
+        which are present with basic info about them.
+    """
+    elsa_group = elsa.cli(app, base_url=base_url, freezer=freezer, invoke_cli=False)
+
+    @click.group()
+    def naucse():
+        pass
+
+    @naucse.command()
+    @click.option("--forks-only", default=False, is_flag=True,
+                  help="Only list courses and runs from forks")
+    def list_courses(forks_only):
+        """ List all courses and runs and info about them.
+
+        Mainly useful for courses from forks, shows where do they sourced from and if
+        they return even the most basic information and will therefore be included in
+        list of courses/runs.
+
+        A practical benefit is that on Travis CI the docker images are pulled/built
+        in this command and the freezing won't timeout after the 10 minute limit if things are taking particularly long.
+        """
+        from naucse.routes import model
+
+        def canonical(course, suffix=""):
+            click.echo(f"  {course.slug}: {course.title}{suffix}")
+
+        def fork_invalid(course):
+            click.echo(f"  {course.slug}, from {course.repo}@{course.branch}: "
+                       f"Fork doesn't return basic info, will be ignored.")
+
+        def fork_valid(course, suffix=""):
+            click.echo(f"  {course.slug}, from {course.repo}@{course.branch}: {course.title}{suffix}")
+
+        click.echo(f"Courses:")
+
+        for course in model.courses.values():
+            if forks_only and not course.is_link():
+                continue
+
+            if not course.is_link():
+                canonical(course)
+            elif forks_enabled():
+                if does_course_return_info(course, force_ignore=True):
+                    fork_valid(course)
+                else:
+                    fork_invalid(course)
+
+        click.echo(f"Runs:")
+
+        for course in model.runs.values():
+            if forks_only and not course.is_link():
+                continue
+
+            if not course.is_link():
+                canonical(course, suffix=f" ({course.start_date} - {course.end_date})")
+            elif forks_enabled():
+                if does_course_return_info(course, ["start_date", "end_date"], force_ignore=True):
+                    fork_valid(course, suffix=f" ({course.start_date} - {course.end_date})")
+                else:
+                    fork_invalid(course)
+
+    cli = click.CommandCollection(sources=[naucse, elsa_group])
+
+    return cli()

naucse/freezer.py

@@ -0,0 +1,43 @@
+import contextlib
+
+from flask_frozen import UrlForLogger, Freezer
+
+from naucse.utils.routes import absolute_urls_to_freeze
+
+
+class AllLinksLogger(UrlForLogger):
+    """ AllLinksLogger primarily logs ``url_for`` calls, but yields urls from  ``absolute_urls_to_freeze`` as well.
+    """
+
+    def iter_calls(self):
+        """ Yields all logged urls and links parsed from content.
+            Unfortunately, ``yield from`` cannot be used as the queues are modified on the go.
+        """
+        while self.logged_calls or absolute_urls_to_freeze:
+            if self.logged_calls:
+                yield self.logged_calls.popleft()
+                # prefer urls from :atrr:`logged_calls` - so, ideally, cache is populated from the base repository
+                continue
+            if absolute_urls_to_freeze:
+                yield absolute_urls_to_freeze.popleft()
+
+
+@contextlib.contextmanager
+def temporary_url_for_logger(app):
+    """ A context manager which temporary adds a new UrlForLogger to the app and yields it, so it can be used
+        to get logged calls.
+    """
+    logger = UrlForLogger(app)
+
+    yield logger
+
+    # reverses the following operating from :class:`UrlForLogger`
+    # self.app.url_default_functions.setdefault(None, []).insert(0, logger)
+    app.url_default_functions[None].pop(0)
+
+
+class NaucseFreezer(Freezer):
+
+    def __init__(self, app):
+        super().__init__(app)
+        self.url_for_logger = AllLinksLogger(app)  # override the default url_for_logger with our modified version