Merge pull request #403 – Move absolute_urls_to_freeze and CLI

#403

Author: encukou

naucse/__init__.py

@@ -1,43 +1,4 @@
-import logging
 import sys
-from logging.handlers import RotatingFileHandler
-from pathlib import Path
-
-from naucse.freezer import NaucseFreezer
 
 if sys.version_info[0] <3 :
     raise RuntimeError('We love Python 3.')
-
-from naucse.cli import cli
-from naucse.views import app, lesson_static_generator
-
-
-def main():
-    arca_log_path = Path(".arca/arca.log")
-    arca_log_path.parent.mkdir(exist_ok=True)
-    arca_log_path.touch()
-
-    naucse_log_path = Path(".arca/naucse.log")
-    naucse_log_path.touch()
-
-    def get_handler(path, **kwargs):
-        handler = RotatingFileHandler(path, **kwargs)
-        formatter = logging.Formatter("[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s")
-
-        handler.setLevel(logging.INFO)
-        handler.setFormatter(formatter)
-
-        return handler
-
-    logger = logging.getLogger("arca")
-    logger.addHandler(get_handler(arca_log_path, maxBytes=10000, backupCount=0))
-
-    logger = logging.getLogger("naucse")
-    logger.addHandler(get_handler(naucse_log_path))
-
-    freezer = NaucseFreezer(app)
-
-    # see the generator for details
-    freezer.register_generator(lesson_static_generator)
-
-    cli(app, base_url='https://naucse.python.cz', freezer=freezer)

naucse/__main__.py

@@ -1,3 +1,3 @@
-from naucse import main
+from naucse.cli import main
 
 main()

naucse/cli.py

@@ -1,7 +1,14 @@
+import logging
+from pathlib import Path
+from logging.handlers import RotatingFileHandler
+
 import click
 import elsa
 
 from naucse.utils.views import forks_enabled, does_course_return_info
+from naucse.views import app, lesson_static_generator
+
+from naucse.freezer import NaucseFreezer
 
 
 def cli(app, *, base_url=None, freezer=None):
@@ -71,3 +78,34 @@ def fork_valid(course, suffix=""):
     cli = click.CommandCollection(sources=[naucse, elsa_group])
 
     return cli()
+
+
+def main():
+    arca_log_path = Path(".arca/arca.log")
+    arca_log_path.parent.mkdir(exist_ok=True)
+    arca_log_path.touch()
+
+    naucse_log_path = Path(".arca/naucse.log")
+    naucse_log_path.touch()
+
+    def get_handler(path, **kwargs):
+        handler = RotatingFileHandler(path, **kwargs)
+        formatter = logging.Formatter("[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s")
+
+        handler.setLevel(logging.INFO)
+        handler.setFormatter(formatter)
+
+        return handler
+
+    logger = logging.getLogger("arca")
+    logger.addHandler(get_handler(arca_log_path, maxBytes=10000, backupCount=0))
+
+    logger = logging.getLogger("naucse")
+    logger.addHandler(get_handler(naucse_log_path))
+
+    freezer = NaucseFreezer(app)
+
+    # see the generator for details
+    freezer.register_generator(lesson_static_generator)
+
+    cli(app, base_url='https://naucse.python.cz', freezer=freezer)

naucse/freezer.py

@@ -1,27 +1,38 @@
 import contextlib
+from collections import deque
 
+from flask import current_app
 from flask_frozen import UrlForLogger, Freezer
 
-from naucse.utils.views import absolute_urls_to_freeze
+def record_url(url):
+    """Logs that `url` should be included in the resulting static site"""
+    urls_to_freeze = current_app.config.get('NAUCSE_ABSOLUTE_URLS_TO_FREEZE')
+    if urls_to_freeze is not None:
+        urls_to_freeze.append(url)
 
 
 class AllLinksLogger(UrlForLogger):
     """Logs ``url_for`` calls, but yields urls from ``absolute_urls_to_freeze`` as well.
     """
 
+    def __init__(self, app, urls_to_freeze):
+        super().__init__(app)
+        self.naucse_urls_to_freeze = urls_to_freeze
+
     def iter_calls(self):
         """Yield all logged urls and links parsed from content.
         """
         # Unfortunately, ``yield from`` cannot be used as the queues are
         # modified on the go.
-        while self.logged_calls or absolute_urls_to_freeze:
-            if self.logged_calls:
+        while self.logged_calls or self.naucse_urls_to_freeze:
+            while self.logged_calls:
                 yield self.logged_calls.popleft()
-                # prefer urls from :atrr:`logged_calls` - so, ideally,
-                # cache is populated from the base repository
-                continue
-            if absolute_urls_to_freeze:
-                yield absolute_urls_to_freeze.popleft()
+            # Prefer URLs from logged_calls - ideally, cache is populated
+            # from the base repository.
+            # That means we only yield from urls_to_freeze
+            # if there are no logged_calls.
+            if self.naucse_urls_to_freeze:
+                yield self.naucse_urls_to_freeze.popleft()
 
 
 @contextlib.contextmanager
@@ -45,5 +56,10 @@ class NaucseFreezer(Freezer):
     def __init__(self, app):
         super().__init__(app)
 
+        urls_to_freeze = deque()
+
+        with app.app_context():
+            app.config['NAUCSE_ABSOLUTE_URLS_TO_FREEZE'] = urls_to_freeze
+
         # override the default url_for_logger with our modified version
-        self.url_for_logger = AllLinksLogger(app)
+        self.url_for_logger = AllLinksLogger(app, urls_to_freeze)

naucse/models.py

@@ -14,7 +14,6 @@
 import naucse.utils.views
 from naucse.utils.models import Model, YamlProperty, DataProperty, DirProperty, MultipleModelDirProperty, ForkProperty
 from naucse.utils.models import reify, arca
-from naucse.utils.views import absolute_urls_to_freeze
 from naucse.validation import AllowedElementsParser
 from naucse.templates import setup_jinja_env, vars_functions
 from naucse.utils.markdown import convert_markdown
@@ -644,11 +643,6 @@ def render(self, page_type, *args, **kwargs):
         if page_type != "calendar_ics" and result.output["content"] is not None:
             allowed_elements_parser.reset_and_feed(result.output["content"])
 
-        if "urls" in result.output:
-            # Freeze URLs generated by the code in fork, but only if
-            # they start with the slug of the course
-            absolute_urls_to_freeze.extend([url for url in result.output["urls"] if url.startswith(f"/{self.slug}/")])
-
         return result.output
 
     def render_course(self, **kwargs):
@@ -706,9 +700,9 @@ def validate_link(link, key):
         for link_type in "prev_link", "session_link", "next_link":
             link = result.output.get(link_type)
 
-            if isinstance(link, dict) and validate_link(link, "url") and validate_link(link, "title"):
-                if link["url"].startswith(f"/{self.slug}/"):
-                    absolute_urls_to_freeze.append(link["url"])
+            if (isinstance(link, dict) and
+                    validate_link(link, "url") and
+                    validate_link(link, "title")):
                 to_return.append(link)
             else:
                 to_return.append(None)

naucse/utils/views.py

@@ -2,14 +2,12 @@
 import hashlib
 import json
 import os
-from collections import deque, defaultdict
+from collections import defaultdict
 from pathlib import Path
 
 from arca.exceptions import PullError, BuildError, RequirementsMismatch
 from arca.utils import get_hash_for_file
 
-absolute_urls_to_freeze = deque()
-
 
 def get_recent_runs(course):
     """Build a list of "recent" runs based on a course.

naucse/views.py

@@ -16,7 +16,7 @@
 from werkzeug.local import LocalProxy
 
 from naucse import models
-from naucse.freezer import temporary_url_for_logger
+from naucse.freezer import temporary_url_for_logger, record_url
 from naucse.models import allowed_elements_parser
 from naucse.templates import setup_jinja_env, vars_functions
 from naucse.urlconverters import register_url_converters
@@ -26,7 +26,7 @@
 from naucse.utils.models import arca
 from naucse.utils.views import get_recent_runs, list_months
 from naucse.utils.views import does_course_return_info
-from naucse.utils.views import absolute_urls_to_freeze, raise_errors_from_forks
+from naucse.utils.views import raise_errors_from_forks
 from naucse.utils.views import page_content_cache_key, get_edit_info
 from naucse.validation import DisallowedStyle, DisallowedElement, InvalidHTML
 
@@ -276,6 +276,13 @@ def lesson_static_generator():
             yield from lesson_static_generator_dir(lesson.slug, static, static)
 
 
+def record_content_urls(data_from_fork, prefix):
+    # Freeze URLs generated by the code in fork, but only if
+    # they start with the given prefix
+    for url in data_from_fork.get("urls", ()):
+        if url.startswith(prefix):
+            record_url(url)
+
 def course_content(course):
     def lesson_url(lesson, *args, **kwargs):
         if kwargs.get("page") == "index":
@@ -319,6 +326,7 @@ def course(course):
                 root_slug=model.meta.slug,
                 travis_build_id=os.environ.get("TRAVIS_BUILD_ID"),
             )
+        record_content_urls(data_from_fork, f"/{course.slug}/")
         kwargs = {
             "course_content": content,
             "edit_info": edit_info,
@@ -464,7 +472,6 @@ def content_creator():
                     subpage_url=subpage_url,
                     vars=variables
                 )
-
             absolute_urls = [url_for(logged[0], **logged[1]) for logged in logger.logged_calls]
 
         relative_urls = [get_relative_url(request.path, x) for x in absolute_urls]
@@ -485,11 +492,10 @@ def content_creator():
     content_key = page_content_cache_key(Repo("."), lesson.slug, page.slug, solution, variables)
     cached = arca.region.get_or_create(content_key, content_creator)
 
-    # The urls are added twice to ``absolute_urls_to_freeze``
-    # when the content is created.
+    # The urls are recorded twice when the content is created.
     # But it doesn't matter, duplicate URLs are skipped.
-    absolute_urls = [urljoin(request.path, x) for x in cached["urls"]]
-    absolute_urls_to_freeze.extend(absolute_urls)
+    for x in cached["urls"]:
+        record_url(urljoin(request.path, x))
 
     return cached
 
@@ -530,20 +536,23 @@ def course_page(course, lesson, page, solution=None):
             if content_offer:
                 fork_kwargs["content_key"] = content_key
 
-            data_from_fork = course.render_page(lesson_slug, page, solution, **fork_kwargs)
+            data_from_fork = course.render_page(
+                lesson_slug, page, solution,
+                **fork_kwargs)
+            record_content_urls(data_from_fork, f"/{course.slug}/")
 
             content = data_from_fork["content"]
 
             if content is None:
                 # the offer was accepted
                 content = content_offer["content"]
-                absolute_urls_to_freeze.extend([urljoin(request.path, x)
-                                                for x in content_offer["urls"]])
+                for x in content_offer["urls"]:
+                    record_url(urljoin(request.path, x))
             else:
                 # the offer was rejected or the the fragment was not in cache
                 arca.region.set(content_key, {"content": content, "urls": data_from_fork["content_urls"]})
-                absolute_urls_to_freeze.extend([urljoin(request.path, x)
-                                                for x in data_from_fork["content_urls"]])
+                for x in data_from_fork["content_urls"]:
+                    record_url(urljoin(request.path, x))
 
             # compatibility
             page = process_page_data(data_from_fork.get("page"))
@@ -573,8 +582,17 @@ def course_page(course, lesson, page, solution=None):
                     title = '{}: {}'.format(course.title, page.title)
 
                     try:
-                        prev_link, session_link, next_link = course.get_footer_links(lesson.slug, page_slug,
-                                                                                     request_url=request.path)
+                        footer_links = course.get_footer_links(
+                            lesson.slug,
+                            page_slug,
+                            request_url=request.path,
+                        )
+                        for link in footer_links:
+                            _prefix = f"/{course.slug}/"
+                            if link and link["url"].startswith(_prefix):
+                                record_url(link["url"])
+                        prev_link, session_link, next_link = footer_links
+
                     except POSSIBLE_FORK_EXCEPTIONS as e:
                         if raise_errors_from_forks():
                             raise
@@ -726,7 +744,9 @@ def session_coverpage(course, session, coverpage):
         naucse.utils.views.forks_raise_if_disabled()
 
         try:
-            data_from_fork = course.render_session_coverpage(session, coverpage, request_url=request.path)
+            data_from_fork = course.render_session_coverpage(
+                session, coverpage, request_url=request.path)
+            record_content_urls(data_from_fork, f"/{course.slug}/")
 
             content = data_from_fork.get("content")
             if content is None:
@@ -790,6 +810,7 @@ def course_calendar(course):
 
         try:
             data_from_fork = course.render_calendar(request_url=request.path)
+            record_content_urls(data_from_fork, f"/{course.slug}/")
 
             course = process_course_data(data_from_fork.get("course"), slug=course.slug)
             edit_info = links.process_edit_info(data_from_fork.get("edit_info"))
@@ -865,7 +886,9 @@ def course_calendar_ics(course):
         naucse.utils.views.forks_raise_if_disabled()
 
         try:
-            data_from_fork = course.render_calendar_ics(request_url=request.path)
+            data_from_fork = course.render_calendar_ics(
+                request_url=request.path)
+            record_content_urls(data_from_fork, f"/{course.slug}/")
 
             calendar = data_from_fork.get("calendar")
 

test_naucse/test_forks.py

@@ -183,7 +183,7 @@ def client(model, mocker):
         delattr(model, "safe_run_years")
 
     mocker.patch("naucse.views._cached_model", model)
-    from naucse import app
+    from naucse.views import app
     app.testing = True
     yield app.test_client()