diff --git a/pycco/generate_index.py b/pycco/generate_index.py index 7652466..00f9f9d 100644 --- a/pycco/generate_index.py +++ b/pycco/generate_index.py @@ -3,23 +3,22 @@ all documentation files generated by Pycco. """ import re -from os import path + +# from os import path +from pathlib import Path from pycco.compat import compat_items from pycco_resources import pycco_template -__all__ = ('generate_index',) +__all__ = ("generate_index",) def build_tree(file_paths, outdir): tree = {} for file_path in file_paths: - entry = { - 'path': file_path, - 'relpath': path.relpath(file_path, outdir) - } - path_steps = entry['relpath'].split(path.sep) + entry = {"path": file_path, "relpath": Path(file_path).relative_to(outdir)} + path_steps = entry["relpath"].parts add_file(entry, path_steps, tree) return tree @@ -39,7 +38,7 @@ def add_file(entry, path_steps, tree): add_file(entry, subpath, tree[node]) else: - tree[node]['entry'] = entry + tree[node]["entry"] = entry def generate_tree_html(tree): @@ -49,16 +48,18 @@ def generate_tree_html(tree): """ items = [] for node, subtree in sorted(compat_items(tree)): - if 'entry' in subtree: - html = u'
  • {}
    • '.format(subtree['entry']['relpath'], node) + if "entry" in subtree: + html = u'
  • {}
    • '.format( + subtree["entry"]["relpath"], node + ) else: - html = u'
    {}
    '.format( + html = u"
    {}
    ".format( node, generate_tree_html(subtree) ) items.append(html) - return ''.join(items) + return "".join(items) def generate_index(files, outdir): @@ -68,11 +69,13 @@ def generate_index(files, outdir): """ tree = build_tree(files, outdir) - rendered = pycco_template({ - "title": 'Index', - "stylesheet": 'pycco.css', - "sections": {'docs_html': generate_tree_html(tree)}, - "source": '', - }) + rendered = pycco_template( + { + "title": "Index", + "stylesheet": "pycco.css", + "sections": {"docs_html": generate_tree_html(tree)}, + "source": "", + } + ) return re.sub(r"__DOUBLE_OPEN_STACHE__", "{{", rendered).encode("utf-8") diff --git a/pycco/languages.py b/pycco/languages.py index fdf7c2d..ae18edd 100644 --- a/pycco/languages.py +++ b/pycco/languages.py @@ -13,15 +13,13 @@ DASH_DASH = "--" TRIPLE_QUOTE = '"""' + def lang(name, comment_symbol, multistart=None, multiend=None): """ Generate a language entry dictionary, given a name and comment symbol and optional start/end strings for multiline comments. """ - result = { - "name": name, - "comment_symbol": comment_symbol - } + result = {"name": name, "comment_symbol": comment_symbol} if multistart is not None and multiend is not None: result.update(multistart=multistart, multiend=multiend) return result @@ -31,46 +29,25 @@ def lang(name, comment_symbol, multistart=None, multiend=None): supported_languages = { ".coffee": lang("coffee-script", HASH, "###", "###"), - ".pl": lang("perl", HASH), - ".sql": lang("sql", DASH_DASH, SLASH_STAR, STAR_SLASH), - ".sh": lang("bash", HASH), - ".c": c_lang, - ".h": c_lang, - ".cl": c_lang, - ".cpp": lang("cpp", SLASH_SLASH), - ".js": lang("javascript", SLASH_SLASH, SLASH_STAR, STAR_SLASH), - ".rb": lang("ruby", HASH, "=begin", "=end"), - ".py": lang("python", HASH, TRIPLE_QUOTE, TRIPLE_QUOTE), - ".pyx": lang("cython", HASH, TRIPLE_QUOTE, TRIPLE_QUOTE), - ".scm": lang("scheme", ";;", "#|", "|#"), - ".lua": lang("lua", DASH_DASH, "--[[", "--]]"), - ".erl": lang("erlang", "%%"), - ".tcl": lang("tcl", HASH), - ".hs": lang("haskell", DASH_DASH, "{-", "-}"), - ".r": lang("r", HASH), ".R": lang("r", HASH), - ".jl": lang("julia", HASH, "#=", "=#"), - ".m": lang("matlab", "%", "%{", "%}"), - - ".do": lang("stata", SLASH_SLASH, SLASH_STAR, STAR_SLASH) - + ".do": lang("stata", SLASH_SLASH, SLASH_STAR, STAR_SLASH), } diff --git a/pycco/main.py b/pycco/main.py index 59f8f1c..6da28a2 100644 --- a/pycco/main.py +++ b/pycco/main.py @@ -43,15 +43,14 @@ python setup.py install """ -from __future__ import absolute_import, print_function - # Import our external dependencies. import argparse import os import re import sys import time -from os import path + +from pathlib import Path import pygments from pygments import formatters, lexers @@ -60,21 +59,23 @@ from pycco.generate_index import generate_index from pycco.languages import supported_languages from pycco_resources import css as pycco_css + # This module contains all of our static resources. from pycco_resources import pycco_template # === Main Documentation Generation Functions === -def generate_documentation(source, outdir=None, preserve_paths=True, - language=None, encoding="utf8"): +def generate_documentation( + source, outdir=None, preserve_paths=True, language=None, encoding="utf8" +): """ Generate the documentation for a source file by reading it in, splitting it up into comment/code sections, highlighting them for the appropriate language, and merging them into an HTML template. """ - if not outdir: + if outdir is None: raise TypeError("Missing the required 'outdir' keyword argument.") code = open(source, "rb").read().decode(encoding) return _generate_documentation(source, code, outdir, preserve_paths, language) @@ -87,7 +88,9 @@ def _generate_documentation(file_path, code, outdir, preserve_paths, language): language = get_language(file_path, code, language_name=language) sections = parse(code, language) highlight(sections, language, preserve_paths=preserve_paths, outdir=outdir) - return generate_html(file_path, sections, preserve_paths=preserve_paths, outdir=outdir) + return generate_html( + file_path, sections, preserve_paths=preserve_paths, outdir=outdir + ) def parse(code, language): @@ -113,39 +116,46 @@ def parse(code, language): if language["name"] == "python": for linenum, line in enumerate(lines[:2]): - if re.search(r'coding[:=]\s*([-\w.]+)', lines[linenum]): + if re.search(r"coding[:=]\s*([-\w.]+)", lines[linenum]): lines.pop(linenum) break def save(docs, code): if docs or code: - sections.append({ - "docs_text": docs, - "code_text": code - }) + sections.append({"docs_text": docs, "code_text": code}) # Setup the variables to get ready to check for multiline comments multi_line = False multi_string = False multistart, multiend = language.get("multistart"), language.get("multiend") - comment_matcher = language['comment_matcher'] + comment_matcher = language["comment_matcher"] for line in lines: process_as_code = False # Only go into multiline comments section when one of the delimiters is # found to be at the start of a line - if multistart and multiend \ - and any(line.lstrip().startswith(delim) or line.rstrip().endswith(delim) - for delim in (multistart, multiend)): + if ( + multistart + and multiend + and any( + line.lstrip().startswith(delim) or line.rstrip().endswith(delim) + for delim in (multistart, multiend) + ) + ): multi_line = not multi_line - if multi_line \ - and line.strip().endswith(multiend) \ - and len(line.strip()) > len(multiend): + if ( + multi_line + and line.strip().endswith(multiend) + and len(line.strip()) > len(multiend) + ): multi_line = False - if not line.strip().startswith(multistart) and not multi_line \ - or multi_string: + if ( + not line.strip().startswith(multistart) + and not multi_line + or multi_string + ): process_as_code = True @@ -158,46 +168,48 @@ def save(docs, code): else: # Get rid of the delimiters so that they aren't in the final # docs - line = line.replace(multistart, '') - line = line.replace(multiend, '') - docs_text += line.strip() + '\n' + line = line.replace(multistart, "") + line = line.replace(multiend, "") + docs_text += line.strip() + "\n" indent_level = re.match(r"\s*", line).group(0) if has_code and docs_text.strip(): save(docs_text, code_text[:-1]) - code_text = code_text.split('\n')[-1] - has_code = docs_text = '' + code_text = code_text.split("\n")[-1] + has_code = docs_text = "" elif multi_line: # Remove leading spaces - if re.match(r' {{{:d}}}'.format(len(indent_level)), line): - docs_text += line[len(indent_level):] + '\n' + if re.match(r" {{{:d}}}".format(len(indent_level)), line): + docs_text += line[len(indent_level) :] + "\n" else: - docs_text += line + '\n' + docs_text += line + "\n" elif re.match(comment_matcher, line): if has_code: save(docs_text, code_text) - has_code = docs_text = code_text = '' + has_code = docs_text = code_text = "" docs_text += re.sub(comment_matcher, "", line) + "\n" else: process_as_code = True if process_as_code: - if code_text and any(line.lstrip().startswith(x) - for x in ['class ', 'def ', '@']): + if code_text and any( + line.lstrip().startswith(x) for x in ["class ", "def ", "@"] + ): if not code_text.lstrip().startswith("@"): save(docs_text, code_text) - code_text = has_code = docs_text = '' + code_text = has_code = docs_text = "" has_code = True - code_text += line + '\n' + code_text += line + "\n" save(docs_text, code_text) return sections + # === Preprocessing the comments === @@ -221,35 +233,38 @@ def sanitize_section_name(name): def replace_crossref(match): # Check if the match contains an anchor - if '#' in match.group(1): - name, anchor = match.group(1).split('#') - return " [{}]({}#{})".format(name, - path.basename(destination(name, - preserve_paths=preserve_paths, - outdir=outdir)), - anchor) + if "#" in match.group(1): + name, anchor = match.group(1).split("#") + return " [{}]({}#{})".format( + name, + destination(name, preserve_paths=preserve_paths, outdir=outdir).stem, + anchor, + ) else: - return " [{}]({})".format(match.group(1), - path.basename(destination(match.group(1), - preserve_paths=preserve_paths, - outdir=outdir))) + return " [{}]({})".format( + match.group(1), + destination( + match.group(1), preserve_paths=preserve_paths, outdir=outdir + ).stem, + ) def replace_section_name(match): """ Replace equals-sign-formatted section names with anchor links. """ return '{lvl} {name}'.format( - lvl=re.sub('=', '#', match.group(1)), + lvl=re.sub("=", "#", match.group(1)), id=sanitize_section_name(match.group(2)), - name=match.group(2) + name=match.group(2), ) - comment = re.sub(r'^([=]+)([^=]+)[=]*\s*$', replace_section_name, comment) - comment = re.sub(r'(?
    "
+highlight_start = '
    '
 
 # The end of each Pygments highlight block.
 highlight_end = "
    "
@@ -482,8 +498,15 @@ def _flatten_sources(sources):
     return _sources
 
 
-def process(sources, preserve_paths=True, outdir=None, language=None,
-            encoding="utf8", index=False, skip=False):
+def process(
+    sources,
+    preserve_paths=True,
+    outdir=None,
+    language=None,
+    encoding="utf8",
+    index=False,
+    skip=False,
+):
     """
     For each source file passed as argument, generate the documentation.
     """
@@ -498,27 +521,27 @@ def process(sources, preserve_paths=True, outdir=None, language=None,
     # Proceed to generating the documentation.
     if sources:
         outdir = ensure_directory(outdir)
-        css = open(path.join(outdir, "pycco.css"), "wb")
-        css.write(pycco_css.encode(encoding))
-        css.close()
+        with open(outdir / "pycco.css", "wb") as css:
+            css.write(pycco_css.encode(encoding))
 
         generated_files = []
 
         def next_file():
             s = sources.pop(0)
             dest = destination(s, preserve_paths=preserve_paths, outdir=outdir)
-
-            try:
-                os.makedirs(path.split(dest)[0])
-            except OSError:
-                pass
+            dest.parent.mkdir(exist_ok=True, parents=True)
 
             try:
                 with open(dest, "wb") as f:
-                    f.write(generate_documentation(s, preserve_paths=preserve_paths,
-                                                   outdir=outdir,
-                                                   language=language,
-                                                   encoding=encoding))
+                    f.write(
+                        generate_documentation(
+                            s,
+                            preserve_paths=preserve_paths,
+                            outdir=outdir,
+                            language=language,
+                            encoding=encoding,
+                        )
+                    )
 
                 print("pycco: {} -> {}".format(s, dest))
                 generated_files.append(dest)
@@ -530,10 +553,11 @@ def next_file():
 
             if sources:
                 next_file()
+
         next_file()
 
         if index:
-            with open(path.join(outdir, "index.html"), "wb") as f:
+            with open(outdir / "index.html", "wb") as f:
                 f.write(generate_index(generated_files, outdir))
 
 
@@ -552,8 +576,7 @@ def monitor(sources, opts):
 
     # Watchdog operates on absolute paths, so map those to original paths
     # as specified on the command line.
-    absolute_sources = dict((os.path.abspath(source), source)
-                            for source in sources)
+    absolute_sources = dict((source.resolve(), source) for source in sources)
 
     class RegenerateHandler(watchdog.events.FileSystemEventHandler):
         """
@@ -567,16 +590,19 @@ def on_modified(self, event):
             # Re-generate documentation from a source file if it was listed on
             # the command line. Watchdog monitors whole directories, so other
             # files may cause notifications as well.
-            if event.src_path in absolute_sources:
-                process([absolute_sources[event.src_path]],
-                        outdir=opts.outdir,
-                        preserve_paths=opts.paths)
+            modified_pth = Path(event.src_path).resolve()
+            if modified_pth in absolute_sources:
+                process(
+                    [absolute_sources[modified_pth]],
+                    outdir=opts.outdir,
+                    preserve_paths=opts.paths,
+                )
 
     # Set up an observer which monitors all directories for files given on
     # the command line and notifies the handler defined above.
     event_handler = RegenerateHandler()
     observer = watchdog.observers.Observer()
-    directories = set(os.path.split(source)[0] for source in sources)
+    directories = set(source.parent for source in absolute_sources)
     for directory in directories:
         observer.schedule(event_handler, path=directory)
 
@@ -596,39 +622,69 @@ def main():
     """
 
     parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--paths', action='store_true',
-                      help='Preserve path structure of original files')
+    parser.add_argument(
+        "-p",
+        "--paths",
+        action="store_true",
+        help="Preserve path structure of original files",
+    )
 
-    parser.add_argument('-d', '--directory', action='store', type=str,
-                      dest='outdir', default='docs',
-                      help='The output directory that the rendered files should go to.')
+    parser.add_argument(
+        "-d",
+        "--directory",
+        action="store",
+        type=Path,
+        dest="outdir",
+        default=(Path.cwd() / "docs"),
+        help="The output directory that the rendered files should go to.",
+    )
 
-    parser.add_argument('-w', '--watch', action='store_true',
-                      help='Watch original files and re-generate documentation on changes')
+    parser.add_argument(
+        "-w",
+        "--watch",
+        action="store_true",
+        help="Watch original files and re-generate documentation on changes",
+    )
 
-    parser.add_argument('-l', '--force-language', action='store', type=str,
-                      dest='language', default=None,
-                      help='Force the language for the given files')
+    parser.add_argument(
+        "-l",
+        "--force-language",
+        action="store",
+        type=str,
+        dest="language",
+        default=None,
+        help="Force the language for the given files",
+    )
 
-    parser.add_argument('-i', '--generate_index', action='store_true',
-                      help='Generate an index.html document with sitemap content')
+    parser.add_argument(
+        "-i",
+        "--generate_index",
+        action="store_true",
+        help="Generate an index.html document with sitemap content",
+    )
 
-    parser.add_argument('-s', '--skip-bad-files', '-e', '--ignore-errors',
-                      action='store_true',
-                      dest='skip_bad_files',
-                      help='Continue processing after hitting a bad file')
+    parser.add_argument(
+        "-s",
+        "--skip-bad-files",
+        "-e",
+        "--ignore-errors",
+        action="store_true",
+        dest="skip_bad_files",
+        help="Continue processing after hitting a bad file",
+    )
 
-    parser.add_argument('sources', nargs='*')
+    parser.add_argument("sources", nargs="*", type=Path)
 
     args = parser.parse_args()
-    if args.outdir == '':
-        outdir = '.'
-    else:
-        outdir = args.outdir
 
-    process(args.sources, outdir=outdir, preserve_paths=args.paths,
-            language=args.language, index=args.generate_index,
-            skip=args.skip_bad_files)
+    process(
+        args.sources,
+        outdir=args.outdir,
+        preserve_paths=args.paths,
+        language=args.language,
+        index=args.generate_index,
+        skip=args.skip_bad_files,
+    )
 
     # If the -w / \-\-watch option was present, monitor the source directories
     # for changes and re-generate documentation for source files whenever they
@@ -638,7 +694,7 @@ def main():
             import watchdog.events
             import watchdog.observers  # noqa
         except ImportError:
-            sys.exit('The -w/--watch option requires the watchdog package.')
+            sys.exit("The -w/--watch option requires the watchdog package.")
 
         monitor(args.sources, args)
 
diff --git a/requirements.txt b/requirements.txt
index 51d4a95..a40dd68 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-pystache==0.5.4
-Pygments==2.5.2
-markdown==2.6.11
+pystache
+Pygments
+markdown
diff --git a/tests/test_pycco.py b/tests/test_pycco.py
index 0bca077..a7ba73f 100644
--- a/tests/test_pycco.py
+++ b/tests/test_pycco.py
@@ -2,7 +2,9 @@
 
 import copy
 import os
-import os.path
+
+import pathlib
+from pathlib import Path
 import tempfile
 import time
 
@@ -11,7 +13,15 @@
 import pycco.generate_index as generate_index
 import pycco.main as p
 from hypothesis import assume, example, given
-from hypothesis.strategies import booleans, lists, none, text, sampled_from, data
+from hypothesis.strategies import (
+    booleans,
+    lists,
+    none,
+    text,
+    sampled_from,
+    data,
+    from_regex,
+)
 from pycco.languages import supported_languages
 
 try:
@@ -20,9 +30,8 @@
     from mock import patch
 
 
-
-PYTHON = supported_languages['.py']
-PYCCO_SOURCE = 'pycco/main.py'
+PYTHON = supported_languages[".py"]
+PYCCO_SOURCE = "pycco/main.py"
 FOO_FUNCTION = """def foo():\n    return True"""
 
 
@@ -41,12 +50,14 @@ def test_shift(fragments, default):
 
 
 @given(text(), booleans(), text(min_size=1))
-@example("/foo", True, "0")
+@example("foo.py", True, "bar_dir")
 def test_destination(filepath, preserve_paths, outdir):
-    dest = p.destination(
-        filepath, preserve_paths=preserve_paths, outdir=outdir)
-    assert dest.startswith(outdir)
-    assert dest.endswith(".html")
+    dest = p.destination(filepath, preserve_paths=preserve_paths, outdir=outdir)
+    assert isinstance(dest, pathlib.PurePath), f"Expected pathlib obj, got {type(dest)}"
+    assert str(dest).startswith(
+        outdir
+    ), f"Expected start to be {outdir}, got {str(dest)}"
+    assert str(dest).endswith(".html")
 
 
 @given(data(), text())
@@ -61,7 +72,7 @@ def test_skip_coding_directive():
     source = "# -*- coding: utf-8 -*-\n" + FOO_FUNCTION
     parsed = p.parse(source, PYTHON)
     for section in parsed:
-        assert "coding" not in section['code_text']
+        assert "coding" not in section["code_text"]
 
 
 def test_multi_line_leading_spaces():
@@ -73,19 +84,21 @@ def test_multi_line_leading_spaces():
 
 
 def test_comment_with_only_cross_ref():
-    source = (
-        '''# ==Link Target==\n\ndef test_link():\n    """[[testing.py#link-target]]"""\n    pass'''
-    )
+    source = '''# ==Link Target==\n\ndef test_link():\n    """[[testing.py#link-target]]"""\n    pass'''
     sections = p.parse(source, PYTHON)
     p.highlight(sections, PYTHON, outdir=tempfile.gettempdir())
-    assert sections[1][
-        'docs_html'] == '
    testing.py
    '
+    assert (
+        sections[1]["docs_html"]
+        == '
    testing.py
    '
+    )
 
 
 @given(text(), text())
 def test_get_language_specify_language(source, code):
-    assert p.get_language(
-        source, code, language_name="python") == supported_languages['.py']
+    assert (
+        p.get_language(source, code, language_name="python")
+        == supported_languages[".py"]
+    )
 
     with pytest.raises(ValueError):
         p.get_language(source, code, language_name="non-existent")
@@ -114,16 +127,15 @@ def test_get_language_bad_code(code):
 
 @given(text(max_size=64))
 def test_ensure_directory(dir_name):
-    tempdir = os.path.join(tempfile.gettempdir(),
-                           str(int(time.time())), dir_name)
+    tempdir = Path(tempfile.gettempdir()) / str(int(time.time())) / dir_name
 
     # Use sanitization from function, but only for housekeeping. We
     # pass in the unsanitized string to the function.
     safe_name = p.remove_control_chars(dir_name)
 
-    if not os.path.isdir(safe_name) and os.access(safe_name, os.W_OK):
+    if not Path(safe_name).is_dir() and os.access(safe_name, os.W_OK):
         p.ensure_directory(tempdir)
-        assert os.path.isdir(safe_name)
+        assert Path(safe_name).is_dir()
 
 
 def test_ensure_multiline_string_support():
@@ -141,8 +153,8 @@ def x():
 
     docs_code_tuple_list = p.parse(code, PYTHON)
 
-    assert docs_code_tuple_list[0]['docs_text'] == ''
-    assert "#" not in docs_code_tuple_list[1]['docs_text']
+    assert docs_code_tuple_list[0]["docs_text"] == ""
+    assert "#" not in docs_code_tuple_list[1]["docs_text"]
 
 
 def test_indented_block():
@@ -154,9 +166,9 @@ def test_indented_block():
 '''
     parsed = p.parse(code, PYTHON)
     highlighted = p.highlight(parsed, PYTHON, outdir=tempfile.gettempdir())
-    pre_block = highlighted[0]['docs_html']
-    assert '
    ' in pre_block
-    assert '
    ' in pre_block
+    pre_block = highlighted[0]["docs_html"]
+    assert "
    " in pre_block
+    assert "
    " in pre_block
 
 
 def test_generate_documentation():
@@ -165,34 +177,42 @@ def test_generate_documentation():
 
 @given(booleans(), booleans(), data())
 def test_process(preserve_paths, index, data):
-    lang_name = data.draw(sampled_from([l["name"] for l in supported_languages.values()]))
-    p.process([PYCCO_SOURCE], preserve_paths=preserve_paths,
-              index=index,
-              outdir=tempfile.gettempdir(),
-              language=lang_name)
+    lang_name = data.draw(
+        sampled_from([l["name"] for l in supported_languages.values()])
+    )
+    p.process(
+        [PYCCO_SOURCE],
+        preserve_paths=preserve_paths,
+        index=index,
+        outdir=tempfile.gettempdir(),
+        language=lang_name,
+    )
 
 
-@patch('pygments.lexers.guess_lexer')
+@patch("pygments.lexers.guess_lexer")
 def test_process_skips_unknown_languages(mock_guess_lexer):
     class Name:
-        name = 'this language does not exist'
+        name = "this language does not exist"
+
     mock_guess_lexer.return_value = Name()
 
     with pytest.raises(ValueError):
-        p.process(['LICENSE'], outdir=tempfile.gettempdir(), skip=False)
+        p.process(["LICENSE"], outdir=tempfile.gettempdir(), skip=False)
 
-    p.process(['LICENSE'], outdir=tempfile.gettempdir(), skip=True)
+    p.process(["LICENSE"], outdir=tempfile.gettempdir(), skip=True)
 
 
 one_or_more_chars = text(min_size=1, max_size=255)
 paths = lists(one_or_more_chars, min_size=1, max_size=30)
+
+
 @given(
     lists(paths, min_size=1, max_size=255),
-    lists(one_or_more_chars, min_size=1, max_size=255)
+    lists(one_or_more_chars, min_size=1, max_size=255),
 )
 def test_generate_index(path_lists, outdir_list):
-    file_paths = [os.path.join(*path_list) for path_list in path_lists]
-    outdir = os.path.join(*outdir_list)
+    file_paths = [Path(*path_list) for path_list in path_lists]
+    outdir = Path(*outdir_list)
     generate_index.generate_index(file_paths, outdir=outdir)