Fix Unicode character error building Japanese PDF documents (python#149)

* fix Japanese PDF geneeration error
* various change for ja/lualatex
* Replace U+FFFD with '?'

Author: atsuoishimoto

build_docs.py

@@ -225,10 +225,40 @@ class Language:
     "-D latex_elements.fontenc=",
 )
 
-PLATEX_DEFAULT = (
-    "-D latex_engine=platex",
+LUALATEX_FOR_JP = (
+    "-D latex_engine=lualatex",
     "-D latex_elements.inputenc=",
     "-D latex_elements.fontenc=",
+    "-D latex_docclass.manual=ltjsbook",
+    "-D latex_docclass.howto=ltjsarticle",
+
+    # supress polyglossia warnings
+    "-D latex_elements.polyglossia=",
+    "-D latex_elements.fontpkg=",
+
+    # preamble
+    "-D latex_elements.preamble="
+
+    # Render non-Japanese letters with luatex
+    # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b
+    r"\\usepackage[noto-otf]{luatexja-preset}"
+    r"\\usepackage{newunicodechar}"
+    r"\\newunicodechar{^^^^212a}{K}"
+
+    # Workaround for the luatex-ja issue (Thanks to @jfbu)
+    # https://github.com/sphinx-doc/sphinx/issues/11179#issuecomment-1420715092
+    # https://osdn.net/projects/luatex-ja/ticket/47321
+    r"\\makeatletter"
+    r"\\titleformat{\\subsubsection}{\\normalsize\\py@HeaderFamily}"
+    r"{\\py@TitleColor\\thesubsubsection}{0.5em}{\\py@TitleColor}"
+    r"\\titleformat{\\paragraph}{\\normalsize\\py@HeaderFamily}"
+    r"{\\py@TitleColor\\theparagraph}{0.5em}{\\py@TitleColor}"
+    r"\\titleformat{\\subparagraph}{\\normalsize\\py@HeaderFamily}"
+    r"{\\py@TitleColor\\thesubparagraph}{0.5em}{\\py@TitleColor}"
+    r"\\makeatother"
+
+    # subpress warning: (fancyhdr)Make it at least 16.4pt
+    r"\\setlength{\\footskip}{16.4pt}"
 )
 
 XELATEX_WITH_FONTSPEC = (
@@ -257,7 +287,7 @@ class Language:
     Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC),
     Language("id", "id", "Indonesian", False, XELATEX_DEFAULT),
     Language("it", "it", "Italian", False, XELATEX_DEFAULT),
-    Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT, html_only=True),  # See https://github.com/python/python-docs-ja/issues/35
+    Language("ja", "ja", "Japanese", True, LUALATEX_FOR_JP),
     Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN),
     Language("pl", "pl", "Polish", False, XELATEX_DEFAULT),
     Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT),
@@ -726,6 +756,17 @@ def build(self):
                     "-D gettext_compact=0",
                 )
             )
+        if self.language.tag == "ja":
+            # Since luatex doesn't support \ufffd, replace \ufffd with '?'.
+            # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b
+            # Luatex already fixed this issue, so we can remove this once Texlive is updated.
+            # (https://github.com/TeX-Live/luatex/commit/eaa95ce0a141eaf7a02)
+            subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/?/g "
+                                    f"{locale_dirs}/ja/LC_MESSAGES/**/*.po",
+                                    shell=True)
+            subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/?/g "
+                                    f"{self.checkout}/Doc/**/*.rst", shell=True)
+
         if self.version.status == "EOL":
             sphinxopts.append("-D html_context.outdated=1")
         maketarget = (