Add renderer for Sphinx .rst files, for user manual previews #2

Merged
Brecht Van Lommel merged 1 commits from brecht/gitea-custom:sphinx into develop 2023-03-09 18:18:51 +01:00
7 changed files with 395 additions and 0 deletions

28
sphinx/README.md Normal file
View File

@ -0,0 +1,28 @@
# Sphinx RST to HTML Preview
Command for generating previews of RST files on projects.blender.org.
The template is adapted from the Blender manual to support the same extensions.
### Deployment
Install dependencies.
pip3 install -r requirements.txt
Add to Gitea app.ini.
[markup.restructuredtext]
ENABLED = true
FILE_EXTENSIONS = .rst
RENDER_COMMAND = "timeout 30s ./custom/sphinx/sphinx_to_html.py --user sphinx --user-work-dir /path/to/dir"
IS_INPUT_FILE = true
[repository.editor]
LINE_WRAP_EXTENSIONS = .txt,.md,.markdown,.mdown,.mkd,.rst
PREVIEWABLE_FILE_MODES = markdown,restructuredtext
The `sphinx` user is required for sandboxing of sphinx-build which we do not
assume to be secure. The work directory should be writable by both the gitea
user and sphinx user, with the sphinx user having as little access as possible
to other directories.

1
sphinx/requirements.txt Normal file
View File

@ -0,0 +1 @@
sphinx==6.1.3

110
sphinx/sphinx_to_html.py Executable file
View File

@ -0,0 +1,110 @@
#!/usr/bin/python3
import argparse
import html
import os
import pathlib
import re
import shutil
import subprocess
import sys
import tempfile
parser = argparse.ArgumentParser(prog="sphinx_to_html")
parser.add_argument("filename_rst", help="Input .rst file")
parser.add_argument("--user", help="Run sphinx as another user", type=str)
parser.add_argument("--user-work-dir", help="Do work in specified folder accessible by user", type=str)
args = parser.parse_args()
base_url = "https://projects.blender.org"
local_url = "http://localhost:3000"
placeholder_url = "https://placeholder.org"
# Gitea sets this environment variable with the URL prefix for the current file.
gitea_prefix = os.environ.get("GITEA_PREFIX_SRC", "")
if gitea_prefix.startswith(base_url):
gitea_prefix = gitea_prefix[len(base_url):]
if gitea_prefix.startswith(local_url):
gitea_prefix = gitea_prefix[len(local_url):]
if len(gitea_prefix):
org, repo, view, ref, branch = gitea_prefix.strip('/').split('/')[:5]
doc_url = f"{base_url}/{org}/{repo}/{view}/{ref}/{branch}"
image_url = f"{base_url}/{org}/{repo}/raw/{ref}/{branch}"
else:
doc_url = ""
image_url = ""
# Set up temporary directory with sphinx configuration.
with tempfile.TemporaryDirectory(dir=args.user_work_dir) as tmp_dir:
work_dir = pathlib.Path(tmp_dir) / "work"
script_dir = pathlib.Path(__file__).parent.resolve()
shutil.copytree(script_dir / "template", work_dir)
page_filepath = work_dir / "contents.rst"
shutil.copyfile(args.filename_rst, page_filepath)
page_contents = page_filepath.read_text()
# Turn links into external links since internal links are not found and stripped.
def doc_link(matchobj):
return f"`{matchobj.group(1)}<{doc_url}/{matchobj.group(2).strip('/')}.rst>`_"
def ref_link(matchobj):
return f"`{matchobj.group(1)} <{placeholder_url}>`_"
def term_link(matchobj):
return f"`{matchobj.group(1)} <{placeholder_url}>`_"
def figure_link(matchobj):
return f"figure:: {image_url}/{matchobj.group(1).strip('/')}"
def image_link(matchobj):
return f"image:: {image_url}/{matchobj.group(1).strip('/')}"
page_contents = re.sub(":doc:`(.*)<(.+)>`", doc_link, page_contents)
page_contents = re.sub(":ref:`(.+)<(.+)>`", ref_link, page_contents)
page_contents = re.sub(":ref:`(.+)`", ref_link, page_contents)
page_contents = re.sub(":term:`(.+)`", term_link, page_contents)
page_contents = re.sub("figure:: (.+)", figure_link, page_contents)
page_contents = re.sub("image:: (.+)", image_link, page_contents)
# Disable include directives and raw for security. They are already disabled
# by docutils.py, this is just to be extra careful.
def include_directive(matchobj):
return f"warning:: include directives disabled: {html.escape(matchobj.group(1))}"
def raw_directive(matchobj):
return f"warning:: raw disabled: {html.escape(matchobj.group(1))}"
page_contents = re.sub("literalinclude::.*", include_directive, page_contents)
page_contents = re.sub("include::(.*)", include_directive, page_contents)
page_contents = re.sub("raw::(.*)", raw_directive, page_contents)
page_filepath.write_text(page_contents)
# Debug processed RST
# print(html.escape(page_contents).replace('\n', '<br/>\n'))
# sys.exit(0)
# Run sphinx-build.
out_dir = work_dir / "out"
out_filepath = out_dir / "contents.html"
sphinx_cmd = ["sphinx-build", "-b", "html", work_dir, out_dir]
if args.user:
result = subprocess.run(sphinx_cmd, capture_output=True, user=args.user)
else:
result = subprocess.run(sphinx_cmd, capture_output=True)
# Output errors.
error = result.stderr.decode("utf-8", "ignore").strip()
if len(error):
error = error.replace(str(page_filepath) + ":", "")
error = html.escape(error)
print("<h2>Sphinx Warnings</h2>\n")
print(f"<pre>{error}</pre>")
print("<p>Note the preview is not accurate and warnings may not indicate real issues.</p>")
# Output contents of body.
if result.returncode == 0 and out_filepath.is_file():
contents = out_filepath.read_text()
body = contents.split("<body>")[1].split("</body>")[0]
body = body.replace(placeholder_url, "#")
body = body.replace('href="http', 'target="_blank" href="http')
print(body)

22
sphinx/template/conf.py Normal file
View File

@ -0,0 +1,22 @@
# Configuration file for RST preview.
import os
import sys
# Extensions
sys.path.append(os.path.abspath('exts'))
extensions = [
'reference',
'peertube',
'sphinx.ext.mathjax',
'sphinx.ext.intersphinx',
'sphinx.ext.todo',
]
# Project
project = 'projects.blender.org'
root_doc = 'contents'
# Theme: epub hides all navigation, sidebars, footers.
html_theme = 'epub'
html_permalinks = False

View File

@ -0,0 +1,6 @@
# Disable file inclusion and raw HTML for security.
# https://docutils.sourceforge.io/docs/howto/security.html
[parsers]
[restructuredtext parser]
file-insertion-enabled:false
raw-enabled:false

160
sphinx/template/exts/peertube.py Executable file
View File

@ -0,0 +1,160 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
from docutils import nodes
from docutils.parsers.rst import directives, Directive
from sphinx.environment import BuildEnvironment
from sphinx.locale import __
from sphinx.util import logging
logger = logging.getLogger(__name__)
def get_size(d, key):
if key not in d:
return None
m = re.match(r"(\d+)(|%|px)$", d[key])
if not m:
raise ValueError("invalid size %r" % d[key])
return int(m.group(1)), m.group(2) or "px"
def css(d):
return "; ".join(sorted("%s: %s" % kv for kv in d.items()))
class peertube(nodes.General, nodes.Element):
pass
def visit_peertube_node(self, node):
instance = node["instance"]
aspect = node["aspect"]
width = node["width"]
height = node["height"]
if not (self.config.peertube_instance or instance):
logger.warning(__("No peertube instance defined"))
return
if instance is None:
instance = self.config.peertube_instance
if aspect is None:
aspect = 16, 9
div_style = {}
if (height is None) and (width is not None) and (width[1] == "%"):
div_style = {
"padding-bottom": "%f%%" % (width[0] * aspect[1] / aspect[0]),
"width": "%d%s" % width,
"position": "relative",
}
style = {
"position": "absolute",
"top": "0",
"left": "0",
"width": "100%",
"height": "100%",
}
attrs = {
"src": instance + "videos/embed/%s" % node["id"],
"style": css(style),
}
else:
if width is None:
if height is None:
width = 560, "px"
else:
width = height[0] * aspect[0] / aspect[1], "px"
if height is None:
height = width[0] * aspect[1] / aspect[0], "px"
style = {
"width": "%d%s" % width,
"height": "%d%s" % (height[0], height[1]),
}
attrs = {
"src": instance + "videos/embed/%s" % node["id"],
"style": css(style),
}
attrs["allowfullscreen"] = "true"
div_attrs = {
"CLASS": "peertube_wrapper",
"style": css(div_style),
}
self.body.append(self.starttag(node, "div", **div_attrs))
self.body.append(self.starttag(node, "iframe", **attrs))
self.body.append("</iframe></div>")
def depart_peertube_node(self, node):
pass
def visit_peertube_node_latex(self, node):
instance = node["instance"]
if not (self.config.peertube_instance or instance):
logger.warning(__("No peertube instance defined"))
return
if instance is None:
instance = self.config.peertube_instance
self.body.append(
r'\begin{quote}\begin{center}\fbox{\url{' +
instance +
r'videos/watch/%s}}\end{center}\end{quote}' %
node['id'])
class PeerTube(Directive):
has_content = True
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = False
option_spec = {
"instance": directives.unchanged,
"width": directives.unchanged,
"height": directives.unchanged,
"aspect": directives.unchanged,
}
def run(self):
instance = self.options.get("instance")
if "aspect" in self.options:
aspect = self.options.get("aspect")
m = re.match(r"(\d+):(\d+)", aspect)
if m is None:
raise ValueError("invalid aspect ratio %r" % aspect)
aspect = tuple(int(x) for x in m.groups())
else:
aspect = None
width = get_size(self.options, "width")
height = get_size(self.options, "height")
return [peertube(id=self.arguments[0], instance=instance, aspect=aspect, width=width, height=height)]
def unsupported_visit_peertube(self, node):
self.builder.warn('PeerTube: unsupported output format (node skipped)')
raise nodes.SkipNode
_NODE_VISITORS = {
'html': (visit_peertube_node, depart_peertube_node),
'latex': (visit_peertube_node_latex, depart_peertube_node),
'man': (unsupported_visit_peertube, None),
'texinfo': (unsupported_visit_peertube, None),
'text': (unsupported_visit_peertube, None)
}
def setup(app):
app.add_node(peertube, **_NODE_VISITORS)
app.add_directive("peertube", PeerTube)
app.add_config_value('peertube_instance', "", True, [str])
return {
'parallel_read_safe': True,
'parallel_write_safe': True,
}

View File

@ -0,0 +1,68 @@
from docutils import nodes
from docutils.nodes import Element, Node
from docutils.parsers.rst import directives
from docutils.parsers.rst.directives.admonitions import BaseAdmonition
from sphinx.locale import _
from sphinx.util.docutils import SphinxDirective
class refbox(nodes.Admonition, nodes.Element):
pass
def visit_refbox_node(self, node):
self.visit_admonition(node)
def depart_refbox_node(self, node):
self.depart_admonition(node)
class ReferenceDirective(BaseAdmonition, SphinxDirective):
node_class = refbox
has_content = True
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = False
option_spec = {
'class': directives.class_option,
'name': directives.unchanged,
}
def run(self):
if not self.options.get('class'):
self.options['class'] = ['refbox']
(reference,) = super().run()
if isinstance(reference, nodes.system_message):
return [reference]
elif isinstance(reference, refbox):
reference.insert(0, nodes.title(text=_('Reference')))
reference['docname'] = self.env.docname
self.add_name(reference)
self.set_source_info(reference)
self.state.document.note_explicit_target(reference)
return [reference]
else:
raise RuntimeError # never reached here
def setup(app):
app.add_node(
refbox,
html=(visit_refbox_node, depart_refbox_node),
latex=(visit_refbox_node, depart_refbox_node),
text=(visit_refbox_node, depart_refbox_node),
man=(visit_refbox_node, depart_refbox_node),
texinfo=(visit_refbox_node, depart_refbox_node),
)
app.add_directive('reference', ReferenceDirective)
return {
'version': '1.0',
'parallel_read_safe': True,
'parallel_write_safe': True,
}