Commit e3842e88 authored by Deimos's avatar Deimos

Enable cmark-gfm extensions (table, strikethrough)

An example was recently added to the github cmark repo to show how to
set up the extensions from Python, so this is heavily based on that
code:
https://github.com/github/cmark/blob/master/wrappers/wrapper_ext.py

This should also fix a memory leak, since I wasn't manually freeing the
returned buffer (as the library recommends that you do).
parent a5664b85
......@@ -28,6 +28,32 @@ def test_basic_markdown_unescaped():
assert '<' not in sanitized
def test_strikethrough():
"""Ensure strikethrough works and doesn't turn into a group link."""
markdown = "This ~should not~ should work"
processed = convert_markdown_to_safe_html(markdown)
assert '<del>' in processed
assert '<a' not in processed
def test_table():
"""Ensure table markdown works."""
markdown = (
'|Header 1|Header 2|Header 3|\n'
'|--------|-------:|:------:|\n'
'|1 - 1 |1 - 2 |1 - 3 |\n'
'|2 - 1|2 - 2|2 - 3|\n'
)
processed = convert_markdown_to_safe_html(markdown)
assert '<table>' in processed
assert processed.count('<tr') == 3
assert processed.count('<td') == 6
assert 'align="right"' in processed
assert 'align="center"' in processed
def test_deliberate_ordered_list():
"""Ensure a "deliberate" ordered list works."""
markdown = (
......@@ -215,14 +241,6 @@ def test_approximately_tilde_not_linkified():
assert '<a' not in processed
def test_strikethrough_attempt_not_linkified():
"""Ensure someone trying to do strikethrough doesn't get a link."""
markdown = "This ~should~ shouldn't work"
processed = convert_markdown_to_safe_html(markdown)
assert '<a' not in processed
def test_uppercase_group_ref_links_correctly():
"""Ensure using uppercase in a group ref works but links correctly."""
markdown = 'That was in ~Music.Metal.Progressive'
......
"""Set up the shared libcmark-gfm library and extensions."""
# pylint: disable=invalid-name
from ctypes import CDLL, c_char_p, c_int, c_size_t, c_void_p
CMARK_DLL = CDLL('/usr/local/lib/libcmark-gfm.so')
CMARK_EXT_DLL = CDLL('/usr/local/lib/libcmark-gfmextensions.so')
# enables the --hardbreaks option for cmark
# (can I import this? it's defined in cmark.h as CMARK_OPT_HARDBREAKS)
CMARK_OPTS = 4
CMARK_EXTENSIONS = (b'strikethrough', b'table')
cmark_parser_new = CMARK_DLL.cmark_parser_new
cmark_parser_new.restype = c_void_p
cmark_parser_new.argtypes = (c_int,)
cmark_parser_feed = CMARK_DLL.cmark_parser_feed
cmark_parser_feed.restype = None
cmark_parser_feed.argtypes = (c_void_p, c_char_p, c_size_t)
cmark_parser_finish = CMARK_DLL.cmark_parser_finish
cmark_parser_finish.restype = c_void_p
cmark_parser_finish.argtypes = (c_void_p,)
cmark_parser_attach_syntax_extension = (
CMARK_DLL.cmark_parser_attach_syntax_extension)
cmark_parser_attach_syntax_extension.restype = c_int
cmark_parser_attach_syntax_extension.argtypes = (c_void_p, c_void_p)
cmark_parser_get_syntax_extensions = (
CMARK_DLL.cmark_parser_get_syntax_extensions)
cmark_parser_get_syntax_extensions.restype = c_void_p
cmark_parser_get_syntax_extensions.argtypes = (c_void_p,)
cmark_parser_free = CMARK_DLL.cmark_parser_free
cmark_parser_free.restype = None
cmark_parser_free.argtypes = (c_void_p,)
cmark_node_free = CMARK_DLL.cmark_node_free
cmark_node_free.restype = None
cmark_node_free.argtypes = (c_void_p,)
cmark_find_syntax_extension = CMARK_DLL.cmark_find_syntax_extension
cmark_find_syntax_extension.restype = c_void_p
cmark_find_syntax_extension.argtypes = (c_char_p,)
cmark_render_html = CMARK_DLL.cmark_render_html
cmark_render_html.restype = c_char_p
cmark_render_html.argtypes = (c_void_p, c_int, c_void_p)
register = CMARK_EXT_DLL.core_extensions_ensure_registered
register.restype = None
register.argtypes = ()
register()
"""Functions/constants related to markdown handling."""
from ctypes import CDLL, c_char_p, c_long
import re
from typing import (
Callable,
......@@ -25,13 +24,19 @@ from html5lib.treewalkers.base import NonRecursiveTreeWalker
from tildes.metrics import histogram_timer
from tildes.schemas.group import is_valid_group_path
from tildes.schemas.user import is_valid_username
# set up the commonmark function to call into libcmark-gfm
CMARK_DLL = CDLL('/usr/local/lib/libcmark-gfm.so')
commonmark = CMARK_DLL.cmark_markdown_to_html # pylint: disable=invalid-name
commonmark.restype = c_char_p
commonmark.argtypes = [c_char_p, c_long, c_long]
from .cmark import (
CMARK_EXTENSIONS,
CMARK_OPTS,
cmark_find_syntax_extension,
cmark_node_free,
cmark_parser_attach_syntax_extension,
cmark_parser_feed,
cmark_parser_finish,
cmark_parser_free,
cmark_parser_get_syntax_extensions,
cmark_parser_new,
cmark_render_html,
)
HTML_TAG_WHITELIST = (
......@@ -62,12 +67,15 @@ HTML_TAG_WHITELIST = (
'tbody',
'td',
'th',
'thead',
'tr',
'ul',
)
HTML_ATTRIBUTE_WHITELIST = {
'a': ['href', 'title'],
'ol': ['start'],
'td': ['align'],
'th': ['align'],
}
PROTOCOL_WHITELIST = ('http', 'https')
......@@ -112,11 +120,19 @@ def convert_markdown_to_safe_html(markdown: str) -> str:
markdown_bytes = markdown.encode('utf8')
# enables the --hardbreaks option
# (can I import this? it's defined in cmark.h as CMARK_OPT_HARDBREAKS)
cmark_options = 4
parser = cmark_parser_new(CMARK_OPTS)
for name in CMARK_EXTENSIONS:
ext = cmark_find_syntax_extension(name)
cmark_parser_attach_syntax_extension(parser, ext)
exts = cmark_parser_get_syntax_extensions(parser)
cmark_parser_feed(parser, markdown_bytes, len(markdown_bytes))
doc = cmark_parser_finish(parser)
html_bytes = cmark_render_html(doc, CMARK_OPTS, exts)
html_bytes = commonmark(markdown_bytes, len(markdown_bytes), cmark_options)
cmark_parser_free(parser)
cmark_node_free(doc)
html = html_bytes.decode('utf8')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment