Added pelican-gfm plugin

Readme.rst

@@ -187,6 +187,8 @@ pelican-ert               Allows you to add estimated reading time of an article
 
 Pelican-flickr            Brings your Flickr photos & sets into your static website
 
+pelican-gfm               A reader that uses GitHub's C based cmark libraries to translate .md files
+
 Pelican Genealogy         Add surnames and people so metadata and context can be accessed from within a theme to provide surname and person pages
 
 Pelican Gist tag          Easily embed GitHub Gists in your Pelican articles

pelican-gfm/Readme.md

@@ -0,0 +1,22 @@
+pelican-gfm
+===========
+A reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html.
+
+Requirements
+============
+
+pelican-gfm has no requirements that are outside of the python standard library aside from pelican itself.
+
+How to Use
+=========
+Drop the entire pelican-gfm directory into the plugin path and invoke it from your pelicanconf.py to have pelican-gfm render contents/\*.md
+
+Syntax
+======
+This plugin leverages [GitHub Flavored Markdown](https://github.github.com/gfm/) in `.md` files to generate html pages.
+
+
+Attribution
+===========
+`pelican-gfm` is based on [pelican_gfm_script](https://github.com/apache/infrastructure-website/blob/master/gfm_reader.py)
+Originally written by: Greg Stein

pelican-gfm/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/environment python -B
+
+from gfm import *

pelican-gfm/gfm.py

@@ -0,0 +1,208 @@
+#!/usr/bin/python -B
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+# gfm_reader.py -- GitHub-Flavored Markdown reader for Pelican
+#
+
+import sys
+import os
+import ctypes
+import re
+import gfmSetup
+import pelican.utils
+import pelican.signals
+import pelican.readers
+
+_LIBDIR = gfmSetup.LIBCMARKLOCATION
+_LIBCMARK = 'libcmark-gfm.so'
+_LIBEXT = 'libcmark-gfmextensions.so'
+try:
+    cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK))
+    cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT))
+except OSError:
+    raise ImportError('%s not found. see build-mark.sh and gfm_reader.py'
+                      % _LIBCMARK)
+
+# Use ctypes to access the functions in libcmark-gfm
+F_cmark_parser_new = cmark.cmark_parser_new
+F_cmark_parser_new.restype = ctypes.c_void_p
+F_cmark_parser_new.argtypes = (ctypes.c_int,)
+
+F_cmark_parser_feed = cmark.cmark_parser_feed
+F_cmark_parser_feed.restype = None
+F_cmark_parser_feed.argtypes = (ctypes.c_void_p,
+                                ctypes.c_char_p,
+                                ctypes.c_size_t,)
+
+F_cmark_parser_finish = cmark.cmark_parser_finish
+F_cmark_parser_finish.restype = ctypes.c_void_p
+F_cmark_parser_finish.argtypes = (ctypes.c_void_p,)
+
+F_cmark_parser_attach_syntax_extension = cmark.cmark_parser_attach_syntax_extension
+F_cmark_parser_attach_syntax_extension.restype = ctypes.c_int
+F_cmark_parser_attach_syntax_extension.argtypes = (ctypes.c_void_p,
+                                                   ctypes.c_void_p,)
+
+F_cmark_parser_get_syntax_extensions = cmark.cmark_parser_get_syntax_extensions
+F_cmark_parser_get_syntax_extensions.restype = ctypes.c_void_p
+F_cmark_parser_get_syntax_extensions.argtypes = (ctypes.c_void_p,)
+
+F_cmark_parser_free = cmark.cmark_parser_free
+F_cmark_parser_free.restype = None
+F_cmark_parser_free.argtypes = (ctypes.c_void_p,)
+
+F_cmark_node_free = cmark.cmark_node_free
+F_cmark_node_free.restype = None
+F_cmark_node_free.argtypes = (ctypes.c_void_p,)
+
+F_cmark_find_syntax_extension = cmark.cmark_find_syntax_extension
+F_cmark_find_syntax_extension.restype = ctypes.c_void_p
+F_cmark_find_syntax_extension.argtypes = (ctypes.c_char_p,)
+
+F_cmark_render_html = cmark.cmark_render_html
+F_cmark_render_html.restype = ctypes.c_char_p
+F_cmark_render_html.argtypes = (ctypes.c_void_p,
+                                ctypes.c_int,
+                                ctypes.c_void_p,)
+
+
+# Set up the libcmark-gfm library and its extensions
+F_register = cmark_ext.core_extensions_ensure_registered
+F_register.restype = None
+F_register.argtypes = ()
+F_register()
+
+# technically, maybe install an atexit() to release the plugins
+
+# Options for the GFM rendering call
+# this could be moved into SETTINGS or somesuch, but meh. not needed now.
+OPTS = 0
+
+# The GFM extensions that we want to use
+EXTENSIONS = (
+    'autolink',
+    'table',
+    'strikethrough',
+    'tagfilter',
+)
+
+
+class GFMReader(pelican.readers.BaseReader):
+    enabled = True
+    """GFM-flavored Reader for the Pelican system.
+
+    Pelican looks for all subclasses of BaseReader, and automatically
+    registers them for the file extensions listed below. Thus, nothing
+    further is required by users of this Reader.
+    """
+
+    # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be
+    #       non-deterministic which Reader will be used for these files.
+    file_extensions = ['md', 'markdown', 'mkd', 'mdown']
+
+    # Metadata is specified as a single, colon-separated line, such as:
+    #
+    # Title: this is the title
+    #
+    # Note: name starts in column 0, no whitespace before colon, will be
+    #       made lower-case, and value will be stripped
+    #
+    RE_METADATA = re.compile('^([A-za-z]+): (.*)$')
+
+    def read(self, source_path):
+        # Prepare the "slug", which is the target file name. It will be the
+        # same as the source file minus the leading:
+        #   ".../content/(articles|pages)"
+        # and with the extension removed (Pelican will add .html)
+        relpath = os.path.relpath(source_path, self.settings['PATH'])
+        parts = relpath.split(os.sep)
+        parts[-1] = os.path.splitext(parts[-1])[0]  # split off ext, keep base
+        slug = os.sep.join(parts[1:])
+        metadata = {
+            'slug': slug,
+        }
+
+        # Fetch the source content, with a few appropriate tweaks
+        with pelican.utils.pelican_open(source_path) as text:
+            # Extract the metadata from the header of the text
+            lines = text.splitlines()
+            for i in range(len(lines)):
+                line = lines[i]
+                match = GFMReader.RE_METADATA.match(line)
+                if match:
+                    name = match.group(1).strip().lower()
+                    if name != 'slug':
+                        value = match.group(2).strip()
+                        if name == 'date':
+                            value = pelican.utils.get_date(value)
+                    metadata[name] = value
+                    # if name != 'title':
+                    #  print 'META:', name, value
+                elif not line.strip():
+                    # blank line
+                    continue
+                else:
+                    # reached actual content
+                    break
+            # Reassemble content, minus the metadata
+            text = '\n'.join(lines[i:])
+            content = self.render(text)
+        # Redo the slug for articles.
+        if parts[0] == 'articles' and 'title' in metadata:
+            metadata['slug'] = pelican.utils.slugify(
+                                metadata['title'],
+                                self.settings.get('SLUG_SUBSTITUTIONS', ()))
+        return content, metadata
+
+    def render(self, text):
+        "Use cmark-gfm to render the Markdown into an HTML fragment."
+
+        parser = F_cmark_parser_new(OPTS)
+        assert parser
+        for name in EXTENSIONS:
+            ext = F_cmark_find_syntax_extension(name)
+            assert ext
+            rv = F_cmark_parser_attach_syntax_extension(parser, ext)
+            assert rv
+        exts = F_cmark_parser_get_syntax_extensions(parser)
+
+        F_cmark_parser_feed(parser, text, len(text))
+        doc = F_cmark_parser_finish(parser)
+        assert doc
+
+        output = F_cmark_render_html(doc, OPTS, exts)
+
+        F_cmark_parser_free(parser)
+        F_cmark_node_free(doc)
+
+        return output
+
+
+def add_readers(readers):
+    if str(gfmSetup.test_configuration()) == "0":
+        readers.reader_classes['md'] = GFMReader
+    else:
+        # Optionally we can have the gfmSetup script run here
+        # as root and configure the system
+        # Probably just easier to run it independently tho.
+        sys.exit(1)
+
+
+def register():
+    pelican.signals.readers_init.connect(add_readers)

pelican-gfm/gfmSetup.py

@@ -0,0 +1,111 @@
+#!/usr/bin/python -B
+
+import os
+import subprocess
+
+# This gets used by the gfm plugin as well as the check_configure function
+LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu"
+
+ARCHIVES = "https://github.com/github/cmark-gfm/archive"
+VERSION = "0.28.3.gfm.12"
+LOCAL = "cmark-gfm.$VERSION.orig.tar.gz"
+WORKSPACE = '/tmp/build-cmark'
+
+
+def dpkg_installed(package):
+    t1 = subprocess.Popen(["dpkg", "-l"], stdout=subprocess.PIPE)
+    t2 = subprocess.Popen(["grep", "-q", package],
+                          stdout=subprocess.PIPE,
+                          stdin=t1.stdout,)
+    ec = t2.wait()
+    return ec
+
+
+def test_setup():
+    installed = ["cmake", "make", "wget"]
+    removed = ["libcmark-gfm-dev",
+               "libcmark-gfm-extensions-dev",
+               "libcmark-gfm0",
+               "libcmark-gfm-extensions0", ]
+    for package in installed:
+        if str(dpkg_installed(package)) == "1":
+            print(package + " not installed")
+            return 1
+
+    for package in removed:
+        if dpkg_installed == "0":
+            print(package + " needs removed")
+            return 1
+
+
+def apt_install(package):
+    # I need to be able to do this a better, in a less sudo + apt-y way
+    subprocess.call(["apt-get", "install", package, "-y"])
+
+
+def apt_remove(package):
+    # I need to be able to do this a better, in a less sudo + apt-y way
+    subprocess.call(["apt-get", "purge", package, "-y"])
+
+
+def cleanUp():
+    subprocess.call(["rm", "-rf", WORKSPACE])
+
+
+def setup():
+    test_setup()
+    # Configure the environment if it's not already configured
+    if not os.path.isdir(WORKSPACE):
+        os.mkdir(WORKSPACE)
+    subprocess.call(["wget",
+                     "--quiet",
+                     ARCHIVES + "/" + VERSION + ".tar.gz", WORKSPACE,
+                     "-P",
+                     WORKSPACE])
+    subprocess.call(['tar',
+                     'zxf',
+                     WORKSPACE + "/" + VERSION + ".tar.gz",
+                     "-C",
+                     WORKSPACE])
+    BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + VERSION + "/build"
+    if not os.path.isdir(BUILDSPACE):
+        os.mkdir(BUILDSPACE)
+    thing1 = subprocess.Popen(["cmake",
+                               "-DCMARK_TESTS=OFF",
+                               "-DCMARK_STATIC=OFF",
+                               ".."],
+                              cwd=BUILDSPACE)
+    thing1.wait()
+
+    thing2 = subprocess.Popen(["make"], cwd=BUILDSPACE)
+    thing2.wait()
+
+    # Move the libcmark.so artifacts in place
+    print("Moving files")
+    gfmfile = BUILDSPACE+"/src/libcmark-gfm.so."+VERSION
+    gfmextfile = BUILDSPACE+"/extensions/libcmark-gfmextensions.so."+VERSION
+    subprocess.call(["mv",
+                     gfmfile,
+                     LIBCMARKLOCATION + "libcmark-gfm.so"])
+    subprocess.call(["mv",
+                     gfmextfile,
+                     LIBCMARKLOCATION + "libcmark-gfmextensions.so"])
+
+
+def test_configuration():
+    gfmfile = LIBCMARKLOCATION + "/libcmark-gfm.so"
+    gfmextfile = LIBCMARKLOCATION + "/libcmark-gfmextensions.so"
+    if os.path.isfile(gfmfile) and os.path.isfile(gfmextfile):
+        return 0
+    else:
+        return 1
+
+
+def configure():
+        print("Configuring!!!")
+        setup()
+        cleanUp()
+
+
+if __name__ == "__main__":
+    configure()