fixes for liquid tags

liquid_tags/Readme.md

@@ -258,8 +258,10 @@ comment line `# <!-- collapse=False -->` will be expanded on load but
 can be collapsed by tapping on their header. Cells without collapsed
 comments are rendered as standard code input cells.
 
-## Configuration settings in custom tags
 
+## Advanced features
+
+### Configuration settings in custom tags
 Tags do not have access to the full Pelicans settings, and instead arrange for 
 the variables to be passed to the tag.  For tag authors who plan to add their 
 tag as in-tree tags, they can just add the variables they need to an array in 
@@ -269,6 +271,15 @@ user's `pelicanconf.py` settings:
 
     LIQUID_CONFIGS = (('PATH', '.', "The default path"), ('SITENAME', 'Default Sitename', 'The name of the site'))
 
+### Custom delimiters
+If you are using Liquid Tags in conjunction with some other plugin that also
+uses the `{%` and `%}` delimiters to mark its blocks, then chaos will occur.
+You can avoid the chaos by defining an alternative set of delimiters for 
+Liquid Tags. Just add them as a tuple in your `pelicanconf.py` settings:
+
+    LT_DELIMITERS = ('<+', '+>')
+
+
 ## Testing
 
 To test the plugin in multiple environments we use [tox](http://tox.readthedocs.org/en/latest/). To run the entire test suite:

liquid_tags/img.py

@@ -22,18 +22,21 @@
 
 [1] https://github.com/imathis/octopress/blob/master/plugins/image_tag.rb
 """
+import os
 import re
 from .mdx_liquid_tags import LiquidTags
 import six
 
 SYNTAX = '{% img [class name(s)] [http[s]:/]/path/to/image [width [height]] [title text | "title text" ["alt text"]] %}'
 
 # Regular expression to match the entire syntax
-ReImg = re.compile("""(?P<class>\S.*\s+)?(?P<src>(?:https?:\/\/|\/|\S+\/)\S+)(?:\s+(?P<width>\d+))?(?:\s+(?P<height>\d+))?(?P<title>\s+.+)?""")
+ReImg = re.compile(r'(?P<class>[-\w\s]+\s+)?(?P<src>(?P<scheme>[a-zA-Z]+://)?(?P<path>\S+))(?:\s+(?P<width>\d+))?(?:\s+(?P<height>\d+))?(?P<title>\s+.+)?')
 
 # Regular expression to split the title and alt text
 ReTitleAlt = re.compile("""(?:"|')(?P<title>[^"']+)?(?:"|')\s+(?:"|')(?P<alt>[^"']+)?(?:"|')""")
 
+# Attributes to keep in the emmitted img tag
+IMG_ATTRS = ['class', 'src', 'width', 'height', 'title',]
 
 @LiquidTags.register('img')
 def img(preprocessor, tag, markup):
@@ -42,8 +45,7 @@ def img(preprocessor, tag, markup):
     # Parse the markup string
     match = ReImg.search(markup)
     if match:
-        attrs = dict([(key, val.strip())
-                      for (key, val) in six.iteritems(match.groupdict()) if val])
+        attrs = {k: v.strip() for k, v in match.groupdict().items() if v}
     else:
         raise ValueError('Error processing input. '
                          'Expected syntax: {0}'.format(SYNTAX))
@@ -56,9 +58,16 @@ def img(preprocessor, tag, markup):
         if not attrs.get('alt'):
             attrs['alt'] = attrs['title']
 
-    # Return the formatted text
-    return "<img {0}>".format(' '.join('{0}="{1}"'.format(key, val)
-                                       for (key, val) in six.iteritems(attrs)))
+    # prepend site url to absolute paths
+    if 'scheme' not in attrs and os.path.isabs(attrs['src']):
+        siteurl = preprocessor.configs.getConfig('SITEURL')
+        attrs['src'] = siteurl + attrs['path']
+
+    # create tag
+    img_attrs = ['{0!s}={1!r}'.format(k, attrs[k])
+            for k in IMG_ATTRS if attrs.get(k)]
+    s = "<img {0}>".format(' '.join(img_attrs))
+    return s
 
 #----------------------------------------------------------------------
 # This import allows image tag to be a Pelican plugin

liquid_tags/include_code.py

@@ -39,9 +39,11 @@
 """
 import re
 import os
-import sys
+import six
 from .mdx_liquid_tags import LiquidTags
 
+if six.PY2:
+    from io import open
 
 SYNTAX = "{% include_code /path/to/code.py [lang:python] [lines:X-Y] "\
          "[:hidefilename:] [:hidelink:] [:hideall:] [title] %}"
@@ -77,7 +79,7 @@ def include_code(preprocessor, tag, markup):
         argdict = match.groupdict()
         title = argdict['title'] or ""
         lang = argdict['lang']
-        codec = argdict['codec'] or "utf8"
+        codec = argdict['codec'] or "utf-8"
         lines = argdict['lines']
         hide_filename = bool(argdict['hidefilename'])
         hide_link = bool(argdict['hidelink'])
@@ -86,6 +88,9 @@ def include_code(preprocessor, tag, markup):
             first_line, last_line = map(int, lines.split("-"))
         src = argdict['src']
 
+    if not codec:
+        codec = 'utf-8'
+
     if not src:
         raise ValueError("Error processing input, "
                          "expected syntax: {0}".format(SYNTAX))
@@ -96,9 +101,6 @@ def include_code(preprocessor, tag, markup):
     if not os.path.exists(code_path):
         raise ValueError("File {0} could not be found".format(code_path))
 
-    if not codec:
-        codec = 'utf-8'
-
     with open(code_path, encoding=codec) as fh:
         if lines:
             code = fh.readlines()[first_line - 1: last_line]
@@ -145,18 +147,11 @@ def include_code(preprocessor, tag, markup):
     else:
         lang_include = ''
 
-    if sys.version_info[0] < 3:
-        source = (open_tag
-                  + '\n\n    '
-                  + lang_include
-                  + '\n    '.join(code.decode(codec).split('\n')) + '\n\n'
-                  + close_tag + '\n')
-    else:
-        source = (open_tag
-                  + '\n\n    '
-                  + lang_include
-                  + '\n    '.join(code.split('\n')) + '\n\n'
-                  + close_tag + '\n')
+    source = (open_tag
+              + '\n\n    '
+              + lang_include
+              + '\n    '.join(code.split('\n')) + '\n\n'
+              + close_tag + '\n')
 
     return source
 

liquid_tags/mdx_liquid_tags.py

@@ -16,46 +16,45 @@
 import os
 from functools import wraps
 
-# Define some regular expressions
-LIQUID_TAG = re.compile(r'\{%.*?%\}', re.MULTILINE | re.DOTALL)
-EXTRACT_TAG = re.compile(r'(?:\s*)(\S+)(?:\s*)')
 LT_CONFIG = { 'CODE_DIR': 'code',
               'NOTEBOOK_DIR': 'notebooks',
               'FLICKR_API_KEY': 'flickr',
-              'GIPHY_API_KEY': 'giphy'
+              'GIPHY_API_KEY': 'giphy',
+              'LT_DELIMITERS': ('{%', '%}'),
+              'SITEURL': '',
 }
 LT_HELP = { 'CODE_DIR' : 'Code directory for include_code subplugin',
             'NOTEBOOK_DIR' : 'Notebook directory for notebook subplugin',
             'FLICKR_API_KEY': 'Flickr key for accessing the API',
-            'GIPHY_API_KEY': 'Giphy key for accessing the API'
+            'GIPHY_API_KEY': 'Giphy key for accessing the API',
+            'LT_DELIMITERS': 'Alternative set of Liquid Tags block delimiters',
+            'SITEURL': 'Base URL of your web site. '
+                       'Inserted before absolute media paths.',
 }
 
 class _LiquidTagsPreprocessor(markdown.preprocessors.Preprocessor):
+    LT_FMT = r'{0}(?:\s*)(?P<tag>\S+)(?:\s*)(?P<markup>.*?)(?:\s*){1}'
+    LT_RE_FLAGS = re.MULTILINE | re.DOTALL | re.UNICODE
     _tags = {}
+
     def __init__(self, configs):
+        cls = self.__class__
+        liquid_tag_re = cls.LT_FMT.format(
+                *map(re.escape, configs.getConfig('LT_DELIMITERS')))
+        self.liquid_tag = re.compile(liquid_tag_re, cls.LT_RE_FLAGS)
         self.configs = configs
 
+    def expand_tag(self, match):
+        tag, markup = match.groups()
+        if tag in self.__class__._tags:
+            return self.__class__._tags[tag](self, tag, markup)
+        else:
+            return match[0]
+
     def run(self, lines):
         page = '\n'.join(lines)
-        liquid_tags = LIQUID_TAG.findall(page)
-
-        for i, markup in enumerate(liquid_tags):
-            # remove {% %}
-            markup = markup[2:-2]
-            tag = EXTRACT_TAG.match(markup).groups()[0]
-            markup = EXTRACT_TAG.sub('', markup, 1)
-            if tag in self._tags:
-                liquid_tags[i] = self._tags[tag](self, tag, markup.strip())
-
-        # add an empty string to liquid_tags so that chaining works
-        liquid_tags.append('')
-
-        # reconstruct string
-        page = ''.join(itertools.chain(*zip(LIQUID_TAG.split(page),
-                                            liquid_tags)))
-
-        # resplit the lines
-        return page.split("\n")
+        page = self.liquid_tag.sub(self.expand_tag, page)
+        return page.splitlines()
 
 
 class LiquidTags(markdown.Extension):
@@ -82,15 +81,19 @@ def dec(func):
             return func
         return dec
 
-    def extendMarkdown(self, md, md_globals):
+    def extendMarkdown(self, md):
         self.htmlStash = md.htmlStash
         md.registerExtension(self)
-        # for the include_code preprocessor, we need to re-run the
-        # fenced code block preprocessor after substituting the code.
-        # Because the fenced code processor is run before, {% %} tags
-        # within equations will not be parsed as an include.
-        md.preprocessors.add('mdincludes',
-                             _LiquidTagsPreprocessor(self), ">html_block")
+        if 'include_code' in _LiquidTagsPreprocessor._tags:
+            # For the include_code preprocessor, we need to re-run the
+            # fenced code block preprocessor after substituting the code.
+            # Because the fenced code processor is run before, {% %} tags
+            # within equations will not be parsed as an include.
+            #
+            # The now deprecated add() function, with ">htmlblock" argument
+            # resulted in a priority of 15. Which is what we use here.
+            md.preprocessors.register(_LiquidTagsPreprocessor(self),
+                    'mdincludes', 15)
 
 
 def makeExtension(configs=None):

liquid_tags/notebook.py

@@ -283,8 +283,9 @@ def notebook(preprocessor, tag, markup):
 
     language_applied_highlighter = partial(custom_highlighter, language=language)
 
+    content_root = preprocessor.configs.getConfig('PATH', default='content')
     nb_dir =  preprocessor.configs.getConfig('NOTEBOOK_DIR')
-    nb_path = os.path.join(settings.get('PATH', 'content'), nb_dir, src)
+    nb_path = os.path.join(content_root, nb_dir, src)
 
     if not os.path.exists(nb_path):
         raise ValueError("File {0} could not be found".format(nb_path))

liquid_tags/test_data/content/test-generic-config-alt-delim.md

@@ -0,0 +1,7 @@
+Title: test alternate tag delimiters
+Date: 2020-06-13
+Authors: M. Stamatogiannakis
+
+Regular tag is not expanded: {% generic config author %} is stupid
+
+Alternate tag is expanded:   <+ generic config author +> is smart

liquid_tags/test_generation.py

@@ -2,6 +2,7 @@
 from __future__ import print_function
 
 import filecmp
+import logging
 import os
 import unittest
 from shutil import rmtree
@@ -32,65 +33,46 @@ def tearDown(self):
         rmtree(self.temp_cache)
         os.chdir(PLUGIN_DIR)
 
-    @unittest.skipIf(IPYTHON_VERSION != 3,
-                     reason="output must be created with ipython version 3")
-    def test_generate_with_ipython3(self):
+    @unittest.skipIf(IPYTHON_VERSION not in (2, 3),
+                     reason="iPython v%d is not supported" % IPYTHON_VERSION)
+    def test_generate(self):
         '''Test generation of site with the plugin.'''
 
-        base_path = os.path.dirname(os.path.abspath(__file__))
-        base_path = os.path.join(base_path, 'test_data')
-        content_path = os.path.join(base_path, 'content')
-        output_path = os.path.join(base_path, 'output')
-        settings_path = os.path.join(base_path, 'pelicanconf.py')
-        settings = read_settings(path=settings_path,
-                                 override={'PATH': content_path,
-                                           'OUTPUT_PATH': self.temp_path,
-                                           'CACHE_PATH': self.temp_cache,
-                                           }
-                                 )
-
+        # set paths
+        _pj = os.path.join
+        base_path = _pj(os.path.dirname(os.path.abspath(__file__)), 'test_data')
+        content_path = _pj(base_path, 'content')
+        output_path = _pj(base_path, 'output')
+        settings_path = _pj(base_path, 'pelicanconf.py')
+
+        # read settings
+        override = {
+                'PATH': content_path,
+                'OUTPUT_PATH': self.temp_path,
+                'CACHE_PATH': self.temp_cache,
+        }
+        settings = read_settings(path=settings_path, override=override)
+
+        # run and test created files
         pelican = Pelican(settings)
         pelican.run()
 
         # test existence
-        assert os.path.exists(os.path.join(self.temp_path,
-                                           'test-ipython-notebook-nb-format-3.html'))
-        assert os.path.exists(os.path.join(self.temp_path,
-                                           'test-ipython-notebook-nb-format-4.html'))
+        assert os.path.exists(_pj(self.temp_path,
+                                  'test-ipython-notebook-nb-format-3.html'))
+        assert os.path.exists(_pj(self.temp_path,
+                                  'test-ipython-notebook-nb-format-4.html'))
 
         # test differences
-        #assert filecmp.cmp(os.path.join(output_path,
-        #                                'test-ipython-notebook-v2.html'),
-        #                   os.path.join(self.temp_path,
-        #                                'test-ipython-notebook.html'))
-
-    @unittest.skipIf(IPYTHON_VERSION != 2,
-                     reason="output must be created with ipython version 2")
-    def test_generate_with_ipython2(self):
-        '''Test generation of site with the plugin.'''
-
-        base_path = os.path.dirname(os.path.abspath(__file__))
-        base_path = os.path.join(base_path, 'test_data')
-        content_path = os.path.join(base_path, 'content')
-        settings_path = os.path.join(base_path, 'pelicanconf.py')
-        settings = read_settings(path=settings_path,
-                                 override={'PATH': content_path,
-                                           'OUTPUT_PATH': self.temp_path,
-                                           'CACHE_PATH': self.temp_cache,
-                                           }
-                                 )
-
-        pelican = Pelican(settings)
-        pelican.run()
+        if IPYTHON_VERSION == 3:
+            f1 = _pj(output_path, 'test-ipython-notebook-v2.html')
+            f2 = _pj(self.temp_path, 'test-ipython-notebook.html')
+            #assert filecmp.cmp(f1, f2)
+        elif IPYTHON_VERSION == 2:
+            f1 = _pj(output_path, 'test-ipython-notebook-v3.html')
+            f2 = _pj(self.temp_path, 'test-ipython-notebook.html')
+            #assert filecmp.cmp(f1, f2)
+        else:
+            logging.error('Unexpected IPYTHON_VERSION: %s', IPYTHON_VERSION)
+            assert False
 
-        # test existence
-        assert os.path.exists(os.path.join(self.temp_path,
-                                           'test-ipython-notebook-nb-format-3.html'))
-        assert os.path.exists(os.path.join(self.temp_path,
-                                           'test-ipython-notebook-nb-format-4.html'))
-
-        # test differences
-        #assert filecmp.cmp(os.path.join(output_path,
-        #                                'test-ipython-notebook-v3.html'),
-        #                   os.path.join(self.temp_path,
-        #                                'test-ipython-notebook.html'))