diff --git a/.gitignore b/.gitignore index 939db29..3bb343a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc -*.log \ No newline at end of file +*.log +*~ \ No newline at end of file diff --git a/liquid_tags/Readme.md b/liquid_tags/Readme.md new file mode 100644 index 0000000..0fffd6a --- /dev/null +++ b/liquid_tags/Readme.md @@ -0,0 +1,98 @@ +# Liquid-style Tags +*Author: Jake Vanderplas * + +This plugin allows liquid-style tags to be inserted into markdown within +Pelican documents. Liquid uses tags bounded by ``{% ... %}``, and is used +to extend markdown in other blogging platforms such as octopress. + +This set of extensions does not actually interface with liquid, but allows +users to define their own liquid-style tags which will be inserted into +the markdown preprocessor stream. There are several built-in tags, which +can be added as follows. + +First, in your pelicanconf.py file, add the plugins you want to use: + + PLUGIN_PATH = '/path/to/pelican-plugins' + PLUGINS = ['liquid_tags.img', 'liquid_tags.video', + 'liquid_tags.include_code', 'liquid_tags.notebook'] + +There are several options available + +## Image Tag +To insert a sized and labeled image in your document, enable the +``liquid_tags.video`` plugin and use the following: + +{% img [class name(s)] path/to/image [width [height]] [title text | "title text" ["alt text"]] %} + + +## Video Tag +To insert flash/HTML5-friendly video into a post, enable the +``liquid_tags.video`` plugin, and add to your document: + + {% video /url/to/video.mp4 [width] [height] [/path/to/poster.png] %} + +The width and height are in pixels, and can be optionally specified. If they +are not, then the original video size will be used. The poster is an image +which is used as a preview of the video. + +To use a video from file, make sure it's in a static directory and put in +the appropriate url. + +## Include Code +To include code from a file in your document with a link to the original +file, enable the ``liquid_tags.include_code`` plugin, and add to your +document: + + {% include_code myscript.py [Title text] %} + +The script must be in the ``code`` subdirectory of your content folder: +this default location can be changed by specifying + + CODE_DIR = 'code' + +within your configuration file. Additionally, in order for the resulting +hyperlink to work, this directory must be listed under the STATIC_PATHS +setting, e.g.: + + STATIC_PATHS = ['images', 'code'] + +## IPython notebooks +To insert an ipython notebook into your post, enable the +``liquid_tags.notebook`` plugin and add to your document: + + {% notebook filename.ipynb %} + +The file should be specified relative to the ``notebooks`` subdirectory of the +content directory. Optionally, this subdirectory can be specified in the +config file: + + NOTEBOOK_DIR = 'notebooks' + +Because the conversion and rendering of notebooks is rather involved, there +are a few extra steps required for this plugin: + +- First, the plugin requires that the nbconvert package [1]_ to be in the + python path. For example, in bash, this can be set via + + >$ export PYTHONPATH=/path/to/nbconvert/ + + The nbconvert package is still in development, so we recommend using the + most recent version. + +- After typing "make html" when using the notebook tag, a file called + ``_nb_header.html`` will be produced in the main directory. The content + of the file should be included in the header of the theme. An easy way + to accomplish this is to add the following lines within the header template + of the theme you use: + + {% if EXTRA_HEADER %} + {{ EXTRA_HEADER }} + {% endif %} + + and in your configuration file, include the line: + + EXTRA_HEADER = open('_nb_header.html').read().decode('utf-8') + + this will insert the proper css formatting into your document. + +[1] https://github.com/ipython/nbconvert \ No newline at end of file diff --git a/liquid_tags/__init__.py b/liquid_tags/__init__.py new file mode 100644 index 0000000..eabcd63 --- /dev/null +++ b/liquid_tags/__init__.py @@ -0,0 +1 @@ +from .liquid_tags import * diff --git a/liquid_tags/img.py b/liquid_tags/img.py new file mode 100644 index 0000000..26039e4 --- /dev/null +++ b/liquid_tags/img.py @@ -0,0 +1,65 @@ +""" +Image Tag +--------- +This implements a Liquid-style image tag for Pelican, +based on the octopress image tag [1]_ + +Syntax +------ +{% img [class name(s)] [http[s]:/]/path/to/image [width [height]] [title text | "title text" ["alt text"]] %} + +Examples +-------- +{% img /images/ninja.png Ninja Attack! %} +{% img left half http://site.com/images/ninja.png Ninja Attack! %} +{% img left half http://site.com/images/ninja.png 150 150 "Ninja Attack!" "Ninja in attack posture" %} + +Output +------ + +Ninja Attack! +Ninja in attack posture + +[1] https://github.com/imathis/octopress/blob/master/plugins/image_tag.rb +""" +import re +from .mdx_liquid_tags import LiquidTags + +SYNTAX = '{% img [class name(s)] [http[s]:/]/path/to/image [width [height]] [title text | "title text" ["alt text"]] %}' + +# Regular expression to match the entire syntax +ReImg = re.compile("""(?P\S.*\s+)?(?P(?:https?:\/\/|\/|\S+\/)\S+)(?:\s+(?P\d+))?(?:\s+(?P\d+))?(?P\s+.+)?""") + +# Regular expression to split the title and alt text +ReTitleAlt = re.compile("""(?:"|')(?P<title>[^"']+)?(?:"|')\s+(?:"|')(?P<alt>[^"']+)?(?:"|')""") + + +@LiquidTags.register('img') +def img(preprocessor, tag, markup): + attrs = None + + # Parse the markup string + match = ReImg.search(markup) + if match: + attrs = dict([(key, val.strip()) + for (key, val) in match.groupdict().iteritems() if val]) + else: + raise ValueError('Error processing input. ' + 'Expected syntax: {0}'.format(SYNTAX)) + + # Check if alt text is present -- if so, split it from title + if 'title' in attrs: + match = ReTitleAlt.search(attrs['title']) + if match: + attrs.update(match.groupdict()) + if not attrs.get('alt'): + attrs['alt'] = attrs['title'] + + # Return the formatted text + return "<img {0}>".format(' '.join('{0}="{1}"'.format(key, val) + for (key, val) in attrs.iteritems())) + +#---------------------------------------------------------------------- +# This import allows image tag to be a Pelican plugin +from liquid_tags import register + diff --git a/liquid_tags/include_code.py b/liquid_tags/include_code.py new file mode 100644 index 0000000..f8572ca --- /dev/null +++ b/liquid_tags/include_code.py @@ -0,0 +1,103 @@ +""" +Include Code Tag +---------------- +This implements a Liquid-style video tag for Pelican, +based on the octopress video tag [1]_ + +Syntax +------ +{% include_code path/to/code [lang:python] [Title text] %} + +The "path to code" is specified relative to the ``code`` subdirectory of +the content directory Optionally, this subdirectory can be specified in the +config file: + + CODE_DIR = 'code' + +Example +------- +{% include_code myscript.py %} + +This will import myscript.py from content/downloads/code/myscript.py +and output the contents in a syntax highlighted code block inside a figure, +with a figcaption listing the file name and download link. + +The file link will be valid only if the 'code' directory is listed +in the STATIC_PATHS setting, e.g.: + + STATIC_PATHS = ['images', 'code'] + +[1] https://github.com/imathis/octopress/blob/master/plugins/include_code.rb +""" +import re +import os +from .mdx_liquid_tags import LiquidTags + + +SYNTAX = "{% include_code /path/to/code.py [lang:python] [title] %}" +FORMAT = re.compile(r"""^(?:\s+)?(?P<src>\S+)(?:\s+)?(?:(?:lang:)(?P<lang>\S+))?(?:\s+)?(?P<title>.+)?$""") + + +@LiquidTags.register('include_code') +def include_code(preprocessor, tag, markup): + title = None + lang = None + src = None + + match = FORMAT.search(markup) + if match: + argdict = match.groupdict() + title = argdict['title'] + lang = argdict['lang'] + src = argdict['src'] + + if not src: + raise ValueError("Error processing input, " + "expected syntax: {0}".format(SYNTAX)) + + settings = preprocessor.configs.config['settings'] + code_dir = settings.get('CODE_DIR', 'code') + code_path = os.path.join('content', code_dir, src) + + if not os.path.exists(code_path): + raise ValueError("File {0} could not be found".format(code_path)) + + code = open(code_path).read() + + if title: + title = "{0} {1}".format(title, os.path.basename(src)) + else: + title = os.path.basename(src) + + static_dir = settings.get('STATIC_OUT_DIR', 'static') + + url = '/{0}/{1}/{2}'.format(static_dir, code_dir, src) + url = re.sub('/+', '/', url) + + open_tag = ("<figure class='code'>\n<figcaption><span>{title}</span> " + "<a href='{url}'>download</a></figcaption>".format(title=title, + url=url)) + close_tag = "</figure>" + + # store HTML tags in the stash. This prevents them from being + # modified by markdown. + open_tag = preprocessor.configs.htmlStash.store(open_tag, safe=True) + close_tag = preprocessor.configs.htmlStash.store(close_tag, safe=True) + + if lang: + lang_include = ':::' + lang + '\n ' + else: + lang_include = '' + + source = (open_tag + + '\n\n ' + + lang_include + + '\n '.join(code.split('\n')) + '\n\n' + + close_tag + '\n') + + return source + + +#---------------------------------------------------------------------- +# This import allows image tag to be a Pelican plugin +from liquid_tags import register diff --git a/liquid_tags/liquid_tags.py b/liquid_tags/liquid_tags.py new file mode 100644 index 0000000..5721cce --- /dev/null +++ b/liquid_tags/liquid_tags.py @@ -0,0 +1,15 @@ +from pelican import signals +from mdx_liquid_tags import LiquidTags +from pelican.readers import EXTENSIONS + +def addLiquidTags(gen): + if not gen.settings.get('MD_EXTENSIONS'): + MDReader = EXTENSIONS['markdown'] + gen.settings['MD_EXTENSIONS'] = MDReader.default_extensions + + if LiquidTags not in gen.settings['MD_EXTENSIONS']: + configs = dict(settings=gen.settings) + gen.settings['MD_EXTENSIONS'].append(LiquidTags(configs)) + +def register(): + signals.initialized.connect(addLiquidTags) diff --git a/liquid_tags/literal.py b/liquid_tags/literal.py new file mode 100644 index 0000000..7c04602 --- /dev/null +++ b/liquid_tags/literal.py @@ -0,0 +1,27 @@ +""" +Literal Tag +----------- +This implements a tag that allows explicitly showing commands which would +otherwise be interpreted as a liquid tag. + +For example, the line + + {% literal video arg1 arg2 %} + +would result in the following line: + + {% video arg1 arg2 %} + +This is useful when the resulting line would be interpreted as another +liquid-style tag. +""" +from .mdx_liquid_tags import LiquidTags + +@LiquidTags.register('literal') +def literal(preprocessor, tag, markup): + return '{%% %s %%}' % markup + +#---------------------------------------------------------------------- +# This import allows image tag to be a Pelican plugin +from liquid_tags import register + diff --git a/liquid_tags/mdx_liquid_tags.py b/liquid_tags/mdx_liquid_tags.py new file mode 100644 index 0000000..3204f87 --- /dev/null +++ b/liquid_tags/mdx_liquid_tags.py @@ -0,0 +1,77 @@ +""" +Markdown Extension for Liquid-style Tags +---------------------------------------- +A markdown extension to allow user-defined tags of the form:: + + {% tag arg1 arg2 ... argn %} + +Where "tag" is associated with some user-defined extension. +These result in a preprocess step within markdown that produces +either markdown or html. +""" +import warnings +import markdown +import itertools +import re +import os +from functools import wraps + +# Define some regular expressions +LIQUID_TAG = re.compile(r'\{%.*?%\}') +EXTRACT_TAG = re.compile(r'(?:\s*)(\S+)(?:\s*)') + + +class _LiquidTagsPreprocessor(markdown.preprocessors.Preprocessor): + _tags = {} + def __init__(self, configs): + self.configs = configs + + def run(self, lines): + page = '\n'.join(lines) + liquid_tags = LIQUID_TAG.findall(page) + + for i, markup in enumerate(liquid_tags): + # remove {% %} + markup = markup[2:-2] + tag = EXTRACT_TAG.match(markup).groups()[0] + markup = EXTRACT_TAG.sub('', markup, 1) + if tag in self._tags: + liquid_tags[i] = self._tags[tag](self, tag, markup.strip()) + + # add an empty string to liquid_tags so that chaining works + liquid_tags.append('') + + # reconstruct string + page = ''.join(itertools.chain(*zip(LIQUID_TAG.split(page), + liquid_tags))) + + # resplit the lines + return page.split("\n") + + +class LiquidTags(markdown.Extension): + """Wrapper for MDPreprocessor""" + @classmethod + def register(cls, tag): + """Decorator to register a new include tag""" + def dec(func): + if tag in _LiquidTagsPreprocessor._tags: + warnings.warn("Enhanced Markdown: overriding tag '%s'" % tag) + _LiquidTagsPreprocessor._tags[tag] = func + return func + return dec + + def extendMarkdown(self, md, md_globals): + self.htmlStash = md.htmlStash + md.registerExtension(self) + # for the include_code preprocessor, we need to re-run the + # fenced code block preprocessor after substituting the code. + # Because the fenced code processor is run before, {% %} tags + # within equations will not be parsed as an include. + md.preprocessors.add('mdincludes', + _LiquidTagsPreprocessor(self), ">html_block") + + +def makeExtension(configs=None): + """Wrapper for a MarkDown extension""" + return LiquidTags(configs=configs) diff --git a/liquid_tags/notebook.py b/liquid_tags/notebook.py new file mode 100644 index 0000000..954efaf --- /dev/null +++ b/liquid_tags/notebook.py @@ -0,0 +1,290 @@ +""" +Notebook Tag +------------ +This is a liquid-style tag to include a static html rendering of an IPython +notebook in a blog post. + +Syntax +------ +{% notebook filename.ipynb [ cells[start:end] ]%} + +The file should be specified relative to the ``notebooks`` subdirectory of the +content directory. Optionally, this subdirectory can be specified in the +config file: + + NOTEBOOK_DIR = 'notebooks' + +The cells[start:end] statement is optional, and can be used to specify which +block of cells from the notebook to include. + +Requirements +------------ +- The plugin requires IPython version 1.0 or above. It no longer supports the + standalone nbconvert package, which has been deprecated. + +Details +------- +Because the notebook relies on some rather extensive custom CSS, the use of +this plugin requires additional CSS to be inserted into the blog theme. +After typing "make html" when using the notebook tag, a file called +``_nb_header.html`` will be produced in the main directory. The content +of the file should be included in the header of the theme. An easy way +to accomplish this is to add the following lines within the header template +of the theme you use: + + {% if EXTRA_HEADER %} + {{ EXTRA_HEADER }} + {% endif %} + +and in your ``pelicanconf.py`` file, include the line: + + EXTRA_HEADER = open('_nb_header.html').read().decode('utf-8') + +this will insert the appropriate CSS. All efforts have been made to ensure +that this CSS will not override formats within the blog theme, but there may +still be some conflicts. +""" +import re +import os +from .mdx_liquid_tags import LiquidTags + +import IPython +if IPython.__version__.split('.')[0] != 1: + raise ValueError("IPython version 1.0+ required for notebook tag") + +from IPython import nbconvert + +from IPython.nbconvert.filters.highlight import _pygment_highlight +from pygments.formatters import HtmlFormatter + +from IPython.nbconvert.exporters import HTMLExporter +from IPython.config import Config + +from IPython.nbformat import current as nbformat + +try: + from IPython.nbconvert.transformers import Transformer +except ImportError: + raise ValueError("IPython version 2.0 is not yet supported") + +from IPython.utils.traitlets import Integer +from copy import deepcopy + +from jinja2 import DictLoader + + +#---------------------------------------------------------------------- +# Some code that will be added to the header: +# Some of the following javascript/css include is adapted from +# IPython/nbconvert/templates/fullhtml.tpl, while some are custom tags +# specifically designed to make the results look good within the +# pelican-octopress theme. +JS_INCLUDE = r""" +<style type="text/css"> +/* Overrides of notebook CSS for static HTML export */ +div.entry-content { + overflow: visible; + padding: 8px; +} +.input_area { + padding: 0.2em; +} + +a.heading-anchor { + white-space: normal; +} + +.rendered_html +code { + font-size: .8em; +} + +pre.ipynb { + color: black; + background: #f7f7f7; + border: none; + box-shadow: none; + margin-bottom: 0; + padding: 0; + margin: 0px; + font-size: 13px; +} + +img.anim_icon{padding:0; border:0; vertical-align:middle; -webkit-box-shadow:none; -box-shadow:none} +</style> + +<script src="https://c328740.ssl.cf1.rackcdn.com/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" type="text/javascript"></script> +<script type="text/javascript"> +init_mathjax = function() { + if (window.MathJax) { + // MathJax loaded + MathJax.Hub.Config({ + tex2jax: { + inlineMath: [ ['$','$'], ["\\(","\\)"] ], + displayMath: [ ['$$','$$'], ["\\[","\\]"] ] + }, + displayAlign: 'left', // Change this to 'center' to center equations. + "HTML-CSS": { + styles: {'.MathJax_Display': {"margin": 0}} + } + }); + MathJax.Hub.Queue(["Typeset",MathJax.Hub]); + } +} +init_mathjax(); +</script> +""" + +CSS_WRAPPER = """ +<style type="text/css"> +{0} +</style> +""" + + +#---------------------------------------------------------------------- +# Create a custom transformer +class SliceIndex(Integer): + """An integer trait that accepts None""" + default_value = None + + def validate(self, obj, value): + if value is None: + return value + else: + return super(SliceIndex, self).validate(obj, value) + + +class SubCell(Transformer): + """A transformer to select a slice of the cells of a notebook""" + start = SliceIndex(0, config=True, + help="first cell of notebook to be converted") + end = SliceIndex(None, config=True, + help="last cell of notebook to be converted") + + def call(self, nb, resources): + nbc = deepcopy(nb) + for worksheet in nbc.worksheets : + cells = worksheet.cells[:] + worksheet.cells = cells[self.start:self.end] + return nbc, resources + + +#---------------------------------------------------------------------- +# Customize the html template: +# This changes the <pre> tags in basic_html.tpl to <pre class="ipynb" +pelican_loader = DictLoader({'pelicanhtml.tpl': +""" +{%- extends 'basichtml.tpl' -%} + +{% block stream_stdout -%} +<div class="box-flex1 output_subarea output_stream output_stdout"> +<pre class="ipynb">{{output.text |ansi2html}}</pre> +</div> +{%- endblock stream_stdout %} + +{% block stream_stderr -%} +<div class="box-flex1 output_subarea output_stream output_stderr"> +<pre class="ipynb">{{output.text |ansi2html}}</pre> +</div> +{%- endblock stream_stderr %} + +{% block pyerr -%} +<div class="box-flex1 output_subarea output_pyerr"> +<pre class="ipynb">{{super()}}</pre> +</div> +{%- endblock pyerr %} + +{%- block data_text %} +<pre class="ipynb">{{output.text | ansi2html}}</pre> +{%- endblock -%} +"""}) + + +#---------------------------------------------------------------------- +# Custom highlighter: +# instead of using class='highlight', use class='highlight-ipynb' +def custom_highlighter(source, language='ipython'): + formatter = HtmlFormatter(cssclass='highlight-ipynb') + output = _pygment_highlight(source, formatter, language) + return output.replace('<pre>', '<pre class="ipynb">') + + +#---------------------------------------------------------------------- +# Below is the pelican plugin code. +# +SYNTAX = "{% notebook /path/to/notebook.ipynb [ cells[start:end] ] %}" +FORMAT = re.compile(r"""^(\s+)?(?P<src>\S+)(\s+)?((cells\[)(?P<start>-?[0-9]*):(?P<end>-?[0-9]*)(\]))?(\s+)?$""") + + +@LiquidTags.register('notebook') +def notebook(preprocessor, tag, markup): + match = FORMAT.search(markup) + if match: + argdict = match.groupdict() + src = argdict['src'] + start = argdict['start'] + end = argdict['end'] + else: + raise ValueError("Error processing input, " + "expected syntax: {0}".format(SYNTAX)) + + if start: + start = int(start) + else: + start = 0 + + if end: + end = int(end) + else: + end = None + + settings = preprocessor.configs.config['settings'] + nb_dir = settings.get('NOTEBOOK_DIR', 'notebooks') + nb_path = os.path.join('content', nb_dir, src) + + if not os.path.exists(nb_path): + raise ValueError("File {0} could not be found".format(nb_path)) + + # Create the custom notebook converter + c = Config({'CSSHTMLHeaderTransformer': + {'enabled':True, 'highlight_class':'.highlight-ipynb'}, + 'SubCell': + {'enabled':True, 'start':start, 'end':end}}) + + exporter = HTMLExporter(config=c, + template_file='basic', + filters={'highlight2html': custom_highlighter}, + transformers=[SubCell], + extra_loaders=[pelican_loader]) + + # read and parse the notebook + with open(nb_path) as f: + nb_text = f.read() + nb_json = nbformat.reads_json(nb_text) + (body, resources) = exporter.from_notebook_node(nb_json) + + # if we haven't already saved the header, save it here. + if not notebook.header_saved: + print ("\n ** Writing styles to _nb_header.html: " + "this should be included in the theme. **\n") + + header = '\n'.join(CSS_WRAPPER.format(css_line) + for css_line in resources['inlining']['css']) + header += JS_INCLUDE + + with open('_nb_header.html', 'w') as f: + f.write(header) + notebook.header_saved = True + + # this will stash special characters so that they won't be transformed + # by subsequent processes. + body = preprocessor.configs.htmlStash.store(body, safe=True) + return body + +notebook.header_saved = False + + +#---------------------------------------------------------------------- +# This import allows notebook to be a Pelican plugin +from liquid_tags import register diff --git a/liquid_tags/video.py b/liquid_tags/video.py new file mode 100644 index 0000000..2bff9fb --- /dev/null +++ b/liquid_tags/video.py @@ -0,0 +1,70 @@ +""" +Video Tag +--------- +This implements a Liquid-style video tag for Pelican, +based on the octopress video tag [1]_ + +Syntax +------ +{% video url/to/video [width height] [url/to/poster] %} + +Example +------- +{% video http://site.com/video.mp4 720 480 http://site.com/poster-frame.jpg %} + +Output +------ +<video width='720' height='480' preload='none' controls poster='http://site.com/poster-frame.jpg'> + <source src='http://site.com/video.mp4' type='video/mp4; codecs=\"avc1.42E01E, mp4a.40.2\"'/> +</video> + +[1] https://github.com/imathis/octopress/blob/master/plugins/video_tag.rb +""" +import os +import re +from .mdx_liquid_tags import LiquidTags + +SYNTAX = "{% video url/to/video [url/to/video] [url/to/video] [width height] [url/to/poster] %}" + +VIDEO = re.compile(r'(/\S+|https?:\S+)(\s+(/\S+|https?:\S+))?(\s+(/\S+|https?:\S+))?(\s+(\d+)\s(\d+))?(\s+(/\S+|https?:\S+))?') + +VID_TYPEDICT = {'.mp4':"type='video/mp4; codecs=\"avc1.42E01E, mp4a.40.2\"'", + '.ogv':"type='video/ogg; codecs=theora, vorbis'", + '.webm':"type='video/webm; codecs=vp8, vorbis'"} + + +@LiquidTags.register('video') +def video(preprocessor, tag, markup): + videos = [] + width = None + height = None + poster = None + + match = VIDEO.search(markup) + if match: + groups = match.groups() + videos = [g for g in groups[0:6:2] if g] + width = groups[6] + height = groups[7] + poster = groups[9] + + if any(videos): + video_out = "<video width='{width}' height='{height}' preload='none' controls poster='{poster}'>".format(width=width, height=height, poster=poster) + for vid in videos: + base, ext = os.path.splitext(vid) + if ext not in VID_TYPEDICT: + raise ValueError("Unrecognized video extension: " + "{0}".format(ext)) + video_out += ("<source src='{0}' " + "{1}>".format(vid, VID_TYPEDICT[ext])) + video_out += "</video>" + else: + raise ValueError("Error processing input, " + "expected syntax: {0}".format(SYNTAX)) + + return video_out + + +#---------------------------------------------------------------------- +# This import allows image tag to be a Pelican plugin +from liquid_tags import register