| """loads Markdown files, converts each one to HTML and parses the HTML into an ElementTree structure. |
| The collection of ElementTrees are further parsed to generate a table of contents structure, and are |
| manipulated to replace various markdown-generated HTML with specific Mako tags before being written |
| to Mako templates, which then re-access the table of contents structure at runtime. |
| |
| Much thanks to Alexey Shamrin, who came up with the original idea and did all the heavy Markdown/Elementtree |
| lifting for this module.""" |
| import sys, re, os |
| from toc import TOCElement |
| |
| try: |
| import xml.etree.ElementTree as et |
| except: |
| try: |
| import elementtree.ElementTree as et |
| except: |
| raise "This module requires ElementTree to run (http://effbot.org/zone/element-index.htm)" |
| |
| import markdown |
| |
| def dump_tree(elem, stream): |
| if elem.tag.startswith('MAKO:'): |
| dump_mako_tag(elem, stream) |
| else: |
| if elem.tag != 'html': |
| if len(elem.attrib): |
| stream.write("<%s %s>" % (elem.tag, " ".join(["%s=%s" % (key, repr(val)) for key, val in elem.attrib.iteritems()]))) |
| else: |
| stream.write("<%s>" % elem.tag) |
| if elem.text: |
| stream.write(elem.text) |
| for child in elem: |
| dump_tree(child, stream) |
| if child.tail: |
| stream.write(child.tail) |
| stream.write("</%s>" % elem.tag) |
| |
| def dump_mako_tag(elem, stream): |
| tag = elem.tag[5:] |
| params = ','.join(['%s=%s' % i for i in elem.items()]) |
| stream.write('<%%call expr="%s(%s)">' % (tag, params)) |
| if elem.text: |
| stream.write(elem.text) |
| for n in elem: |
| dump_tree(n, stream) |
| if n.tail: |
| stream.write(n.tail) |
| stream.write("</%call>") |
| |
| def create_toc(filename, tree, tocroot): |
| title = [None] |
| current = [tocroot] |
| level = [0] |
| def process(tree): |
| while True: |
| i = find_header_index(tree) |
| if i is None: |
| return |
| node = tree[i] |
| taglevel = int(node.tag[1]) |
| start, end = i, end_of_header(tree, taglevel, i+1) |
| content = tree[start+1:end] |
| description = node.text.strip() |
| if title[0] is None: |
| title[0] = description |
| name = node.get('name') |
| if name is None: |
| name = description.split()[0].lower() |
| |
| taglevel = node.tag[1] |
| if taglevel > level[0]: |
| current[0] = TOCElement(filename, name, description, current[0]) |
| elif taglevel == level[0]: |
| current[0] = TOCElement(filename, name, description, current[0].parent) |
| else: |
| current[0] = TOCElement(filename, name, description, current[0].parent.parent) |
| |
| level[0] = taglevel |
| |
| tag = et.Element("MAKO:formatting.section", path=repr(current[0].path), paged='paged', extension='extension', toc='toc') |
| tag.text = (node.tail or "") + '\n' |
| tag.tail = '\n' |
| tag[:] = content |
| tree[start:end] = [tag] |
| |
| process(tag) |
| |
| process(tree) |
| return (title[0], tocroot.get_by_file(filename)) |
| |
| def index(parent, item): |
| for n, i in enumerate(parent): |
| if i is item: |
| return n |
| |
| def find_header_index(tree): |
| for i, node in enumerate(tree): |
| if is_header(node): |
| return i |
| |
| def is_header(node): |
| t = node.tag |
| return (isinstance(t, str) and len(t) == 2 and t[0] == 'h' |
| and t[1] in '123456789') |
| |
| def end_of_header(tree, level, start): |
| for i, node in enumerate(tree[start:]): |
| if is_header(node) and int(node.tag[1]) <= level: |
| return start + i |
| return len(tree) |
| |
| def process_rel_href(tree): |
| parent = get_parent_map(tree) |
| for a in tree.findall('.//a'): |
| m = re.match(r'(bold)?rel\:(.+)', a.get('href')) |
| if m: |
| (bold, path) = m.group(1,2) |
| text = a.text |
| if text == path: |
| tag = et.Element("MAKO:nav.toclink", path=repr(path), extension='extension', paged='paged', toc='toc') |
| else: |
| tag = et.Element("MAKO:nav.toclink", path=repr(path), description=repr(text), extension='extension', paged='paged', toc='toc') |
| a_parent = parent[a] |
| if bold: |
| bold = et.Element('strong') |
| bold.tail = a.tail |
| bold.append(tag) |
| a_parent[index(a_parent, a)] = bold |
| else: |
| tag.tail = a.tail |
| a_parent[index(a_parent, a)] = tag |
| |
| def replace_pre_with_mako(tree): |
| parents = get_parent_map(tree) |
| |
| for precode in tree.findall('.//pre/code'): |
| reg = re.compile(r'\{(python|mako|html|ini)(?: title="(.*?)"){0,1}\}(.*)', re.S) |
| m = reg.match(precode[0].text.lstrip()) |
| if m: |
| code = m.group(1) |
| title = m.group(2) |
| precode[0].text = m.group(3) |
| else: |
| code = title = None |
| |
| tag = et.Element("MAKO:formatting.code") |
| if code: |
| tag.attrib["syntaxtype"] = repr(code) |
| if title: |
| tag.attrib["title"] = repr(title) |
| tag.text = precode.text |
| [tag.append(x) for x in precode] |
| pre = parents[precode] |
| tag.tail = pre.tail |
| |
| pre_parent = parents[pre] |
| pre_parent[reverse_parent(pre_parent, pre)] = tag |
| |
| def safety_code(tree): |
| parents = get_parent_map(tree) |
| for code in tree.findall('.//code'): |
| tag = et.Element('%text') |
| if parents[code].tag != 'pre': |
| tag.attrib["filter"] = "h" |
| tag.text = code.text |
| code.append(tag) |
| code.text = "" |
| |
| def reverse_parent(parent, item): |
| for n, i in enumerate(parent): |
| if i is item: |
| return n |
| |
| def get_parent_map(tree): |
| return dict([(c, p) for p in tree.getiterator() for c in p]) |
| |
| def header(toc, title, filename): |
| return """# -*- coding: utf-8 -*- |
| <%%inherit file="content_layout.html"/> |
| <%%page args="toc, extension, paged"/> |
| <%%namespace name="formatting" file="formatting.html"/> |
| <%%namespace name="nav" file="nav.html"/> |
| <%%def name="title()">%s - %s</%%def> |
| <%%! |
| filename = '%s' |
| %%> |
| ## This file is generated. Edit the .txt files instead of this one. |
| """ % (toc.root.doctitle, title, filename) |
| |
| class utf8stream(object): |
| def __init__(self, stream): |
| self.stream = stream |
| def write(self, str): |
| self.stream.write(str.encode('utf8')) |
| |
| def parse_markdown_files(toc, files): |
| for inname in files: |
| infile = 'content/%s.txt' % inname |
| if not os.access(infile, os.F_OK): |
| continue |
| html = markdown.markdown(file(infile).read()) |
| tree = et.fromstring("<html>" + html + "</html>") |
| (title, toc_element) = create_toc(inname, tree, toc) |
| safety_code(tree) |
| replace_pre_with_mako(tree) |
| process_rel_href(tree) |
| outname = 'output/%s.html' % inname |
| print infile, '->', outname |
| outfile = utf8stream(file(outname, 'w')) |
| outfile.write(header(toc, title, inname)) |
| dump_tree(tree, outfile) |
| |
| |
