markdown-tokens.cpp

Go to the documentation of this file.

 
/*
    Copyright (c) 2009 by Chad Nelson
    Released under the MIT License.
    See the provided LICENSE.TXT file for details.
*/
 
#include "markdown-tokens.h"
 
#include <boost/regex.hpp>
 
#include <stack>
#include <set>
#include <sstream>
 
namespace markdown::token
{
 
    namespace
    {
 
        const std::string cEscapedCharacters("\\`*_{}[]()#+-.!>");
 
        std::optional<size_t> isEscapedCharacter(char c)
        {
            std::string::const_iterator i = std::find(cEscapedCharacters.begin(),
                                            cEscapedCharacters.end(), c);
 
            if (i != cEscapedCharacters.end())
            {
                return std::distance(cEscapedCharacters.begin(), i);
            }
            else
            {
                return std::nullopt;
            }
        }
 
        char escapedCharacter(size_t index)
        {
            return cEscapedCharacters[index];
        }
 
        std::string encodeString(const std::string& src, int encodingFlags)
        {
            bool amps = (encodingFlags & cAmps) != 0,
                 doubleAmps = (encodingFlags & cDoubleAmps) != 0,
                 angleBrackets = (encodingFlags & cAngles) != 0,
                 quotes = (encodingFlags & cQuotes) != 0;
 
            std::string tgt;
 
            for (std::string::const_iterator i = src.begin(), ie = src.end(); i != ie; ++i)
            {
                if (*i == '&' && amps)
                {
                    static const boost::regex cIgnore("^(&amp;)|(&#[0-9]{1,3};)|(&#[xX][0-9a-fA-F]{1,2};)");
 
                    if (boost::regex_search(i, ie, cIgnore))
                    {
                        tgt.push_back(*i);
                    }
                    else
                    {
                        tgt += "&amp;";
                    }
                }
                else if (*i == '&' && doubleAmps)
                {
                    tgt += "&amp;";
                }
                else if (*i == '<' && angleBrackets)
                {
                    tgt += "&lt;";
                }
                else if (*i == '>' && angleBrackets)
                {
                    tgt += "&gt;";
                }
                else if (*i == '\"' && quotes)
                {
                    tgt += "&quot;";
                }
                else
                {
                    tgt.push_back(*i);
                }
            }
 
            return tgt;
        }
 
        bool looksLikeUrl(const std::string& str)
        {
            const char* schemes[] = { "http://", "https://", "ftp://", "ftps://",
                                      "file://", "www.", "ftp.", nullptr
                                    };
 
            for (size_t x = 0; schemes[x] != nullptr; ++x)
            {
                const char* s = str.c_str(), *t = schemes[x];
 
                while (*s != 0 && *t != 0 && *s == *t)
                {
                    ++s;
                    ++t;
                }
 
                if (*t == 0)
                {
                    return true;
                }
            }
 
            return false;
        }
 
        bool notValidNameCharacter(char c)
        {
            return !(isalnum(c) || c == '.' || c == '_' || c == '%' || c == '-' || c == '+');
        }
 
        bool notValidSiteCharacter(char c)
        {
            // NOTE: Kludge alert! The official spec for site characters is only
            // "a-zA-Z._%-". However, MDTest supports "international domain names,"
            // which use characters other than that; I'm kind of cheating here, handling
            // those by allowing all utf8-encoded characters too.
            return !(isalnum(c) || c == '.' || c == '_' || c == '%' || c == '-' || (c & 0x80));
        }
 
        bool isNotAlpha(char c)
        {
            return !isalpha(c);
        }
 
        std::string emailEncode(const std::string& src)
        {
            std::ostringstream out;
            bool inHex = false;
 
            for (char i : src)
            {
                if (i & 0x80)
                {
                    out << i;
                }
                else if (inHex)
                {
                    out << "&#x" << std::hex << static_cast<int>(i) << ';';
                }
                else
                {
                    out << "&#" << std::dec << static_cast<int>(i) << ';';
                }
 
                inHex = !inHex;
            }
 
            return out.str();
        }
 
        bool looksLikeEmailAddress(const std::string& str)
        {
            using Iter = std::string::const_iterator;
            using RIter = std::string::const_reverse_iterator;
            Iter i = std::find_if(str.begin(), str.end(), notValidNameCharacter);
 
            if (i != str.end() && *i == '@' && i != str.begin())
            {
                // The name part is valid.
                i = std::find_if(i + 1, str.end(), notValidSiteCharacter);
 
                if (i == str.end())
                {
                    // The site part doesn't contain any invalid characters.
                    RIter ri = std::find_if(str.rbegin(), str.rend(), isNotAlpha);
 
                    if (ri != str.rend() && *ri == '.')
                    {
                        // It ends with a dot and only alphabetic characters.
                        size_t d = std::distance(ri.base(), str.end());
 
                        if (d >= 2 && d <= 4)
                        {
                            // There are two-to-four of them. It's valid.
                            return true;
                        }
                    }
                }
            }
 
            return false;
        }
 
        // From <http://en.wikipedia.org/wiki/HTML_element>
 
        const char* cOtherTagInit[] =
        {
            // Header tags
            "title/", "base", "link", "basefont", "script/", "style/",
            "object/", "meta",
 
            // Inline tags
            "em/", "strong/", "q/", "cite/", "dfn/", "abbr/", "acronym/",
            "code/", "samp/", "kbd/", "var/", "sub/", "sup/", "del/", "ins/",
            "isindex", "a/", "img", "br", "map/", "area", "object/", "param",
            "applet/", "span/",
 
            nullptr
        };
 
        const char* cBlockTagInit[] = { "p/", "blockquote/", "hr", "h1/", "h2/",
                                        "h3/", "h4/", "h5/", "h6/", "dl/", "dt/", "dd/", "ol/", "ul/",
                                        "li/", "dir/", "menu/", "table/", "tr/", "th/", "td/", "col",
                                        "colgroup/", "caption/", "thead/", "tbody/", "tfoot/", "form/",
                                        "select/", "option", "input", "label/", "textarea/", "div/", "pre/",
                                        "address/", "iframe/", "frame/", "frameset/", "noframes/",
                                        "center/", "b/", "i/", "big/", "small/", /*"s/",*/ "strike/", "tt/",
                                        "u/", "font/", "ins/", "del/", nullptr
                                      };
 
        // Other official ones (not presently in use in this code)
        //"!doctype", "bdo", "body", "button", "fieldset", "head", "html",
        //"legend", "noscript", "optgroup", "xmp",
 
        std::set<std::string> otherTags, blockTags;
 
        void initTag(std::set<std::string>& set, const char* init[])
        {
            for (size_t x = 0; init[x] != nullptr; ++x)
            {
                std::string str = init[x];
 
                if (*str.rbegin() == '/')
                {
                    // Means it can have a closing tag
                    str = str.substr(0, str.length() - 1);
                }
 
                set.insert(str);
            }
        }
 
        std::string cleanTextLinkRef(const std::string& ref)
        {
            std::string r;
 
            for (char i : ref)
            {
                if (i == ' ')
                {
                    if (r.empty() || *r.rbegin() != ' ')
                    {
                        r.push_back(' ');
                    }
                }
                else
                {
                    r.push_back(i);
                }
            }
 
            return r;
        }
 
    } // namespace
 
 
 
    size_t isValidTag(const std::string& tag, bool nonBlockFirst)
    {
        if (blockTags.empty())
        {
            initTag(otherTags, cOtherTagInit);
            initTag(blockTags, cBlockTagInit);
        }
 
        if (nonBlockFirst)
        {
            if (otherTags.find(tag) != otherTags.end())
            {
                return 1;
            }
 
            if (blockTags.find(tag) != blockTags.end())
            {
                return 2;
            }
        }
        else
        {
            if (blockTags.find(tag) != blockTags.end())
            {
                return 2;
            }
 
            if (otherTags.find(tag) != otherTags.end())
            {
                return 1;
            }
        }
 
        return 0;
    }
 
 
 
    void TextHolder::writeAsHtml(std::ostream& out) const
    {
        preWrite(out);
 
        if (mEncodingFlags != 0)
        {
            out << encodeString(mText, mEncodingFlags);
        }
        else
        {
            out << mText;
        }
 
        postWrite(out);
    }
 
    std::optional<TokenGroup> RawText::processSpanElements(const LinkIds& idTable)
    {
        if (!canContainMarkup())
        {
            return std::nullopt;
        }
 
        ReplacementTable replacements;
        std::string str = _processHtmlTagAttributes(*text(), replacements);
        str = _processCodeSpans(str, replacements);
        str = _processEscapedCharacters(str);
        str = _processLinksImagesAndTags(str, replacements, idTable);
        return _processBoldAndItalicSpans(str, replacements);
    }
 
    std::string RawText::_processHtmlTagAttributes(std::string src, ReplacementTable&
            replacements)
    {
        // Because "Attribute Content Is Not A Code Span"
        std::string tgt;
        std::string::const_iterator prev = src.begin(), end = src.end();
 
        while (true)
        {
            static const boost::regex cHtmlToken("<((/?)([a-zA-Z0-9]+)(?:( +[a-zA-Z0-9]+?(?: ?= ?(\"|').*?\\5))+? */? *))>");
            boost::smatch m;
 
            if (boost::regex_search(prev, end, m, cHtmlToken))
            {
                // NOTE: Kludge alert! The `isValidTag` test is a cheat, only here
                // to handle some edge cases between the Markdown test suite and the
                // PHP-Markdown one, which seem to conflict.
                if (isValidTag(m[3]))
                {
                    tgt += std::string(prev, m[0].first);
 
                    std::string fulltag = m[0], tgttag;
                    std::string::const_iterator prevtag = fulltag.begin(), endtag = fulltag.end();
 
                    while (true)
                    {
                        static const boost::regex cAttributeStrings("= ?(\"|').*?\\1");
                        boost::smatch mtag;
 
                        if (boost::regex_search(prevtag, endtag, mtag, cAttributeStrings))
                        {
                            tgttag += std::string(prevtag, mtag[0].first);
                            tgttag += "\x01@" + std::to_string(replacements.size()) + "@htmlTagAttr\x01";
                            prevtag = mtag[0].second;
 
                            replacements.push_back(TokenPtr(new TextHolder(std::string(mtag[0]), false, cAmps | cAngles)));
                        }
                        else
                        {
                            tgttag += std::string(prevtag, endtag);
                            break;
                        }
                    }
 
                    tgt += tgttag;
                    prev = m[0].second;
                }
                else
                {
                    tgt += std::string(prev, m[0].second);
                    prev = m[0].second;
                }
            }
            else
            {
                tgt += std::string(prev, end);
                break;
            }
        }
 
        return tgt;
    }
 
    std::string RawText::_processCodeSpans(std::string src, ReplacementTable&
                                           replacements)
    {
        static const boost::regex cCodeSpan[2] =
        {
            boost::regex("(?:^|(?<=[^\\\\]))`` (.+?) ``"),
            boost::regex("(?:^|(?<=[^\\\\]))`(.+?)`")
        };
 
        for (const auto& pass : cCodeSpan)
        {
            std::string tgt;
            std::string::const_iterator prev = src.begin(), end = src.end();
 
            while (true)
            {
                boost::smatch m;
 
                if (boost::regex_search(prev, end, m, pass))
                {
                    tgt += std::string(prev, m[0].first);
                    tgt += "\x01@" + std::to_string(replacements.size()) + "@codeSpan\x01";
                    prev = m[0].second;
                    replacements.push_back(TokenPtr(new CodeSpan(_restoreProcessedItems(m[1], replacements))));
                }
                else
                {
                    tgt += std::string(prev, end);
                    break;
                }
            }
 
            src.swap(tgt);
            tgt.clear();
        }
 
        return src;
    }
 
    std::string RawText::_processEscapedCharacters(const std::string& src)
    {
        std::string tgt;
        std::string::const_iterator prev = src.begin(), end = src.end();
 
        while (true)
        {
            std::string::const_iterator i = std::find(prev, end, '\\');
 
            if (i != end)
            {
                tgt += std::string(prev, i);
                ++i;
 
                if (i != end)
                {
                    std::optional<size_t> e = isEscapedCharacter(*i);
 
                    if (e)
                    {
                        tgt += "\x01@#" + std::to_string(*e) + "@escaped\x01";
                    }
                    else
                    {
                        tgt = tgt + '\\' + *i;
                    }
 
                    prev = i + 1;
                }
                else
                {
                    tgt += '\\';
                    break;
                }
            }
            else
            {
                tgt += std::string(prev, end);
                break;
            }
        }
 
        return tgt;
    }
 
    std::string RawText::_processSpaceBracketedGroupings(const std::string& src,
            ReplacementTable& replacements)
    {
        static const boost::regex cRemove("(?:(?: \\*+ )|(?: _+ ))");
 
        std::string tgt;
        std::string::const_iterator prev = src.begin(), end = src.end();
 
        while (true)
        {
            boost::smatch m;
 
            if (boost::regex_search(prev, end, m, cRemove))
            {
                tgt += std::string(prev, m[0].first);
                tgt += "\x01@" + std::to_string(replacements.size()) + "@spaceBracketed\x01";
                replacements.push_back(TokenPtr(new RawText(m[0])));
                prev = m[0].second;
            }
            else
            {
                tgt += std::string(prev, end);
                break;
            }
        }
 
        return tgt;
    }
 
    std::string RawText::_processLinksImagesAndTags(const std::string& src,
            ReplacementTable& replacements, const LinkIds& idTable)
    {
        // NOTE: Kludge alert! The "inline link or image" regex should be...
        //
        //   "(?:(!?)\\[(.+?)\\] *\\((.*?)\\))"
        //
        // ...but that fails on the 'Images' test because it includes a "stupid URL"
        // that has parentheses within it. The proper way to deal with this would be
        // to match any nested parentheses, but regular expressions can't handle an
        // unknown number of nested items, so I'm cheating -- the regex for it
        // allows for one (and *only* one) pair of matched parentheses within the
        // URL. It makes the regex hard to follow (it was even harder to get right),
        // but it allows it to pass the test.
        //
        // The "reference link or image" one has a similar problem; it should be...
        //
        //   "|(?:(!?)\\[(.+?)\\](?: *\\[(.*?)\\])?)"
        //
        static const boost::regex cExpression(
            "(?:(!?)\\[([^\\]]+?)\\] *\\(([^\\(]*(?:\\(.*?\\).*?)*?)\\))" // Inline link or image
            "|(?:(!?)\\[((?:[^]]*?\\[.*?\\].*?)|(?:.+?))\\](?: *\\[(.*?)\\])?)" // Reference link or image
            "|(?:<(/?([a-zA-Z0-9]+).*?)>)" // potential HTML tag or auto-link
        );
        // Important captures: 1/4=image indicator, 2/5=contents/alttext,
        // 3=URL/title, 6=optional link ID, 7=potential HTML tag or auto-link
        // contents, 8=actual tag from 7.
 
        std::string tgt;
        std::string::const_iterator prev = src.begin(), end = src.end();
 
        while (true)
        {
            boost::smatch m;
 
            if (boost::regex_search(prev, end, m, cExpression))
            {
                assert(m[0].matched);
                assert(m[0].length() != 0);
 
                tgt += std::string(prev, m[0].first);
                tgt += "\x01@" + std::to_string(replacements.size()) + "@links&Images1\x01";
                prev = m[0].second;
 
                bool isImage = false, isLink = false, isReference = false;
 
                if (m[4].matched && m[4].length())
                {
                    isImage = isReference = true;
                }
                else if (m[1].matched && m[1].length())
                {
                    isImage = true;
                }
                else if (m[5].matched)
                {
                    isLink = isReference = true;
                }
                else if (m[2].matched)
                {
                    isLink = true;
                }
 
                if (isImage || isLink)
                {
                    std::string contentsOrAlttext, url, title;
                    bool resolved = false;
 
                    if (isReference)
                    {
                        contentsOrAlttext = m[5];
                        std::string linkId = (m[6].matched ? std::string(m[6]) : std::string());
 
                        if (linkId.empty())
                        {
                            linkId = cleanTextLinkRef(contentsOrAlttext);
                        }
 
                        std::optional<markdown::LinkIds::Target> target = idTable.find(linkId);
 
                        if (target)
                        {
                            url = target->url;
                            title = target->title;
                            resolved = true;
                        };
                    }
                    else
                    {
                        static const boost::regex cReference("^<?([^ >]*)>?(?: *(?:('|\")(.*)\\2)|(?:\\((.*)\\)))? *$");
                        // Useful captures: 1=url, 3/4=title
                        contentsOrAlttext = m[2];
                        std::string urlAndTitle = m[3];
                        boost::smatch mm;
 
                        if (boost::regex_match(urlAndTitle, mm, cReference))
                        {
                            url = mm[1];
 
                            if (mm[3].matched)
                            {
                                title = mm[3];
                            }
                            else if (mm[4].matched)
                            {
                                title = mm[4];
                            }
 
                            resolved = true;
                        }
                    }
 
                    if (!resolved)
                    {
                        // Just encode the first character as-is, and continue
                        // searching after it.
                        prev = m[0].first + 1;
                        replacements.push_back(TokenPtr(new RawText(std::string(m[0].first, prev))));
                    }
                    else if (isImage)
                    {
                        replacements.push_back(TokenPtr(new Image(contentsOrAlttext,
                                                        url, title)));
                    }
                    else
                    {
                        replacements.push_back(TokenPtr(new HtmlAnchorTag(url, title)));
                        tgt += contentsOrAlttext;
                        tgt += "\x01@" + std::to_string(replacements.size()) + "@links&Images2\x01";
                        replacements.push_back(TokenPtr(new HtmlTag("/a")));
                    }
                }
                else
                {
                    // Otherwise it's an HTML tag or auto-link.
                    std::string contents = m[7];
 
                    //              cerr << "Evaluating potential HTML or auto-link: " << contents << endl;
                    //              cerr << "m[8]=" << m[8] << endl;
 
                    if (looksLikeUrl(contents))
                    {
                        TokenGroup subgroup;
                        subgroup.push_back(TokenPtr(new HtmlAnchorTag(contents)));
                        subgroup.push_back(TokenPtr(new RawText(contents, false)));
                        subgroup.push_back(TokenPtr(new HtmlTag("/a")));
                        replacements.push_back(TokenPtr(new Container(subgroup)));
                    }
                    else if (looksLikeEmailAddress(contents))
                    {
                        TokenGroup subgroup;
                        subgroup.push_back(TokenPtr(new HtmlAnchorTag(emailEncode("mailto:" + contents))));
                        subgroup.push_back(TokenPtr(new RawText(emailEncode(contents), false)));
                        subgroup.push_back(TokenPtr(new HtmlTag("/a")));
                        replacements.push_back(TokenPtr(new Container(subgroup)));
                    }
                    else if (isValidTag(m[8]))
                    {
                        replacements.push_back(TokenPtr(new HtmlTag(_restoreProcessedItems(contents, replacements))));
                    }
                    else
                    {
                        // Just encode it as-is
                        replacements.push_back(TokenPtr(new RawText(m[0])));
                    }
                }
            }
            else
            {
                tgt += std::string(prev, end);
                break;
            }
        }
 
        return tgt;
    }
 
    TokenGroup RawText::_processBoldAndItalicSpans(const std::string& src,
            ReplacementTable& replacements)
    {
        static const boost::regex cEmphasisExpression(
            "(?:(?<![*_])([*_]{1,3})([^*_ ]+?)\\1(?![*_]))"                                    // Mid-word emphasis
            "|((?:(?<!\\*)\\*{1,3}(?!\\*)|(?<!_)_{1,3}(?!_))(?=.)(?! )(?![.,:;] )(?![.,:;]$))" // Open
            "|((?<![* ])\\*{1,3}(?!\\*)|(?<![ _])_{1,3}(?!_))"                                 // Close
        );
 
        TokenGroup tgt;
        std::string::const_iterator i = src.begin(), end = src.end(), prev = i;
 
        while (true)
        {
            boost::smatch m;
 
            if (boost::regex_search(prev, end, m, cEmphasisExpression))
            {
                if (prev != m[0].first) tgt.push_back(TokenPtr(new
                                                          RawText(std::string(prev, m[0].first))));
 
                if (m[3].matched)
                {
                    std::string token = m[3];
                    tgt.push_back(TokenPtr(new BoldOrItalicMarker(true, token[0],
                                           token.length())));
                    prev = m[0].second;
                }
                else if (m[4].matched)
                {
                    std::string token = m[4];
                    tgt.push_back(TokenPtr(new BoldOrItalicMarker(false, token[0],
                                           token.length())));
                    prev = m[0].second;
                }
                else
                {
                    std::string token = m[1], contents = m[2];
                    tgt.push_back(TokenPtr(new BoldOrItalicMarker(true, token[0],
                                           token.length())));
                    tgt.push_back(TokenPtr(new RawText(std::string(contents))));
                    tgt.push_back(TokenPtr(new BoldOrItalicMarker(false, token[0],
                                           token.length())));
                    prev = m[0].second;
                }
            }
            else
            {
                if (prev != end) tgt.push_back(TokenPtr(new RawText(std::string(prev,
                                                            end))));
 
                break;
            }
        }
 
        int id = 0;
 
        for (TokenGroup::iterator ii = tgt.begin(), iie = tgt.end(); ii != iie; ++ii)
        {
            if ((*ii)->isUnmatchedOpenMarker())
            {
                BoldOrItalicMarker* openToken = dynamic_cast<BoldOrItalicMarker*>(ii->get());
 
                // Find a matching close-marker, if it's there
                TokenGroup::iterator iii = ii;
 
                for (++iii; iii != iie; ++iii)
                {
                    if ((*iii)->isUnmatchedCloseMarker())
                    {
                        BoldOrItalicMarker* closeToken = dynamic_cast<BoldOrItalicMarker*>(iii->get());
 
                        if (closeToken->size() == 3 && openToken->size() != 3)
                        {
                            // Split the close-token into a match for the open-token
                            // and a second for the leftovers.
                            closeToken->disable();
                            TokenGroup g;
                            g.push_back(TokenPtr(new BoldOrItalicMarker(false,
                                                 closeToken->tokenCharacter(), closeToken->size() -
                                                 openToken->size())));
                            g.push_back(TokenPtr(new BoldOrItalicMarker(false,
                                                 closeToken->tokenCharacter(), openToken->size())));
                            TokenGroup::iterator after = iii;
                            ++after;
                            tgt.splice(after, g);
                            continue;
                        }
 
                        if (closeToken->tokenCharacter() == openToken->tokenCharacter()
                            && closeToken->size() == openToken->size())
                        {
                            openToken->matched(closeToken, id);
                            closeToken->matched(openToken, id);
                            ++id;
                            break;
                        }
                        else if (openToken->size() == 3)
                        {
                            // Split the open-token into a match for the close-token
                            // and a second for the leftovers.
                            openToken->disable();
                            TokenGroup g;
                            g.push_back(TokenPtr(new BoldOrItalicMarker(true,
                                                 openToken->tokenCharacter(), openToken->size() -
                                                 closeToken->size())));
                            g.push_back(TokenPtr(new BoldOrItalicMarker(true,
                                                 openToken->tokenCharacter(), closeToken->size())));
                            TokenGroup::iterator after = ii;
                            ++after;
                            tgt.splice(after, g);
                            break;
                        }
                    }
                }
            }
        }
 
        // "Unmatch" invalidly-nested matches.
        std::stack<BoldOrItalicMarker*> openMatches;
 
        for (auto& ii : tgt)
        {
            if (ii->isMatchedOpenMarker())
            {
                BoldOrItalicMarker* open = dynamic_cast<BoldOrItalicMarker*>(ii.get());
                openMatches.push(open);
            }
            else if (ii->isMatchedCloseMarker())
            {
                BoldOrItalicMarker* close = dynamic_cast<BoldOrItalicMarker*>(ii.get());
 
                if (close->id() != openMatches.top()->id())
                {
                    close->matchedTo()->matched(nullptr);
                    close->matched(nullptr);
                }
                else
                {
                    openMatches.pop();
 
                    while (!openMatches.empty() && openMatches.top()->matchedTo() == nullptr)
                    {
                        openMatches.pop();
                    }
                }
            }
        }
 
        TokenGroup r;
 
        for (auto& ii : tgt)
        {
            if (ii->text() && ii->canContainMarkup())
            {
                TokenGroup t = _encodeProcessedItems(*ii->text(), replacements);
                r.splice(r.end(), t);
            }
            else
            {
                r.push_back(ii);
            }
        }
 
        return r;
    }
 
    TokenGroup RawText::_encodeProcessedItems(const std::string& src,
            ReplacementTable& replacements)
    {
        static const boost::regex cReplaced("\x01@(#?[0-9]*)@.+?\x01");
 
        TokenGroup r;
        std::string::const_iterator prev = src.begin();
 
        while (true)
        {
            boost::smatch m;
 
            if (boost::regex_search(prev, src.end(), m, cReplaced))
            {
                std::string pre = std::string(prev, m[0].first);
 
                if (!pre.empty())
                {
                    r.push_back(TokenPtr(new RawText(pre)));
                }
 
                prev = m[0].second;
 
                std::string ref = m[1];
 
                if (ref[0] == '#')
                {
                    size_t n = std::stoul(ref.substr(1));
                    r.push_back(TokenPtr(new EscapedCharacter(escapedCharacter(n))));
                }
                else if (!ref.empty())
                {
                    size_t n = std::stoul(ref);
 
                    assert(n < replacements.size());
                    r.push_back(replacements[n]);
                } // Otherwise just eat it
            }
            else
            {
                std::string pre = std::string(prev, src.end());
 
                if (!pre.empty())
                {
                    r.push_back(TokenPtr(new RawText(pre)));
                }
 
                break;
            }
        }
 
        return r;
    }
 
    std::string RawText::_restoreProcessedItems(const std::string& src,
            ReplacementTable& replacements)
    {
        static const boost::regex cReplaced("\x01@(#?[0-9]*)@.+?\x01");
 
        std::ostringstream r;
        std::string::const_iterator prev = src.begin();
 
        while (true)
        {
            boost::smatch m;
 
            if (boost::regex_search(prev, src.end(), m, cReplaced))
            {
                std::string pre = std::string(prev, m[0].first);
 
                if (!pre.empty())
                {
                    r << pre;
                }
 
                prev = m[0].second;
 
                std::string ref = m[1];
 
                if (ref[0] == '#')
                {
                    size_t n = std::stoul(ref.substr(1));
                    r << '\\' << escapedCharacter(n);
                }
                else if (!ref.empty())
                {
                    size_t n = std::stoul(ref);
 
                    assert(n < replacements.size());
                    replacements[n]->writeAsOriginal(r);
                } // Otherwise just eat it
            }
            else
            {
                std::string pre = std::string(prev, src.end());
 
                if (!pre.empty())
                {
                    r << pre;
                }
 
                break;
            }
        }
 
        return r.str();
    }
 
    HtmlAnchorTag::HtmlAnchorTag(const std::string& url, const std::string& title):
        TextHolder("<a href=\"" + encodeString(url, cQuotes | cAmps) + "\""
                   + (title.empty() ? std::string() : " title=\"" + encodeString(title, cQuotes | cAmps) + "\"")
                   + ">", false, 0)
    {
        // This space deliberately blank. ;-)
    }
 
    void CodeBlock::writeAsHtml(std::ostream& out) const
    {
        out << "<pre><code>";
        TextHolder::writeAsHtml(out);
        out << "</code></pre>\n\n";
    }
 
    void CodeSpan::writeAsHtml(std::ostream& out) const
    {
        out << "<code>";
        TextHolder::writeAsHtml(out);
        out << "</code>";
    }
 
    void CodeSpan::writeAsOriginal(std::ostream& out) const
    {
        out << '`' << *text() << '`';
    }
 
 
 
    void Container::writeAsHtml(std::ostream& out) const
    {
        preWrite(out);
 
        for (const auto& mSubToken : mSubTokens)
        {
            mSubToken->writeAsHtml(out);
        }
 
        postWrite(out);
    }
 
    void Container::writeToken(size_t indent, std::ostream& out) const
    {
        out << std::string(indent * 2, ' ') << containerName() << "\n";
 
        for (const auto& mSubToken : mSubTokens)
        {
            mSubToken->writeToken(indent + 1, out);
        }
    }
 
    std::optional<TokenGroup> Container::processSpanElements(const LinkIds& idTable)
    {
        TokenGroup t;
 
        for (CTokenGroupIter ii = mSubTokens.begin(), iie = mSubTokens.end(); ii != iie;
             ++ii)
        {
            if ((*ii)->text())
            {
                std::optional<TokenGroup> subt = (*ii)->processSpanElements(idTable);
 
                if (subt)
                {
                    if (subt->size() > 1)
                    {
                        t.push_back(TokenPtr(new Container(*subt)));
                    }
                    else if (!subt->empty())
                    {
                        t.push_back(*subt->begin());
                    }
                }
                else
                {
                    t.push_back(*ii);
                }
            }
            else
            {
                std::optional<TokenGroup> subt = (*ii)->processSpanElements(idTable);
 
                if (subt)
                {
                    const Container* c = dynamic_cast<const Container*>((*ii).get());
                    assert(c != 0);
                    t.push_back(c->clone(*subt));
                }
                else
                {
                    t.push_back(*ii);
                }
            }
        }
 
        swapSubtokens(t);
        return std::nullopt;
    }
 
    UnorderedList::UnorderedList(const TokenGroup& contents, bool paragraphMode)
    {
        if (paragraphMode)
        {
            // Change each of the text items into paragraphs
            for (const auto& content : contents)
            {
                token::ListItem* item = dynamic_cast<token::ListItem*>(content.get());
                assert(item != 0);
                item->inhibitParagraphs(false);
                mSubTokens.push_back(content);
            }
        }
        else
        {
            mSubTokens = contents;
        }
    }
 
 
 
    void BoldOrItalicMarker::writeAsHtml(std::ostream& out) const
    {
        if (!mDisabled)
        {
            if (mMatch != nullptr)
            {
                assert(mSize >= 1 && mSize <= 3);
 
                if (mOpenMarker)
                {
                    out << (mSize == 1 ? "<em>" : mSize == 2 ? "<strong>" : "<strong><em>");
                }
                else
                {
                    out << (mSize == 1 ? "</em>" : mSize == 2 ? "</strong>" : "</em></strong>");
                }
            }
            else
            {
                out << std::string(mSize, mTokenCharacter);
            }
        }
    }
 
    void BoldOrItalicMarker::writeToken(std::ostream& out) const
    {
        if (!mDisabled)
        {
            if (mMatch != nullptr)
            {
                std::string type = (mSize == 1 ? "italic" : mSize == 2 ? "bold" : "italic&bold");
 
                if (mOpenMarker)
                {
                    out << "Matched open-" << type << " marker\n";
                }
                else
                {
                    out << "Matched close-" << type << " marker\n";
                }
            }
            else
            {
                if (mOpenMarker) out << "Unmatched bold/italic open marker: " <<
                                         std::string(mSize, mTokenCharacter) << "\n";
                else out << "Unmatched bold/italic close marker: " <<
                             std::string(mSize, mTokenCharacter) << "\n";
            }
        }
    }
 
    void Image::writeAsHtml(std::ostream& out) const
    {
        out << "<img src=\"" << mUrl << "\" alt=\"" << mAltText << "\"";
 
        if (!mTitle.empty())
        {
            out << " title=\"" << mTitle << "\"";
        }
 
        out << "/>";
    }
 
}