markdown-tokens.h

Go to the documentation of this file.

 
/*
    Copyright (c) 2009 by Chad Nelson
    Released under the MIT License.
    See the provided LICENSE.TXT file for details.
*/
 
#pragma once
 
#include "markdown.h"
 
#include <unordered_map>
#include <optional>
#include <vector>
 
namespace markdown
{
    using TokenGroupIter = TokenGroup::iterator;
    using CTokenGroupIter = TokenGroup::const_iterator;
 
    class LinkIds
    {
    public:
        struct Target
        {
            std::string url;
            std::string title;
 
            Target(const std::string& url_, const std::string& title_):
                url(url_), title(title_) { }
        };
 
        std::optional<Target> find(const std::string& id) const;
        void add(const std::string& id, const std::string& url, const
                 std::string& title);
 
    private:
        using Table = std::unordered_map<std::string, Target>;
 
        static std::string _scrubKey(std::string str);
 
        Table mTable;
    };
 
    class Token
    {
    public:
        Token() { }
 
        virtual void writeAsHtml(std::ostream&) const = 0;
        virtual void writeAsOriginal(std::ostream& out) const
        {
            writeAsHtml(out);
        }
        virtual void writeToken(std::ostream& out) const = 0;
        virtual void writeToken(size_t indent, std::ostream& out) const
        {
            out << std::string(indent * 2, ' ');
            writeToken(out);
        }
 
        virtual std::optional<TokenGroup> processSpanElements(const LinkIds& idTable)
        {
            return std::nullopt;
        }
 
        virtual std::optional<std::string> text() const
        {
            return std::nullopt;
        }
 
        virtual bool canContainMarkup() const
        {
            return false;
        }
        virtual bool isBlankLine() const
        {
            return false;
        }
        virtual bool isContainer() const
        {
            return false;
        }
        virtual bool isUnmatchedOpenMarker() const
        {
            return false;
        }
        virtual bool isUnmatchedCloseMarker() const
        {
            return false;
        }
        virtual bool isMatchedOpenMarker() const
        {
            return false;
        }
        virtual bool isMatchedCloseMarker() const
        {
            return false;
        }
        virtual bool inhibitParagraphs() const
        {
            return false;
        }
 
    protected:
        virtual void preWrite(std::ostream& out) const { }
        virtual void postWrite(std::ostream& out) const { }
    };
}
 
namespace markdown::token
{
 
    size_t isValidTag(const std::string& tag, bool nonBlockFirst = false);
 
    enum EncodingFlags { cAmps = 0x01, cDoubleAmps = 0x02, cAngles = 0x04, cQuotes = 0x08 };
 
    class TextHolder: public Token
    {
    public:
        TextHolder(const std::string& text, bool canContainMarkup, unsigned int
                   encodingFlags): mText(text), mCanContainMarkup(canContainMarkup),
            mEncodingFlags(encodingFlags) { }
 
        void writeAsHtml(std::ostream& out) const override;
 
        void writeToken(std::ostream& out) const override
        {
            out << "TextHolder: " << mText << '\n';
        }
 
        std::optional<std::string> text() const override
        {
            return mText;
        }
 
        bool canContainMarkup() const override
        {
            return mCanContainMarkup;
        }
 
    private:
        const std::string mText;
        const bool mCanContainMarkup;
        const int mEncodingFlags;
    };
 
    class RawText: public TextHolder
    {
    public:
        RawText(const std::string& text, bool canContainMarkup = true):
            TextHolder(text, canContainMarkup, cAmps | cAngles | cQuotes) { }
 
        void writeToken(std::ostream& out) const override
        {
            out << "RawText: " << *text() << '\n';
        }
 
        std::optional<TokenGroup> processSpanElements(const LinkIds& idTable) override;
 
    private:
        using ReplacementTable = std::vector<TokenPtr>;
 
        static std::string _processHtmlTagAttributes(std::string src, ReplacementTable& replacements);
        static std::string _processCodeSpans(std::string src, ReplacementTable& replacements);
        static std::string _processEscapedCharacters(const std::string& src);
        static std::string _processLinksImagesAndTags(const std::string& src, ReplacementTable& replacements, const LinkIds& idTable);
        static std::string _processSpaceBracketedGroupings(const std::string& src, ReplacementTable& replacements);
        static TokenGroup _processBoldAndItalicSpans(const std::string& src, ReplacementTable& replacements);
 
        static TokenGroup _encodeProcessedItems(const std::string& src, ReplacementTable& replacements);
        static std::string _restoreProcessedItems(const std::string& src, ReplacementTable& replacements);
    };
 
    class HtmlTag: public TextHolder
    {
    public:
        HtmlTag(const std::string& contents): TextHolder(contents, false, cAmps | cAngles) { }
 
        void writeToken(std::ostream& out) const override
        {
            out << "HtmlTag: " << *text() << '\n';
        }
 
    protected:
        void preWrite(std::ostream& out) const override
        {
            out << '<';
        }
        void postWrite(std::ostream& out) const override
        {
            out << '>';
        }
    };
 
    class HtmlAnchorTag: public TextHolder
    {
    public:
        HtmlAnchorTag(const std::string& url, const std::string& title = std::string());
 
        void writeToken(std::ostream& out) const override
        {
            out << "HtmlAnchorTag: " << *text() << '\n';
        }
    };
 
    class InlineHtmlContents: public TextHolder
    {
    public:
        InlineHtmlContents(const std::string& contents): TextHolder(contents, false,
                    cAmps | cAngles) { }
 
        void writeToken(std::ostream& out) const override
        {
            out << "InlineHtmlContents: " << *text() << '\n';
        }
    };
 
    class InlineHtmlComment: public TextHolder
    {
    public:
        InlineHtmlComment(const std::string& contents): TextHolder(contents, false,
                    0) { }
 
        void writeToken(std::ostream& out) const override
        {
            out << "InlineHtmlComment: " << *text() << '\n';
        }
    };
 
    class CodeBlock: public TextHolder
    {
    public:
        CodeBlock(const std::string& actualContents): TextHolder(actualContents,
                    false, cDoubleAmps | cAngles | cQuotes) { }
 
        void writeAsHtml(std::ostream& out) const override;
 
        void writeToken(std::ostream& out) const override
        {
            out << "CodeBlock: " << *text() << '\n';
        }
    };
 
    class CodeSpan: public TextHolder
    {
    public:
        CodeSpan(const std::string& actualContents): TextHolder(actualContents,
                    false, cDoubleAmps | cAngles | cQuotes) { }
 
        void writeAsHtml(std::ostream& out) const override;
        void writeAsOriginal(std::ostream& out) const override;
        void writeToken(std::ostream& out) const override
        {
            out << "CodeSpan: " << *text() << '\n';
        }
    };
 
    class Header: public TextHolder
    {
    public:
        Header(size_t level, const std::string& text): TextHolder(text, true,
                    cAmps | cAngles | cQuotes), mLevel(level) { }
 
        void writeToken(std::ostream& out) const override
        {
            out << "Header " <<
                mLevel << ": " << *text() << '\n';
        }
 
        bool inhibitParagraphs() const override
        {
            return true;
        }
 
    protected:
        void preWrite(std::ostream& out) const override
        {
            out << "<h" << mLevel << ">";
        }
        void postWrite(std::ostream& out) const override
        {
            out << "</h" << mLevel << ">\n";
        }
 
    private:
        size_t mLevel;
    };
 
    class BlankLine: public TextHolder
    {
    public:
        BlankLine(const std::string& actualContents = std::string()):
            TextHolder(actualContents, false, 0) { }
 
        void writeToken(std::ostream& out) const override
        {
            out << "BlankLine: " << *text() << '\n';
        }
 
        bool isBlankLine() const override
        {
            return true;
        }
    };
 
 
 
    class EscapedCharacter: public Token
    {
    public:
        EscapedCharacter(char c): mChar(c) { }
 
        void writeAsHtml(std::ostream& out) const override
        {
            out << mChar;
        }
        void writeAsOriginal(std::ostream& out) const override
        {
            out << '\\' << mChar;
        }
        void writeToken(std::ostream& out) const override
        {
            out << "EscapedCharacter: " << mChar << '\n';
        }
 
    private:
        const char mChar;
    };
 
 
 
    class Container: public Token
    {
    public:
        Container(const TokenGroup& contents = TokenGroup()): mSubTokens(contents),
            mParagraphMode(false) { }
 
        const TokenGroup& subTokens() const
        {
            return mSubTokens;
        }
        void appendSubtokens(TokenGroup& tokens)
        {
            mSubTokens.splice(mSubTokens.end(), tokens);
        }
        void swapSubtokens(TokenGroup& tokens)
        {
            mSubTokens.swap(tokens);
        }
 
        bool isContainer() const override
        {
            return true;
        }
 
        void writeAsHtml(std::ostream& out) const override;
 
        void writeToken(std::ostream& out) const override
        {
            out << "Container: error!" << '\n';
        }
        void writeToken(size_t indent, std::ostream& out) const override;
 
        std::optional<TokenGroup> processSpanElements(const LinkIds& idTable) override;
 
        virtual TokenPtr clone(const TokenGroup& newContents) const
        {
            return TokenPtr(new Container(newContents));
        }
        virtual std::string containerName() const
        {
            return "Container";
        }
 
    protected:
        TokenGroup mSubTokens;
        bool mParagraphMode;
    };
 
    class InlineHtmlBlock: public Container
    {
    public:
        InlineHtmlBlock(const TokenGroup& contents, bool isBlockTag = false):
            Container(contents), mIsBlockTag(isBlockTag) { }
        InlineHtmlBlock(const std::string& contents): mIsBlockTag(false)
        {
            mSubTokens.push_back(TokenPtr(new InlineHtmlContents(contents)));
        }
 
        bool inhibitParagraphs() const override
        {
            return !mIsBlockTag;
        }
 
        TokenPtr clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new InlineHtmlBlock(newContents));
        }
        std::string containerName() const override
        {
            return "InlineHtmlBlock";
        }
 
        // Inline HTML blocks always end with a blank line, so report it as one for
        // parsing purposes.
        bool isBlankLine() const override
        {
            return true;
        }
 
    private:
        bool mIsBlockTag;
    };
 
    class ListItem: public Container
    {
    public:
        ListItem(const TokenGroup& contents): Container(contents),
            mInhibitParagraphs(true) { }
 
        void inhibitParagraphs(bool set)
        {
            mInhibitParagraphs = set;
        }
 
        bool inhibitParagraphs() const override
        {
            return mInhibitParagraphs;
        }
 
        TokenPtr clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new ListItem(newContents));
        }
        std::string containerName() const override
        {
            return "ListItem";
        }
 
    protected:
        void preWrite(std::ostream& out) const override
        {
            out << "<li>";
        }
        void postWrite(std::ostream& out) const override
        {
            out << "</li>\n";
        }
 
    private:
        bool mInhibitParagraphs;
    };
 
    class UnorderedList: public Container
    {
    public:
        UnorderedList(const TokenGroup& contents, bool paragraphMode = false);
 
        TokenPtr clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new UnorderedList(newContents));
        }
        std::string containerName() const override
        {
            return "UnorderedList";
        }
 
    protected:
        void preWrite(std::ostream& out) const override
        {
            out << "\n<ul>\n";
        }
        void postWrite(std::ostream& out) const override
        {
            out << "</ul>\n\n";
        }
    };
 
    class OrderedList: public UnorderedList
    {
    public:
        OrderedList(const TokenGroup& contents, bool paragraphMode = false):
            UnorderedList(contents, paragraphMode) { }
 
        TokenPtr clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new OrderedList(newContents));
        }
        std::string containerName() const override
        {
            return "OrderedList";
        }
 
    protected:
        void preWrite(std::ostream& out) const override
        {
            out << "<ol>\n";
        }
        void postWrite(std::ostream& out) const override
        {
            out << "</ol>\n\n";
        }
    };
 
    class BlockQuote: public Container
    {
    public:
        BlockQuote(const TokenGroup& contents): Container(contents) { }
 
        TokenPtr clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new BlockQuote(newContents));
        }
        std::string containerName() const override
        {
            return "BlockQuote";
        }
 
    protected:
        void preWrite(std::ostream& out) const override
        {
            out << "<blockquote>\n";
        }
        void postWrite(std::ostream& out) const override
        {
            out << "\n</blockquote>\n";
        }
    };
 
    class Paragraph: public Container
    {
    public:
        Paragraph() { }
        Paragraph(const TokenGroup& contents): Container(contents) { }
 
        TokenPtr clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new Paragraph(newContents));
        }
        std::string containerName() const override
        {
            return "Paragraph";
        }
 
    protected:
        void preWrite(std::ostream& out) const override
        {
            out << "<p>";
        }
        void postWrite(std::ostream& out) const override
        {
            out << "</p>\n\n";
        }
    };
 
 
 
    class BoldOrItalicMarker: public Token
    {
    public:
        BoldOrItalicMarker(bool open, char c, size_t size): mOpenMarker(open),
            mTokenCharacter(c), mSize(size), mMatch(0), mCannotMatch(false),
            mDisabled(false), mId(-1) { }
 
        bool isUnmatchedOpenMarker() const override
        {
            return (mOpenMarker && mMatch == 0 && !mCannotMatch);
        }
        bool isUnmatchedCloseMarker() const override
        {
            return (!mOpenMarker && mMatch == 0 && !mCannotMatch);
        }
        bool isMatchedOpenMarker() const override
        {
            return (mOpenMarker && mMatch != 0);
        }
        bool isMatchedCloseMarker() const override
        {
            return (!mOpenMarker && mMatch != 0);
        }
        void writeAsHtml(std::ostream& out) const override;
        void writeToken(std::ostream& out) const override;
 
        bool isOpenMarker() const
        {
            return mOpenMarker;
        }
        char tokenCharacter() const
        {
            return mTokenCharacter;
        }
        size_t size() const
        {
            return mSize;
        }
        bool matched() const
        {
            return (mMatch != 0);
        }
        BoldOrItalicMarker* matchedTo() const
        {
            return mMatch;
        }
        int id() const
        {
            return mId;
        }
 
        void matched(BoldOrItalicMarker* match, int id = -1)
        {
            mMatch = match;
            mId = id;
        }
        void cannotMatch(bool set)
        {
            mCannotMatch = set;
        }
        void disable()
        {
            mCannotMatch = mDisabled = true;
        }
 
    private:
        bool mOpenMarker; // Otherwise it's a close-marker
        char mTokenCharacter; // Underscore or asterisk
        size_t mSize; // 1=italics, 2=bold, 3=both
        BoldOrItalicMarker* mMatch;
        bool mCannotMatch;
        bool mDisabled;
        int mId;
    };
 
    class Image: public Token
    {
    public:
        Image(const std::string& altText, const std::string& url, const std::string&
              title): mAltText(altText), mUrl(url), mTitle(title) { }
 
        void writeAsHtml(std::ostream& out) const override;
 
        void writeToken(std::ostream& out) const override
        {
            out << "Image: " << mUrl << '\n';
        }
 
    private:
        const std::string mAltText, mUrl, mTitle;
    };
 
}