markdown-tokens.h

Go to the documentation of this file.

 
/*
    Copyright (c) 2009 by Chad Nelson
    Released under the MIT License.
    See the provided LICENSE.TXT file for details.
*/
 
#pragma once
 
#include <optional>
#include <unordered_map>
#include <vector>
 
#include "markdown.h"
 
namespace markdown
{
    using TokenGroupIter = TokenGroup::iterator;
    using CTokenGroupIter = TokenGroup::const_iterator;
 
    class LinkIds
    {
    public:
        struct Target
        {
            std::string url;
            std::string title;
 
            Target(const std::string& url_, const std::string& title_) : url(url_), title(title_)
            {
            }
        };
 
        std::optional<Target> find(const std::string& id) const;
        void add(const std::string& id, const std::string& url, const std::string& title);
 
    private:
        using Table = std::unordered_map<std::string, Target>;
 
        static std::string _scrubKey(std::string str);
 
        Table mTable;
    };
 
    class Token
    {
    public:
        Token()
        {
        }
 
        virtual void writeAsHtml(std::ostream&) const = 0;
 
        virtual void
        writeAsOriginal(std::ostream& out) const
        {
            writeAsHtml(out);
        }
 
        virtual void writeToken(std::ostream& out) const = 0;
 
        virtual void
        writeToken(size_t indent, std::ostream& out) const
        {
            out << std::string(indent * 2, ' ');
            writeToken(out);
        }
 
        virtual std::optional<TokenGroup>
        processSpanElements(const LinkIds& idTable)
        {
            return std::nullopt;
        }
 
        virtual std::optional<std::string>
        text() const
        {
            return std::nullopt;
        }
 
        virtual bool
        canContainMarkup() const
        {
            return false;
        }
 
        virtual bool
        isBlankLine() const
        {
            return false;
        }
 
        virtual bool
        isContainer() const
        {
            return false;
        }
 
        virtual bool
        isUnmatchedOpenMarker() const
        {
            return false;
        }
 
        virtual bool
        isUnmatchedCloseMarker() const
        {
            return false;
        }
 
        virtual bool
        isMatchedOpenMarker() const
        {
            return false;
        }
 
        virtual bool
        isMatchedCloseMarker() const
        {
            return false;
        }
 
        virtual bool
        inhibitParagraphs() const
        {
            return false;
        }
 
    protected:
        virtual void
        preWrite(std::ostream& out) const
        {
        }
 
        virtual void
        postWrite(std::ostream& out) const
        {
        }
    };
} // namespace markdown
 
namespace markdown::token
{
 
    size_t isValidTag(const std::string& tag, bool nonBlockFirst = false);
 
    enum EncodingFlags
    {
        cAmps = 0x01,
        cDoubleAmps = 0x02,
        cAngles = 0x04,
        cQuotes = 0x08
    };
 
    class TextHolder : public Token
    {
    public:
        TextHolder(const std::string& text, bool canContainMarkup, unsigned int encodingFlags) :
            mText(text), mCanContainMarkup(canContainMarkup), mEncodingFlags(encodingFlags)
        {
        }
 
        void writeAsHtml(std::ostream& out) const override;
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "TextHolder: " << mText << '\n';
        }
 
        std::optional<std::string>
        text() const override
        {
            return mText;
        }
 
        bool
        canContainMarkup() const override
        {
            return mCanContainMarkup;
        }
 
    private:
        const std::string mText;
        const bool mCanContainMarkup;
        const int mEncodingFlags;
    };
 
    class RawText : public TextHolder
    {
    public:
        RawText(const std::string& text, bool canContainMarkup = true) :
            TextHolder(text, canContainMarkup, cAmps | cAngles | cQuotes)
        {
        }
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "RawText: " << *text() << '\n';
        }
 
        std::optional<TokenGroup> processSpanElements(const LinkIds& idTable) override;
 
    private:
        using ReplacementTable = std::vector<TokenPtr>;
 
        static std::string _processHtmlTagAttributes(std::string src,
                                                     ReplacementTable& replacements);
        static std::string _processCodeSpans(std::string src, ReplacementTable& replacements);
        static std::string _processEscapedCharacters(const std::string& src);
        static std::string _processLinksImagesAndTags(const std::string& src,
                                                      ReplacementTable& replacements,
                                                      const LinkIds& idTable);
        static std::string _processSpaceBracketedGroupings(const std::string& src,
                                                           ReplacementTable& replacements);
        static TokenGroup _processBoldAndItalicSpans(const std::string& src,
                                                     ReplacementTable& replacements);
 
        static TokenGroup _encodeProcessedItems(const std::string& src,
                                                ReplacementTable& replacements);
        static std::string _restoreProcessedItems(const std::string& src,
                                                  ReplacementTable& replacements);
    };
 
    class HtmlTag : public TextHolder
    {
    public:
        HtmlTag(const std::string& contents) : TextHolder(contents, false, cAmps | cAngles)
        {
        }
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "HtmlTag: " << *text() << '\n';
        }
 
    protected:
        void
        preWrite(std::ostream& out) const override
        {
            out << '<';
        }
 
        void
        postWrite(std::ostream& out) const override
        {
            out << '>';
        }
    };
 
    class HtmlAnchorTag : public TextHolder
    {
    public:
        HtmlAnchorTag(const std::string& url, const std::string& title = std::string());
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "HtmlAnchorTag: " << *text() << '\n';
        }
    };
 
    class InlineHtmlContents : public TextHolder
    {
    public:
        InlineHtmlContents(const std::string& contents) :
            TextHolder(contents, false, cAmps | cAngles)
        {
        }
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "InlineHtmlContents: " << *text() << '\n';
        }
    };
 
    class InlineHtmlComment : public TextHolder
    {
    public:
        InlineHtmlComment(const std::string& contents) : TextHolder(contents, false, 0)
        {
        }
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "InlineHtmlComment: " << *text() << '\n';
        }
    };
 
    class CodeBlock : public TextHolder
    {
    public:
        CodeBlock(const std::string& actualContents) :
            TextHolder(actualContents, false, cDoubleAmps | cAngles | cQuotes)
        {
        }
 
        void writeAsHtml(std::ostream& out) const override;
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "CodeBlock: " << *text() << '\n';
        }
    };
 
    class CodeSpan : public TextHolder
    {
    public:
        CodeSpan(const std::string& actualContents) :
            TextHolder(actualContents, false, cDoubleAmps | cAngles | cQuotes)
        {
        }
 
        void writeAsHtml(std::ostream& out) const override;
        void writeAsOriginal(std::ostream& out) const override;
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "CodeSpan: " << *text() << '\n';
        }
    };
 
    class Header : public TextHolder
    {
    public:
        Header(size_t level, const std::string& text) :
            TextHolder(text, true, cAmps | cAngles | cQuotes), mLevel(level)
        {
        }
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "Header " << mLevel << ": " << *text() << '\n';
        }
 
        bool
        inhibitParagraphs() const override
        {
            return true;
        }
 
    protected:
        void
        preWrite(std::ostream& out) const override
        {
            out << "<h" << mLevel << ">";
        }
 
        void
        postWrite(std::ostream& out) const override
        {
            out << "</h" << mLevel << ">\n";
        }
 
    private:
        size_t mLevel;
    };
 
    class BlankLine : public TextHolder
    {
    public:
        BlankLine(const std::string& actualContents = std::string()) :
            TextHolder(actualContents, false, 0)
        {
        }
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "BlankLine: " << *text() << '\n';
        }
 
        bool
        isBlankLine() const override
        {
            return true;
        }
    };
 
    class EscapedCharacter : public Token
    {
    public:
        EscapedCharacter(char c) : mChar(c)
        {
        }
 
        void
        writeAsHtml(std::ostream& out) const override
        {
            out << mChar;
        }
 
        void
        writeAsOriginal(std::ostream& out) const override
        {
            out << '\\' << mChar;
        }
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "EscapedCharacter: " << mChar << '\n';
        }
 
    private:
        const char mChar;
    };
 
    class Container : public Token
    {
    public:
        Container(const TokenGroup& contents = TokenGroup()) :
            mSubTokens(contents), mParagraphMode(false)
        {
        }
 
        const TokenGroup&
        subTokens() const
        {
            return mSubTokens;
        }
 
        void
        appendSubtokens(TokenGroup& tokens)
        {
            mSubTokens.splice(mSubTokens.end(), tokens);
        }
 
        void
        swapSubtokens(TokenGroup& tokens)
        {
            mSubTokens.swap(tokens);
        }
 
        bool
        isContainer() const override
        {
            return true;
        }
 
        void writeAsHtml(std::ostream& out) const override;
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "Container: error!" << '\n';
        }
 
        void writeToken(size_t indent, std::ostream& out) const override;
 
        std::optional<TokenGroup> processSpanElements(const LinkIds& idTable) override;
 
        virtual TokenPtr
        clone(const TokenGroup& newContents) const
        {
            return TokenPtr(new Container(newContents));
        }
 
        virtual std::string
        containerName() const
        {
            return "Container";
        }
 
    protected:
        TokenGroup mSubTokens;
        bool mParagraphMode;
    };
 
    class InlineHtmlBlock : public Container
    {
    public:
        InlineHtmlBlock(const TokenGroup& contents, bool isBlockTag = false) :
            Container(contents), mIsBlockTag(isBlockTag)
        {
        }
 
        InlineHtmlBlock(const std::string& contents) : mIsBlockTag(false)
        {
            mSubTokens.push_back(TokenPtr(new InlineHtmlContents(contents)));
        }
 
        bool
        inhibitParagraphs() const override
        {
            return !mIsBlockTag;
        }
 
        TokenPtr
        clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new InlineHtmlBlock(newContents));
        }
 
        std::string
        containerName() const override
        {
            return "InlineHtmlBlock";
        }
 
        // Inline HTML blocks always end with a blank line, so report it as one for
        // parsing purposes.
        bool
        isBlankLine() const override
        {
            return true;
        }
 
    private:
        bool mIsBlockTag;
    };
 
    class ListItem : public Container
    {
    public:
        ListItem(const TokenGroup& contents) : Container(contents), mInhibitParagraphs(true)
        {
        }
 
        void
        inhibitParagraphs(bool set)
        {
            mInhibitParagraphs = set;
        }
 
        bool
        inhibitParagraphs() const override
        {
            return mInhibitParagraphs;
        }
 
        TokenPtr
        clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new ListItem(newContents));
        }
 
        std::string
        containerName() const override
        {
            return "ListItem";
        }
 
    protected:
        void
        preWrite(std::ostream& out) const override
        {
            out << "<li>";
        }
 
        void
        postWrite(std::ostream& out) const override
        {
            out << "</li>\n";
        }
 
    private:
        bool mInhibitParagraphs;
    };
 
    class UnorderedList : public Container
    {
    public:
        UnorderedList(const TokenGroup& contents, bool paragraphMode = false);
 
        TokenPtr
        clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new UnorderedList(newContents));
        }
 
        std::string
        containerName() const override
        {
            return "UnorderedList";
        }
 
    protected:
        void
        preWrite(std::ostream& out) const override
        {
            out << "\n<ul>\n";
        }
 
        void
        postWrite(std::ostream& out) const override
        {
            out << "</ul>\n\n";
        }
    };
 
    class OrderedList : public UnorderedList
    {
    public:
        OrderedList(const TokenGroup& contents, bool paragraphMode = false) :
            UnorderedList(contents, paragraphMode)
        {
        }
 
        TokenPtr
        clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new OrderedList(newContents));
        }
 
        std::string
        containerName() const override
        {
            return "OrderedList";
        }
 
    protected:
        void
        preWrite(std::ostream& out) const override
        {
            out << "<ol>\n";
        }
 
        void
        postWrite(std::ostream& out) const override
        {
            out << "</ol>\n\n";
        }
    };
 
    class BlockQuote : public Container
    {
    public:
        BlockQuote(const TokenGroup& contents) : Container(contents)
        {
        }
 
        TokenPtr
        clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new BlockQuote(newContents));
        }
 
        std::string
        containerName() const override
        {
            return "BlockQuote";
        }
 
    protected:
        void
        preWrite(std::ostream& out) const override
        {
            out << "<blockquote>\n";
        }
 
        void
        postWrite(std::ostream& out) const override
        {
            out << "\n</blockquote>\n";
        }
    };
 
    class Paragraph : public Container
    {
    public:
        Paragraph()
        {
        }
 
        Paragraph(const TokenGroup& contents) : Container(contents)
        {
        }
 
        TokenPtr
        clone(const TokenGroup& newContents) const override
        {
            return TokenPtr(new Paragraph(newContents));
        }
 
        std::string
        containerName() const override
        {
            return "Paragraph";
        }
 
    protected:
        void
        preWrite(std::ostream& out) const override
        {
            out << "<p>";
        }
 
        void
        postWrite(std::ostream& out) const override
        {
            out << "</p>\n\n";
        }
    };
 
    class BoldOrItalicMarker : public Token
    {
    public:
        BoldOrItalicMarker(bool open, char c, size_t size) :
            mOpenMarker(open),
            mTokenCharacter(c),
            mSize(size),
            mMatch(0),
            mCannotMatch(false),
            mDisabled(false),
            mId(-1)
        {
        }
 
        bool
        isUnmatchedOpenMarker() const override
        {
            return (mOpenMarker && mMatch == 0 && !mCannotMatch);
        }
 
        bool
        isUnmatchedCloseMarker() const override
        {
            return (!mOpenMarker && mMatch == 0 && !mCannotMatch);
        }
 
        bool
        isMatchedOpenMarker() const override
        {
            return (mOpenMarker && mMatch != 0);
        }
 
        bool
        isMatchedCloseMarker() const override
        {
            return (!mOpenMarker && mMatch != 0);
        }
 
        void writeAsHtml(std::ostream& out) const override;
        void writeToken(std::ostream& out) const override;
 
        bool
        isOpenMarker() const
        {
            return mOpenMarker;
        }
 
        char
        tokenCharacter() const
        {
            return mTokenCharacter;
        }
 
        size_t
        size() const
        {
            return mSize;
        }
 
        bool
        matched() const
        {
            return (mMatch != 0);
        }
 
        BoldOrItalicMarker*
        matchedTo() const
        {
            return mMatch;
        }
 
        int
        id() const
        {
            return mId;
        }
 
        void
        matched(BoldOrItalicMarker* match, int id = -1)
        {
            mMatch = match;
            mId = id;
        }
 
        void
        cannotMatch(bool set)
        {
            mCannotMatch = set;
        }
 
        void
        disable()
        {
            mCannotMatch = mDisabled = true;
        }
 
    private:
        bool mOpenMarker; // Otherwise it's a close-marker
        char mTokenCharacter; // Underscore or asterisk
        size_t mSize; // 1=italics, 2=bold, 3=both
        BoldOrItalicMarker* mMatch;
        bool mCannotMatch;
        bool mDisabled;
        int mId;
    };
 
    class Image : public Token
    {
    public:
        Image(const std::string& altText, const std::string& url, const std::string& title) :
            mAltText(altText), mUrl(url), mTitle(title)
        {
        }
 
        void writeAsHtml(std::ostream& out) const override;
 
        void
        writeToken(std::ostream& out) const override
        {
            out << "Image: " << mUrl << '\n';
        }
 
    private:
        const std::string mAltText, mUrl, mTitle;
    };
 
} // namespace markdown::token