Some repairs for md_parser and template engine.

apps/website-generator/WebsiteGenerator.cpp

@@ -16,6 +16,8 @@
 
 #include "FileLogger.h"
 
+#include <iostream>
+
 WebsiteGenerator::WebsiteGenerator()
     : mConfig(std::make_unique<SiteGeneratorConfig>()),
       mTemplateEngine()

src/compiler/template_engine/TemplateElements.cpp

@@ -35,8 +35,9 @@ void TemplateBlock::addLine(const std::string& line)
 
 std::string TemplateBlock::getRawContent() const
 {
-    std::string content = "TemplateBlock: " + mName + "\n";
+    std::string content = "TemplateBlock: " + mName + " - Start \n";
     content += TemplateNode::getRawContent();
+    content += "TemplateBlock: " + mName + " - End";
     return content;
 }
 
@@ -53,10 +54,8 @@ std::string TemplateBlock::renderAsParent(TemplateSubstitutionContext* substitut
         if (child->getType() == Type::EXPRESSION)
         {
             auto expression = dynamic_cast<TemplateExpression*>(child.get());
-            std::cout << "Got expression with content " << expression->getContent() << std::endl;
             if (expression->getContent() == "super()")
             {
-                std::cout << "Adding expression base" << std::endl;
                 content += base->render(substitutions, nullptr);
             }
         }
@@ -116,7 +115,7 @@ const std::string& TemplateExpression::getContent() const
 
 std::string TemplateExpression::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
 {
-    if (substitutions->hasSubstitution(mContent))
+    if (substitutions && substitutions->hasSubstitution(mContent))
     {
         return substitutions->getSubstitution(mContent);
     }
@@ -147,9 +146,13 @@ bool TemplateTextBody::hasContent() const
 std::string TemplateTextBody::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
 {
     std::string content;
-    for (const auto& line : mContent)
+    for(unsigned idx=0; idx<mContent.size(); idx++)
     {
-        content += line + '\n';
+        content += mContent[idx];
+        if(idx != mContent.size() - 1)
+        {
+            content += '\n';
+        }
     }
     return content;
 }
@@ -157,9 +160,13 @@ std::string TemplateTextBody::render(TemplateSubstitutionContext* substitutions,
 std::string TemplateTextBody::getRawContent() const
 {
     std::string content;
-    for (const auto& line : mContent)
+    for(unsigned idx=0; idx<mContent.size(); idx++)
     {
-        content += "TemplateBody: " + line + "\n";
+        content += "Template Body L-" + std::to_string(idx) + ": " + mContent[idx];
+        if(idx != mContent.size() - 1)
+        {
+            content += '\n';
+        }
     }
     return content;
 }

src/compiler/template_engine/TemplateFile.cpp

@@ -1,9 +1,11 @@
 #include "TemplateFile.h"
 
-#include "StringUtils.h"
 #include "TemplateElements.h"
+#include "TemplateNode.h"
 
-#include <iostream>
+#include "Lexer.h"
+#include "File.h"
+#include "StringUtils.h"
 
 TemplateFile::TemplateFile(const Path& path)
     : mPath(path),
@@ -38,114 +40,119 @@ void TemplateFile::loadContent()
     {
         return;
     }
-
     mRawContent = File(mPath).readLines();
-    mWorkingLine = 0;
+
     mWorkingNode = mRootNode.get();
-    mWorkingTextBody = std::make_unique<TemplateTextBody>(mWorkingNode);
+    mWorkingTextSpan = std::make_unique<TemplateTextBody>(mWorkingNode);
     for (const auto& line : mRawContent)
     {
         processLine(line);
-        mWorkingLine++;
     }
-    onTextBodyFinished();
+    onTextSpanFinished();
 
     mHasLoaded = true;
 }
 
-void TemplateFile::onTextBodyFinished(std::string working_string)
+std::string TemplateFile::dumpContent()
 {
-    if (!working_string.empty())
+    return mRootNode->getRawContent();
-    {
-        mWorkingTextBody->addLine(working_string);
 }
 
-    if (mWorkingTextBody->hasContent())
+void TemplateFile::onTextSpanFinished()
 {
-        mWorkingNode->addChild(std::move(mWorkingTextBody));
+    if (!mWorkingLineContent.empty())
-        mWorkingTextBody = std::make_unique<TemplateTextBody>(mWorkingNode);
+    {
+        mWorkingTextSpan->addLine(mWorkingLineContent);
     }
+
+    if (mWorkingTextSpan->hasContent())
+    {
+        mWorkingNode->addChild(std::move(mWorkingTextSpan));
+        mWorkingTextSpan = std::make_unique<TemplateTextBody>(mWorkingNode);
+    }
+    mWorkingLineContent.clear();
+}
+
+unsigned TemplateFile::checkForStatement(const std::string& lineSection)
+{
+    if (lineSection.empty())
+    {
+        return 0;
+    }
+
+    std::vector<std::string> hits;
+    unsigned hit_size{0};
+    if (Lexer::matchPattern("{%@%}", lineSection, '@', hits))
+    {
+        if (hits.size() == 1)
+        {
+            auto content = hits[0];
+
+            onTextSpanFinished();
+
+            onFoundStatement(content);
+            hit_size = 4 + content.size();
+        }
+    }
+    return hit_size;
+}
+
+unsigned TemplateFile::checkForExpression(const std::string& lineSection)
+{
+    if (lineSection.empty())
+    {
+        return 0;
+    }
+
+    std::vector<std::string> hits;
+    unsigned hit_size{0};
+    if (Lexer::matchPattern("{{@}}", lineSection, '@', hits))
+    {
+        if (hits.size() == 1)
+        {
+            auto content = hits[0];
+
+            onTextSpanFinished();
+
+            onFoundExpression(content);
+            hit_size = 4 + content.size();
+        }
+    }
+    return hit_size;
 }
 
 void TemplateFile::processLine(const std::string& line)
 {
-    bool last_was_ldelimiter{ false };
+    unsigned line_position = 0;
-    bool last_was_statement_rdelimiter{ false };
+    mWorkingLineContent.clear();
-    bool last_was_expression_rdelimiter{ false };
+    while(line_position < line.size())
-    bool in_statement{ false };
-    bool in_expression{ false };
-    std::string working_string;
-    std::string last_working_string;
-    for (auto c : line)
     {
-        if (c == '{')
+        const auto remaining = line.substr(line_position, line.size() - line_position);
+        if(auto length = checkForStatement(remaining))
         {
-            if (last_was_ldelimiter)
+            line_position += length;
+        }
+        else if(auto length = checkForExpression(remaining))
         {
-                in_expression = true;
+            line_position += length;
-                last_was_ldelimiter = false;
-                working_string = "";
         }
         else
         {
-                last_was_ldelimiter = true;
+            mWorkingLineContent += line[line_position];
-            }
+            line_position++;
-        }
-        else if (c == '%' && last_was_ldelimiter)
-        {
-            last_was_ldelimiter = false;
-            in_statement = true;
-            last_working_string = working_string;
-            working_string = "";
-        }
-        else if (c == '%' && in_statement)
-        {
-            last_was_statement_rdelimiter = true;
-        }
-        else if (c == '}' && (last_was_statement_rdelimiter || in_expression || last_was_expression_rdelimiter))
-        {
-            if (last_was_statement_rdelimiter)
-            {
-                onTextBodyFinished(last_working_string);
-                onFoundStatement(working_string);
-                last_was_statement_rdelimiter = false;
-                working_string = "";
-            }
-            else if (in_expression)
-            {
-                last_was_expression_rdelimiter = true;
-                in_expression = false;
-            }
-            else
-            {
-                onTextBodyFinished();
-                onFoundExpression(working_string);
-                last_was_expression_rdelimiter = false;
-                working_string = "";
-            }
-        }
-        else if (last_was_ldelimiter && (!in_statement && !in_expression))
-        {
-            last_was_ldelimiter = false;
-            working_string += '{';
-            working_string += c;
-        }
-        else
-        {
-            working_string += c;
         }
     }
 
-    if (!working_string.empty())
+    if (!mWorkingLineContent.empty())
     {
-        mWorkingTextBody->addLine(working_string);
+        mWorkingTextSpan->addLine(mWorkingLineContent);
+        mWorkingLineContent.clear();
     }
 }
 
 void TemplateFile::onFoundStatement(const std::string& statement_string)
 {
     const auto statement_elements = StringUtils::split(statement_string);
-
     if (statement_elements.size() == 0)
     {
         return;
@@ -163,7 +170,13 @@ void TemplateFile::onFoundStatement(const std::string& statement_string)
     {
         onFoundExtends(statement_elements);
     }
+}
 
+void TemplateFile::onFoundExpression(const std::string& expression_string)
+{
+    const auto stripped = StringUtils::stripSurroundingWhitepsace(expression_string);
+    auto expression = std::make_unique<TemplateExpression>(mWorkingNode, stripped);
+    mWorkingNode->addChild(std::move(expression));
 }
 
 void TemplateFile::onFoundBlock(const std::vector<std::string> args)
@@ -192,15 +205,3 @@ void TemplateFile::onFoundExtends(const std::vector<std::string> args)
     auto extends = std::make_unique<TemplateExtends>(mWorkingNode, args[1]);
     mWorkingNode->addChild(std::move(extends));
 }
-
-void TemplateFile::onFoundExpression(const std::string& expression_string)
-{
-    const auto stripped = StringUtils::stripSurroundingWhitepsace(expression_string);
-    auto expression = std::make_unique<TemplateExpression>(mWorkingNode, stripped);
-    mWorkingNode->addChild(std::move(expression));
-}
-
-std::string TemplateFile::dumpContent()
-{
-    return mRootNode->getRawContent();
-}

src/compiler/template_engine/TemplateFile.h

@@ -1,14 +1,14 @@
 #pragma once
 
-#include "File.h"
-
-#include "TemplateNode.h"
-
 #include <vector>
+#include <string>
+#include <filesystem>
 
 class TemplateNode;
 class TemplateTextBody;
 
+using Path = std::filesystem::path;
+
 class TemplateFile
 {
 public:
@@ -27,27 +27,32 @@ public:
     void loadContent();
 
 private:
-    void onTextBodyFinished(std::string working_string = {});
+    unsigned checkForStatement(const std::string& lineSection);
 
-    void processLine(const std::string& line);
+    unsigned checkForExpression(const std::string& lineSection);
+
+    void onTextSpanFinished();
 
     void onFoundStatement(const std::string& statement_string);
 
+    void onFoundExpression(const std::string& expression_string);
+
     void onFoundBlock(const std::vector<std::string> args);
 
     void onFoundEndBlock(const std::vector<std::string> args);
 
     void onFoundExtends(const std::vector<std::string> args);
 
-    void onFoundExpression(const std::string& expression_string);
+    void processLine(const std::string& line);
 
     Path mPath;
     std::string mParentName;
     std::vector<std::string> mRawContent;
     bool mHasLoaded{false};
 
-    unsigned mWorkingLine{ 0 };
     std::unique_ptr<TemplateNode> mRootNode;
     TemplateNode* mWorkingNode{ nullptr };
-    std::unique_ptr<TemplateTextBody> mWorkingTextBody;
+
+    std::string mWorkingLineContent;
+    std::unique_ptr<TemplateTextBody> mWorkingTextSpan;
 };

src/compiler/template_engine/TemplateNode.cpp

@@ -75,8 +75,18 @@ void TemplateNode::setExtensionParent(TemplateNode* parent)
     mExtensionParent = parent;
 }
 
+void TemplateNode::setExtensionBase(TemplateNode* base)
+{
+    mExtensionBase = base;
+}
+
 std::string TemplateNode::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
 {
+    if (mExtensionBase)
+    {
+        return mExtensionBase->render(substitutions, this);
+    }
+
     if (!parentContext && mExtensionParent)
     {
         parentContext = mExtensionParent;

src/compiler/template_engine/TemplateNode.h

@@ -42,10 +42,13 @@ public:
 
     void setExtensionParent(TemplateNode* parent);
 
+    void setExtensionBase(TemplateNode* base);
+
 protected:
     std::vector<std::unique_ptr<TemplateNode> > mChildren;
     TemplateNode* mParent{ nullptr };
     TemplateNode* mExtensionParent{ nullptr };
+    TemplateNode* mExtensionBase{ nullptr };
 };
 
 using TemplateNodePtr = std::unique_ptr<TemplateNode>;

src/compiler/template_engine/TemplatingEngine.cpp

@@ -26,18 +26,16 @@ std::string TemplatingEngine::renderTemplate(const std::string& name, TemplateSu
         if (!file->hasLoaded())
         {
             file->loadContent();
-            std::cout << file->dumpContent();
+            //std::cout << file->dumpContent();
 
             processTemplate(file, nullptr);
         }
-
         return file->getContent()->render(substitutionContext, nullptr);
     }
     else
     {
         return {};
     }
-
 }
 
 void TemplatingEngine::loadTemplateFiles()
@@ -62,21 +60,20 @@ TemplateFile* TemplatingEngine::getTemplateFile(const std::string& name)
 
 void TemplatingEngine::processTemplate(TemplateFile* file, TemplateNode* parent)
 {
-    std::cout << "Processing file " << file->getName() << std::endl;
-
     auto content = file->getContent();
     if (parent)
     {
-        std::cout << "Setting extension parent" << std::endl;
         content->setExtensionParent(parent);
+        parent->setExtensionBase(content);
     }
 
     if (auto extension_node = dynamic_cast<TemplateExtends*>(content->getFirstChildShallow(TemplateNode::Type::EXTENDS)))
     {
-        std::cout << "Found extension node " << std::endl;
         if (auto extension_template = getTemplateFile(Path(extension_node->getPath())))
         {
-            std::cout << "Found extension template " << std::endl;
+            extension_template->loadContent();
+            //std::cout << extension_template->dumpContent();
+
             processTemplate(extension_template, content);
         }
     }

src/web/html/HtmlTextRun.h

@@ -17,7 +17,7 @@ public:
 
     std::string toString(unsigned depth = 0, bool keepInline = false) const override
     {
-        const auto prefix = std::string(2*depth, ' ');
+
-        return prefix + getText();
+        return getText();
     }
 };

src/web/markdown/MarkdownDocument.cpp

@@ -34,9 +34,18 @@ std::vector<MarkdownLink*> MarkdownDocument::getAllLinks() const
     {
         if (element->getType() == MarkdownElement::Type::PARAGRAPH)
         {
-            auto para_links = dynamic_cast<MarkdownParagraph*>(element.get())->getAllLinks();
+            auto para_links = dynamic_cast<MarkdownElementWithChildren*>(element.get())->getAllLinks();
             links.insert(links.end(), para_links.begin(), para_links.end());
         }
+        else if (element->getType() == MarkdownElement::Type::BULLET_LIST)
+        {
+            auto bullet_list = dynamic_cast<MarkdownBulletList*>(element.get());
+            for(unsigned idx=0; idx<bullet_list->getNumChildren(); idx++)
+            {
+                auto para_links = bullet_list->getChild(idx)->getAllLinks();
+                links.insert(links.end(), para_links.begin(), para_links.end());
+            }
+        }
     }
     return links;
 }

src/web/markdown/MarkdownParser.cpp

@@ -7,7 +7,6 @@
 #include "StringUtils.h"
 
 #include <sstream>
-#include <iostream>
 
 static constexpr char MULTILINE_QUOTE_DELIMITER[]{"```"};
 static constexpr char HEADING_DELIMITER{'#'};
@@ -151,12 +150,10 @@ void MarkdownParser::onTextSpanFinished()
     {
         if (mWorkingTextSpan)
         {
-            std::cout << "Adding to existing text span: " << std::endl;
             mWorkingTextSpan->appendTextContent(mWorkingLine);
         }
         else
         {
-            std::cout << "Adding new text span: " << mWorkingLine << std::endl;
             auto text_span = std::make_unique<MarkdownTextSpan>();
             text_span->addLine(mWorkingLine);
             mWorkingTextSpan = text_span.get();
@@ -183,7 +180,6 @@ void MarkdownParser::processLine(const std::string& line)
 
     if (!mWorkingElement)
     {
-        std::cout << "Adding new paragraph " << std::endl;
         auto paragraph = std::make_unique<MarkdownParagraph>();
         mWorkingElement = paragraph.get();
         mMarkdownDocument->addElement(std::move(paragraph));
@@ -341,7 +337,6 @@ void MarkdownParser::onFoundBulletItem(const std::string& line)
         }
         else
         {
-            std::cout << "Starting new bullet list" << std::endl;
             auto bullet_list = std::make_unique<MarkdownBulletList>();
             mWorkingBulletList = bullet_list.get();
 
@@ -350,15 +345,13 @@ void MarkdownParser::onFoundBulletItem(const std::string& line)
             auto bullet_item = std::make_unique<MarkdownBulletItem>();
             mWorkingElement = bullet_item.get();
             mWorkingBulletList->addChild(std::move(bullet_item));
-
-            processLine(StringUtils::removeUpTo(line, "*"));
         }
+        processLine(StringUtils::removeUpTo(line, "*"));
     }
 }
 
 void MarkdownParser::onSectionFinished()
 {
-    std::cout << "Section is finished" << std::endl;
     mWorkingElement = nullptr;
     mWorkingBulletList = nullptr;
     mWorkingTextSpan = nullptr;
@@ -373,36 +366,29 @@ std::unique_ptr<MarkdownDocument> MarkdownParser::run(const std::string& content
 
     while (std::getline(ss, line, '\n'))
     {
-        std::cout << "Processing line " << line << std::endl;
         if (StringUtils::isWhitespaceOnly(line))
         {
-            std::cout << "Is whitespace only " << std::endl;
             onEmptyLine();
             continue;
         }
         else if (startsWithMultiLineQuote(line))
         {
-            std::cout << "Found multiline quote" << std::endl;
             onFoundMultiLineQuote(line);
         }
         else if (auto result = startsWithCustomMultilineBlock(line); result >= 0)
         {
-            std::cout << "Found custom multiline" << std::endl;
             onFoundCustomMultiLineBlock(line, result);
         }
         else if (startsWithHeading(line))
         {
-            std::cout << "Found heading" << std::endl;
             onFoundHeading(line);
         }
         else if(startsWithBulletItem(line))
         {
-            std::cout << "Found bulletitem" << std::endl;
             onFoundBulletItem(line);
         }
         else
         {
-            std::cout << "Found nothing - process line" << std::endl;
             processLine(line);
         }
     }

src/web/markdown/MarkdownParser.h

@@ -28,11 +28,6 @@ private:
 
     bool isInMultilineBlock() const;
 
-    bool startsWithMultiLineQuote(const std::string& line) const;
-    int startsWithCustomMultilineBlock(const std::string& line) const;
-    bool startsWithHeading(const std::string& line) const;
-    bool startsWithBulletItem(const std::string& line) const;
-
     void onFoundMultiLineQuote(const std::string& line);
     void onFoundCustomMultiLineBlock(const std::string& line, unsigned blockSlot);
     void onFoundHeading(const std::string& line);
@@ -44,6 +39,11 @@ private:
 
     void processLine(const std::string& line);
 
+    bool startsWithMultiLineQuote(const std::string& line) const;
+    int startsWithCustomMultilineBlock(const std::string& line) const;
+    bool startsWithHeading(const std::string& line) const;
+    bool startsWithBulletItem(const std::string& line) const;
+
     unsigned mCustomDelimiterIndex{0};
     std::vector<std::string> mCustomMultilineDelimiters;
     std::vector<std::string> mCustomInlineDelimiters;

test/compiler/TestTemplatingEngine.cpp

@@ -1,15 +1,29 @@
 #include "TemplatingEngine.h"
+#include "TemplateSubstitutionContext.h"
 
 #include "File.h"
 
 #include "TestFramework.h"
 #include "TestUtils.h"
 
-TEST_CASE(TestTemplatingEngine, "compiler")
+TEST_CASE(TestTemplatingEngine_BlockInherit, "compiler")
 {
     auto engine = TemplatingEngine(TestUtils::getTestDataDir());
     const auto content = engine.renderTemplate("index", nullptr);
 
-    File outfile(TestUtils::getTestOutputDir() / "index.html");
+    File outfile(TestUtils::getTestOutputDir(__FILE__) / "BlockInherit.html");
+    outfile.writeText(content);
+}
+
+TEST_CASE(TestTemplatingEngine_Simple, "compiler")
+{
+    auto engine = TemplatingEngine(TestUtils::getTestDataDir());
+
+    TemplateSubstitutionContext sub_context;
+    sub_context.addSubstitution("content", "<div><p>test</p></div>");
+
+    const auto content = engine.renderTemplate("simple_template", &sub_context);
+
+    File outfile(TestUtils::getTestOutputDir(__FILE__) / "Simple.html");
     outfile.writeText(content);
 }

test/data/simple_markdown.md

@@ -1,30 +1,7 @@
-# I'm a level one header
-I'm some text under level one
-
-## I'm a level two header
-I'm some text under level two
-
-```
-I'm a code block
-```
-
-I'm a line under the code block, with some `inline code`.
-
-### I'm a level three header
 I'm a bullet point list:
 
 * First point
 * Second point
 * Third point
 
-With a [hyperlink](www.imahyperlink.com) embedded.
+With some test after.
-
-# I'm another level one header
-
-I'm some inline math $a = b + c$ and I'm some standalone math:
-
-$$
-d = e + f
-$$
-
-![This is an image](https://myoctocat.com/assets/images/base-octocat.svg)

test/data/simple_template.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <link rel="stylesheet" href="style.css" />
+    <title>My title</title>
+</head>
+<body>
+    <div id="content">{{ content }}</div>
+</body>
+</html>