Some repairs for md_parser and template engine.

This commit is contained in:
James Grogan 2022-12-07 10:21:28 +00:00
parent 8705859115
commit 22157169c0
14 changed files with 180 additions and 159 deletions

View file

@ -16,6 +16,8 @@
#include "FileLogger.h" #include "FileLogger.h"
#include <iostream>
WebsiteGenerator::WebsiteGenerator() WebsiteGenerator::WebsiteGenerator()
: mConfig(std::make_unique<SiteGeneratorConfig>()), : mConfig(std::make_unique<SiteGeneratorConfig>()),
mTemplateEngine() mTemplateEngine()

View file

@ -35,8 +35,9 @@ void TemplateBlock::addLine(const std::string& line)
std::string TemplateBlock::getRawContent() const std::string TemplateBlock::getRawContent() const
{ {
std::string content = "TemplateBlock: " + mName + "\n"; std::string content = "TemplateBlock: " + mName + " - Start \n";
content += TemplateNode::getRawContent(); content += TemplateNode::getRawContent();
content += "TemplateBlock: " + mName + " - End";
return content; return content;
} }
@ -53,10 +54,8 @@ std::string TemplateBlock::renderAsParent(TemplateSubstitutionContext* substitut
if (child->getType() == Type::EXPRESSION) if (child->getType() == Type::EXPRESSION)
{ {
auto expression = dynamic_cast<TemplateExpression*>(child.get()); auto expression = dynamic_cast<TemplateExpression*>(child.get());
std::cout << "Got expression with content " << expression->getContent() << std::endl;
if (expression->getContent() == "super()") if (expression->getContent() == "super()")
{ {
std::cout << "Adding expression base" << std::endl;
content += base->render(substitutions, nullptr); content += base->render(substitutions, nullptr);
} }
} }
@ -116,7 +115,7 @@ const std::string& TemplateExpression::getContent() const
std::string TemplateExpression::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext) std::string TemplateExpression::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
{ {
if (substitutions->hasSubstitution(mContent)) if (substitutions && substitutions->hasSubstitution(mContent))
{ {
return substitutions->getSubstitution(mContent); return substitutions->getSubstitution(mContent);
} }
@ -147,9 +146,13 @@ bool TemplateTextBody::hasContent() const
std::string TemplateTextBody::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext) std::string TemplateTextBody::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
{ {
std::string content; std::string content;
for (const auto& line : mContent) for(unsigned idx=0; idx<mContent.size(); idx++)
{ {
content += line + '\n'; content += mContent[idx];
if(idx != mContent.size() - 1)
{
content += '\n';
}
} }
return content; return content;
} }
@ -157,9 +160,13 @@ std::string TemplateTextBody::render(TemplateSubstitutionContext* substitutions,
std::string TemplateTextBody::getRawContent() const std::string TemplateTextBody::getRawContent() const
{ {
std::string content; std::string content;
for (const auto& line : mContent) for(unsigned idx=0; idx<mContent.size(); idx++)
{ {
content += "TemplateBody: " + line + "\n"; content += "Template Body L-" + std::to_string(idx) + ": " + mContent[idx];
if(idx != mContent.size() - 1)
{
content += '\n';
}
} }
return content; return content;
} }

View file

@ -1,9 +1,11 @@
#include "TemplateFile.h" #include "TemplateFile.h"
#include "StringUtils.h"
#include "TemplateElements.h" #include "TemplateElements.h"
#include "TemplateNode.h"
#include <iostream> #include "Lexer.h"
#include "File.h"
#include "StringUtils.h"
TemplateFile::TemplateFile(const Path& path) TemplateFile::TemplateFile(const Path& path)
: mPath(path), : mPath(path),
@ -38,114 +40,119 @@ void TemplateFile::loadContent()
{ {
return; return;
} }
mRawContent = File(mPath).readLines(); mRawContent = File(mPath).readLines();
mWorkingLine = 0;
mWorkingNode = mRootNode.get(); mWorkingNode = mRootNode.get();
mWorkingTextBody = std::make_unique<TemplateTextBody>(mWorkingNode); mWorkingTextSpan = std::make_unique<TemplateTextBody>(mWorkingNode);
for (const auto& line : mRawContent) for (const auto& line : mRawContent)
{ {
processLine(line); processLine(line);
mWorkingLine++;
} }
onTextBodyFinished(); onTextSpanFinished();
mHasLoaded = true; mHasLoaded = true;
} }
void TemplateFile::onTextBodyFinished(std::string working_string) std::string TemplateFile::dumpContent()
{ {
if (!working_string.empty()) return mRootNode->getRawContent();
}
void TemplateFile::onTextSpanFinished()
{
if (!mWorkingLineContent.empty())
{ {
mWorkingTextBody->addLine(working_string); mWorkingTextSpan->addLine(mWorkingLineContent);
} }
if (mWorkingTextBody->hasContent()) if (mWorkingTextSpan->hasContent())
{ {
mWorkingNode->addChild(std::move(mWorkingTextBody)); mWorkingNode->addChild(std::move(mWorkingTextSpan));
mWorkingTextBody = std::make_unique<TemplateTextBody>(mWorkingNode); mWorkingTextSpan = std::make_unique<TemplateTextBody>(mWorkingNode);
} }
mWorkingLineContent.clear();
}
unsigned TemplateFile::checkForStatement(const std::string& lineSection)
{
if (lineSection.empty())
{
return 0;
}
std::vector<std::string> hits;
unsigned hit_size{0};
if (Lexer::matchPattern("{%@%}", lineSection, '@', hits))
{
if (hits.size() == 1)
{
auto content = hits[0];
onTextSpanFinished();
onFoundStatement(content);
hit_size = 4 + content.size();
}
}
return hit_size;
}
unsigned TemplateFile::checkForExpression(const std::string& lineSection)
{
if (lineSection.empty())
{
return 0;
}
std::vector<std::string> hits;
unsigned hit_size{0};
if (Lexer::matchPattern("{{@}}", lineSection, '@', hits))
{
if (hits.size() == 1)
{
auto content = hits[0];
onTextSpanFinished();
onFoundExpression(content);
hit_size = 4 + content.size();
}
}
return hit_size;
} }
void TemplateFile::processLine(const std::string& line) void TemplateFile::processLine(const std::string& line)
{ {
bool last_was_ldelimiter{ false }; unsigned line_position = 0;
bool last_was_statement_rdelimiter{ false }; mWorkingLineContent.clear();
bool last_was_expression_rdelimiter{ false }; while(line_position < line.size())
bool in_statement{ false };
bool in_expression{ false };
std::string working_string;
std::string last_working_string;
for (auto c : line)
{ {
if (c == '{') const auto remaining = line.substr(line_position, line.size() - line_position);
if(auto length = checkForStatement(remaining))
{ {
if (last_was_ldelimiter) line_position += length;
}
else if(auto length = checkForExpression(remaining))
{ {
in_expression = true; line_position += length;
last_was_ldelimiter = false;
working_string = "";
} }
else else
{ {
last_was_ldelimiter = true; mWorkingLineContent += line[line_position];
} line_position++;
}
else if (c == '%' && last_was_ldelimiter)
{
last_was_ldelimiter = false;
in_statement = true;
last_working_string = working_string;
working_string = "";
}
else if (c == '%' && in_statement)
{
last_was_statement_rdelimiter = true;
}
else if (c == '}' && (last_was_statement_rdelimiter || in_expression || last_was_expression_rdelimiter))
{
if (last_was_statement_rdelimiter)
{
onTextBodyFinished(last_working_string);
onFoundStatement(working_string);
last_was_statement_rdelimiter = false;
working_string = "";
}
else if (in_expression)
{
last_was_expression_rdelimiter = true;
in_expression = false;
}
else
{
onTextBodyFinished();
onFoundExpression(working_string);
last_was_expression_rdelimiter = false;
working_string = "";
}
}
else if (last_was_ldelimiter && (!in_statement && !in_expression))
{
last_was_ldelimiter = false;
working_string += '{';
working_string += c;
}
else
{
working_string += c;
} }
} }
if (!working_string.empty()) if (!mWorkingLineContent.empty())
{ {
mWorkingTextBody->addLine(working_string); mWorkingTextSpan->addLine(mWorkingLineContent);
mWorkingLineContent.clear();
} }
} }
void TemplateFile::onFoundStatement(const std::string& statement_string) void TemplateFile::onFoundStatement(const std::string& statement_string)
{ {
const auto statement_elements = StringUtils::split(statement_string); const auto statement_elements = StringUtils::split(statement_string);
if (statement_elements.size() == 0) if (statement_elements.size() == 0)
{ {
return; return;
@ -163,7 +170,13 @@ void TemplateFile::onFoundStatement(const std::string& statement_string)
{ {
onFoundExtends(statement_elements); onFoundExtends(statement_elements);
} }
}
void TemplateFile::onFoundExpression(const std::string& expression_string)
{
const auto stripped = StringUtils::stripSurroundingWhitepsace(expression_string);
auto expression = std::make_unique<TemplateExpression>(mWorkingNode, stripped);
mWorkingNode->addChild(std::move(expression));
} }
void TemplateFile::onFoundBlock(const std::vector<std::string> args) void TemplateFile::onFoundBlock(const std::vector<std::string> args)
@ -192,15 +205,3 @@ void TemplateFile::onFoundExtends(const std::vector<std::string> args)
auto extends = std::make_unique<TemplateExtends>(mWorkingNode, args[1]); auto extends = std::make_unique<TemplateExtends>(mWorkingNode, args[1]);
mWorkingNode->addChild(std::move(extends)); mWorkingNode->addChild(std::move(extends));
} }
void TemplateFile::onFoundExpression(const std::string& expression_string)
{
const auto stripped = StringUtils::stripSurroundingWhitepsace(expression_string);
auto expression = std::make_unique<TemplateExpression>(mWorkingNode, stripped);
mWorkingNode->addChild(std::move(expression));
}
std::string TemplateFile::dumpContent()
{
return mRootNode->getRawContent();
}

View file

@ -1,14 +1,14 @@
#pragma once #pragma once
#include "File.h"
#include "TemplateNode.h"
#include <vector> #include <vector>
#include <string>
#include <filesystem>
class TemplateNode; class TemplateNode;
class TemplateTextBody; class TemplateTextBody;
using Path = std::filesystem::path;
class TemplateFile class TemplateFile
{ {
public: public:
@ -27,27 +27,32 @@ public:
void loadContent(); void loadContent();
private: private:
void onTextBodyFinished(std::string working_string = {}); unsigned checkForStatement(const std::string& lineSection);
void processLine(const std::string& line); unsigned checkForExpression(const std::string& lineSection);
void onTextSpanFinished();
void onFoundStatement(const std::string& statement_string); void onFoundStatement(const std::string& statement_string);
void onFoundExpression(const std::string& expression_string);
void onFoundBlock(const std::vector<std::string> args); void onFoundBlock(const std::vector<std::string> args);
void onFoundEndBlock(const std::vector<std::string> args); void onFoundEndBlock(const std::vector<std::string> args);
void onFoundExtends(const std::vector<std::string> args); void onFoundExtends(const std::vector<std::string> args);
void onFoundExpression(const std::string& expression_string); void processLine(const std::string& line);
Path mPath; Path mPath;
std::string mParentName; std::string mParentName;
std::vector<std::string> mRawContent; std::vector<std::string> mRawContent;
bool mHasLoaded{false}; bool mHasLoaded{false};
unsigned mWorkingLine{ 0 };
std::unique_ptr<TemplateNode> mRootNode; std::unique_ptr<TemplateNode> mRootNode;
TemplateNode* mWorkingNode{ nullptr }; TemplateNode* mWorkingNode{ nullptr };
std::unique_ptr<TemplateTextBody> mWorkingTextBody;
std::string mWorkingLineContent;
std::unique_ptr<TemplateTextBody> mWorkingTextSpan;
}; };

View file

@ -75,8 +75,18 @@ void TemplateNode::setExtensionParent(TemplateNode* parent)
mExtensionParent = parent; mExtensionParent = parent;
} }
void TemplateNode::setExtensionBase(TemplateNode* base)
{
mExtensionBase = base;
}
std::string TemplateNode::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext) std::string TemplateNode::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
{ {
if (mExtensionBase)
{
return mExtensionBase->render(substitutions, this);
}
if (!parentContext && mExtensionParent) if (!parentContext && mExtensionParent)
{ {
parentContext = mExtensionParent; parentContext = mExtensionParent;

View file

@ -42,10 +42,13 @@ public:
void setExtensionParent(TemplateNode* parent); void setExtensionParent(TemplateNode* parent);
void setExtensionBase(TemplateNode* base);
protected: protected:
std::vector<std::unique_ptr<TemplateNode> > mChildren; std::vector<std::unique_ptr<TemplateNode> > mChildren;
TemplateNode* mParent{ nullptr }; TemplateNode* mParent{ nullptr };
TemplateNode* mExtensionParent{ nullptr }; TemplateNode* mExtensionParent{ nullptr };
TemplateNode* mExtensionBase{ nullptr };
}; };
using TemplateNodePtr = std::unique_ptr<TemplateNode>; using TemplateNodePtr = std::unique_ptr<TemplateNode>;

View file

@ -26,18 +26,16 @@ std::string TemplatingEngine::renderTemplate(const std::string& name, TemplateSu
if (!file->hasLoaded()) if (!file->hasLoaded())
{ {
file->loadContent(); file->loadContent();
std::cout << file->dumpContent(); //std::cout << file->dumpContent();
processTemplate(file, nullptr); processTemplate(file, nullptr);
} }
return file->getContent()->render(substitutionContext, nullptr); return file->getContent()->render(substitutionContext, nullptr);
} }
else else
{ {
return {}; return {};
} }
} }
void TemplatingEngine::loadTemplateFiles() void TemplatingEngine::loadTemplateFiles()
@ -62,21 +60,20 @@ TemplateFile* TemplatingEngine::getTemplateFile(const std::string& name)
void TemplatingEngine::processTemplate(TemplateFile* file, TemplateNode* parent) void TemplatingEngine::processTemplate(TemplateFile* file, TemplateNode* parent)
{ {
std::cout << "Processing file " << file->getName() << std::endl;
auto content = file->getContent(); auto content = file->getContent();
if (parent) if (parent)
{ {
std::cout << "Setting extension parent" << std::endl;
content->setExtensionParent(parent); content->setExtensionParent(parent);
parent->setExtensionBase(content);
} }
if (auto extension_node = dynamic_cast<TemplateExtends*>(content->getFirstChildShallow(TemplateNode::Type::EXTENDS))) if (auto extension_node = dynamic_cast<TemplateExtends*>(content->getFirstChildShallow(TemplateNode::Type::EXTENDS)))
{ {
std::cout << "Found extension node " << std::endl;
if (auto extension_template = getTemplateFile(Path(extension_node->getPath()))) if (auto extension_template = getTemplateFile(Path(extension_node->getPath())))
{ {
std::cout << "Found extension template " << std::endl; extension_template->loadContent();
//std::cout << extension_template->dumpContent();
processTemplate(extension_template, content); processTemplate(extension_template, content);
} }
} }

View file

@ -17,7 +17,7 @@ public:
std::string toString(unsigned depth = 0, bool keepInline = false) const override std::string toString(unsigned depth = 0, bool keepInline = false) const override
{ {
const auto prefix = std::string(2*depth, ' ');
return prefix + getText(); return getText();
} }
}; };

View file

@ -34,9 +34,18 @@ std::vector<MarkdownLink*> MarkdownDocument::getAllLinks() const
{ {
if (element->getType() == MarkdownElement::Type::PARAGRAPH) if (element->getType() == MarkdownElement::Type::PARAGRAPH)
{ {
auto para_links = dynamic_cast<MarkdownParagraph*>(element.get())->getAllLinks(); auto para_links = dynamic_cast<MarkdownElementWithChildren*>(element.get())->getAllLinks();
links.insert(links.end(), para_links.begin(), para_links.end()); links.insert(links.end(), para_links.begin(), para_links.end());
} }
else if (element->getType() == MarkdownElement::Type::BULLET_LIST)
{
auto bullet_list = dynamic_cast<MarkdownBulletList*>(element.get());
for(unsigned idx=0; idx<bullet_list->getNumChildren(); idx++)
{
auto para_links = bullet_list->getChild(idx)->getAllLinks();
links.insert(links.end(), para_links.begin(), para_links.end());
}
}
} }
return links; return links;
} }

View file

@ -7,7 +7,6 @@
#include "StringUtils.h" #include "StringUtils.h"
#include <sstream> #include <sstream>
#include <iostream>
static constexpr char MULTILINE_QUOTE_DELIMITER[]{"```"}; static constexpr char MULTILINE_QUOTE_DELIMITER[]{"```"};
static constexpr char HEADING_DELIMITER{'#'}; static constexpr char HEADING_DELIMITER{'#'};
@ -151,12 +150,10 @@ void MarkdownParser::onTextSpanFinished()
{ {
if (mWorkingTextSpan) if (mWorkingTextSpan)
{ {
std::cout << "Adding to existing text span: " << std::endl;
mWorkingTextSpan->appendTextContent(mWorkingLine); mWorkingTextSpan->appendTextContent(mWorkingLine);
} }
else else
{ {
std::cout << "Adding new text span: " << mWorkingLine << std::endl;
auto text_span = std::make_unique<MarkdownTextSpan>(); auto text_span = std::make_unique<MarkdownTextSpan>();
text_span->addLine(mWorkingLine); text_span->addLine(mWorkingLine);
mWorkingTextSpan = text_span.get(); mWorkingTextSpan = text_span.get();
@ -183,7 +180,6 @@ void MarkdownParser::processLine(const std::string& line)
if (!mWorkingElement) if (!mWorkingElement)
{ {
std::cout << "Adding new paragraph " << std::endl;
auto paragraph = std::make_unique<MarkdownParagraph>(); auto paragraph = std::make_unique<MarkdownParagraph>();
mWorkingElement = paragraph.get(); mWorkingElement = paragraph.get();
mMarkdownDocument->addElement(std::move(paragraph)); mMarkdownDocument->addElement(std::move(paragraph));
@ -341,7 +337,6 @@ void MarkdownParser::onFoundBulletItem(const std::string& line)
} }
else else
{ {
std::cout << "Starting new bullet list" << std::endl;
auto bullet_list = std::make_unique<MarkdownBulletList>(); auto bullet_list = std::make_unique<MarkdownBulletList>();
mWorkingBulletList = bullet_list.get(); mWorkingBulletList = bullet_list.get();
@ -350,15 +345,13 @@ void MarkdownParser::onFoundBulletItem(const std::string& line)
auto bullet_item = std::make_unique<MarkdownBulletItem>(); auto bullet_item = std::make_unique<MarkdownBulletItem>();
mWorkingElement = bullet_item.get(); mWorkingElement = bullet_item.get();
mWorkingBulletList->addChild(std::move(bullet_item)); mWorkingBulletList->addChild(std::move(bullet_item));
processLine(StringUtils::removeUpTo(line, "*"));
} }
processLine(StringUtils::removeUpTo(line, "*"));
} }
} }
void MarkdownParser::onSectionFinished() void MarkdownParser::onSectionFinished()
{ {
std::cout << "Section is finished" << std::endl;
mWorkingElement = nullptr; mWorkingElement = nullptr;
mWorkingBulletList = nullptr; mWorkingBulletList = nullptr;
mWorkingTextSpan = nullptr; mWorkingTextSpan = nullptr;
@ -373,36 +366,29 @@ std::unique_ptr<MarkdownDocument> MarkdownParser::run(const std::string& content
while (std::getline(ss, line, '\n')) while (std::getline(ss, line, '\n'))
{ {
std::cout << "Processing line " << line << std::endl;
if (StringUtils::isWhitespaceOnly(line)) if (StringUtils::isWhitespaceOnly(line))
{ {
std::cout << "Is whitespace only " << std::endl;
onEmptyLine(); onEmptyLine();
continue; continue;
} }
else if (startsWithMultiLineQuote(line)) else if (startsWithMultiLineQuote(line))
{ {
std::cout << "Found multiline quote" << std::endl;
onFoundMultiLineQuote(line); onFoundMultiLineQuote(line);
} }
else if (auto result = startsWithCustomMultilineBlock(line); result >= 0) else if (auto result = startsWithCustomMultilineBlock(line); result >= 0)
{ {
std::cout << "Found custom multiline" << std::endl;
onFoundCustomMultiLineBlock(line, result); onFoundCustomMultiLineBlock(line, result);
} }
else if (startsWithHeading(line)) else if (startsWithHeading(line))
{ {
std::cout << "Found heading" << std::endl;
onFoundHeading(line); onFoundHeading(line);
} }
else if(startsWithBulletItem(line)) else if(startsWithBulletItem(line))
{ {
std::cout << "Found bulletitem" << std::endl;
onFoundBulletItem(line); onFoundBulletItem(line);
} }
else else
{ {
std::cout << "Found nothing - process line" << std::endl;
processLine(line); processLine(line);
} }
} }

View file

@ -28,11 +28,6 @@ private:
bool isInMultilineBlock() const; bool isInMultilineBlock() const;
bool startsWithMultiLineQuote(const std::string& line) const;
int startsWithCustomMultilineBlock(const std::string& line) const;
bool startsWithHeading(const std::string& line) const;
bool startsWithBulletItem(const std::string& line) const;
void onFoundMultiLineQuote(const std::string& line); void onFoundMultiLineQuote(const std::string& line);
void onFoundCustomMultiLineBlock(const std::string& line, unsigned blockSlot); void onFoundCustomMultiLineBlock(const std::string& line, unsigned blockSlot);
void onFoundHeading(const std::string& line); void onFoundHeading(const std::string& line);
@ -44,6 +39,11 @@ private:
void processLine(const std::string& line); void processLine(const std::string& line);
bool startsWithMultiLineQuote(const std::string& line) const;
int startsWithCustomMultilineBlock(const std::string& line) const;
bool startsWithHeading(const std::string& line) const;
bool startsWithBulletItem(const std::string& line) const;
unsigned mCustomDelimiterIndex{0}; unsigned mCustomDelimiterIndex{0};
std::vector<std::string> mCustomMultilineDelimiters; std::vector<std::string> mCustomMultilineDelimiters;
std::vector<std::string> mCustomInlineDelimiters; std::vector<std::string> mCustomInlineDelimiters;

View file

@ -1,15 +1,29 @@
#include "TemplatingEngine.h" #include "TemplatingEngine.h"
#include "TemplateSubstitutionContext.h"
#include "File.h" #include "File.h"
#include "TestFramework.h" #include "TestFramework.h"
#include "TestUtils.h" #include "TestUtils.h"
TEST_CASE(TestTemplatingEngine, "compiler") TEST_CASE(TestTemplatingEngine_BlockInherit, "compiler")
{ {
auto engine = TemplatingEngine(TestUtils::getTestDataDir()); auto engine = TemplatingEngine(TestUtils::getTestDataDir());
const auto content = engine.renderTemplate("index", nullptr); const auto content = engine.renderTemplate("index", nullptr);
File outfile(TestUtils::getTestOutputDir() / "index.html"); File outfile(TestUtils::getTestOutputDir(__FILE__) / "BlockInherit.html");
outfile.writeText(content);
}
TEST_CASE(TestTemplatingEngine_Simple, "compiler")
{
auto engine = TemplatingEngine(TestUtils::getTestDataDir());
TemplateSubstitutionContext sub_context;
sub_context.addSubstitution("content", "<div><p>test</p></div>");
const auto content = engine.renderTemplate("simple_template", &sub_context);
File outfile(TestUtils::getTestOutputDir(__FILE__) / "Simple.html");
outfile.writeText(content); outfile.writeText(content);
} }

View file

@ -1,30 +1,7 @@
# I'm a level one header
I'm some text under level one
## I'm a level two header
I'm some text under level two
```
I'm a code block
```
I'm a line under the code block, with some `inline code`.
### I'm a level three header
I'm a bullet point list: I'm a bullet point list:
* First point * First point
* Second point * Second point
* Third point * Third point
With a [hyperlink](www.imahyperlink.com) embedded. With some test after.
# I'm another level one header
I'm some inline math $a = b + c$ and I'm some standalone math:
$$
d = e + f
$$
![This is an image](https://myoctocat.com/assets/images/base-octocat.svg)

View file

@ -0,0 +1,10 @@
<!DOCTYPE html>
<html lang="en">
<head>
<link rel="stylesheet" href="style.css" />
<title>My title</title>
</head>
<body>
<div id="content">{{ content }}</div>
</body>
</html>