jgrogan/stuff-from-scratch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Some repairs for md_parser and template engine.

Browse source
This commit is contained in:
James Grogan 2022-12-07 10:21:28 +00:00
parent 8705859115
commit 22157169c0
14 changed files with 180 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/website-generator/WebsiteGenerator.cpp
View file

@ -16,6 +16,8 @@
#include "FileLogger.h"
#include <iostream>
WebsiteGenerator::WebsiteGenerator()
: mConfig(std::make_unique<SiteGeneratorConfig>()),
mTemplateEngine()

23
src/compiler/template_engine/TemplateElements.cpp
View file

@ -35,8 +35,9 @@ void TemplateBlock::addLine(const std::string& line)
std::string TemplateBlock::getRawContent() const
{
std::string content = "TemplateBlock: " + mName + "\n";
std::string content = "TemplateBlock: " + mName + " - Start \n";
content += TemplateNode::getRawContent();
content += "TemplateBlock: " + mName + " - End";
return content;
}
@ -53,10 +54,8 @@ std::string TemplateBlock::renderAsParent(TemplateSubstitutionContext* substitut
if (child->getType() == Type::EXPRESSION)
{
auto expression = dynamic_cast<TemplateExpression*>(child.get());
std::cout << "Got expression with content " << expression->getContent() << std::endl;
if (expression->getContent() == "super()")
{
std::cout << "Adding expression base" << std::endl;
content += base->render(substitutions, nullptr);
}
}
@ -116,7 +115,7 @@ const std::string& TemplateExpression::getContent() const
std::string TemplateExpression::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
{
if (substitutions->hasSubstitution(mContent))
if (substitutions && substitutions->hasSubstitution(mContent))
{
return substitutions->getSubstitution(mContent);
}
@ -147,9 +146,13 @@ bool TemplateTextBody::hasContent() const
std::string TemplateTextBody::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
{
std::string content;
for (const auto& line : mContent)
for(unsigned idx=0; idx<mContent.size(); idx++)
{
content += line + '\n';
content += mContent[idx];
if(idx != mContent.size() - 1)
{
content += '\n';
}
}
return content;
}
@ -157,9 +160,13 @@ std::string TemplateTextBody::render(TemplateSubstitutionContext* substitutions,
std::string TemplateTextBody::getRawContent() const
{
std::string content;
for (const auto& line : mContent)
for(unsigned idx=0; idx<mContent.size(); idx++)
{
content += "TemplateBody: " + line + "\n";
content += "Template Body L-" + std::to_string(idx) + ": " + mContent[idx];
if(idx != mContent.size() - 1)
{
content += '\n';
}
}
return content;
}

171
src/compiler/template_engine/TemplateFile.cpp
View file

@ -1,9 +1,11 @@
#include "TemplateFile.h"
#include "StringUtils.h"
#include "TemplateElements.h"
#include "TemplateNode.h"
#include <iostream>
#include "Lexer.h"
#include "File.h"
#include "StringUtils.h"
TemplateFile::TemplateFile(const Path& path)
: mPath(path),
@ -38,114 +40,119 @@ void TemplateFile::loadContent()
{
return;
}
mRawContent = File(mPath).readLines();
mWorkingLine = 0;
mWorkingNode = mRootNode.get();
mWorkingTextBody = std::make_unique<TemplateTextBody>(mWorkingNode);
mWorkingTextSpan = std::make_unique<TemplateTextBody>(mWorkingNode);
for (const auto& line : mRawContent)
{
processLine(line);
mWorkingLine++;
}
onTextBodyFinished();
onTextSpanFinished();
mHasLoaded = true;
}
void TemplateFile::onTextBodyFinished(std::string working_string)
std::string TemplateFile::dumpContent()
{
if (!working_string.empty())
return mRootNode->getRawContent();
}
void TemplateFile::onTextSpanFinished()
{
if (!mWorkingLineContent.empty())
{
mWorkingTextBody->addLine(working_string);
mWorkingTextSpan->addLine(mWorkingLineContent);
}
if (mWorkingTextBody->hasContent())
if (mWorkingTextSpan->hasContent())
{
mWorkingNode->addChild(std::move(mWorkingTextBody));
mWorkingTextBody = std::make_unique<TemplateTextBody>(mWorkingNode);
mWorkingNode->addChild(std::move(mWorkingTextSpan));
mWorkingTextSpan = std::make_unique<TemplateTextBody>(mWorkingNode);
}
mWorkingLineContent.clear();
}
unsigned TemplateFile::checkForStatement(const std::string& lineSection)
{
if (lineSection.empty())
{
return 0;
}
std::vector<std::string> hits;
unsigned hit_size{0};
if (Lexer::matchPattern("{%@%}", lineSection, '@', hits))
{
if (hits.size() == 1)
{
auto content = hits[0];
onTextSpanFinished();
onFoundStatement(content);
hit_size = 4 + content.size();
}
}
return hit_size;
}
unsigned TemplateFile::checkForExpression(const std::string& lineSection)
{
if (lineSection.empty())
{
return 0;
}
std::vector<std::string> hits;
unsigned hit_size{0};
if (Lexer::matchPattern("{{@}}", lineSection, '@', hits))
{
if (hits.size() == 1)
{
auto content = hits[0];
onTextSpanFinished();
onFoundExpression(content);
hit_size = 4 + content.size();
}
}
return hit_size;
}
void TemplateFile::processLine(const std::string& line)
{
bool last_was_ldelimiter{ false };
bool last_was_statement_rdelimiter{ false };
bool last_was_expression_rdelimiter{ false };
bool in_statement{ false };
bool in_expression{ false };
std::string working_string;
std::string last_working_string;
for (auto c : line)
unsigned line_position = 0;
mWorkingLineContent.clear();
while(line_position < line.size())
{
if (c == '{')
const auto remaining = line.substr(line_position, line.size() - line_position);
if(auto length = checkForStatement(remaining))
{
if (last_was_ldelimiter)
{
in_expression = true;
last_was_ldelimiter = false;
working_string = "";
}
else
{
last_was_ldelimiter = true;
}
line_position += length;
}
else if (c == '%' && last_was_ldelimiter)
else if(auto length = checkForExpression(remaining))
{
last_was_ldelimiter = false;
in_statement = true;
last_working_string = working_string;
working_string = "";
}
else if (c == '%' && in_statement)
{
last_was_statement_rdelimiter = true;
}
else if (c == '}' && (last_was_statement_rdelimiter || in_expression || last_was_expression_rdelimiter))
{
if (last_was_statement_rdelimiter)
{
onTextBodyFinished(last_working_string);
onFoundStatement(working_string);
last_was_statement_rdelimiter = false;
working_string = "";
}
else if (in_expression)
{
last_was_expression_rdelimiter = true;
in_expression = false;
}
else
{
onTextBodyFinished();
onFoundExpression(working_string);
last_was_expression_rdelimiter = false;
working_string = "";
}
}
else if (last_was_ldelimiter && (!in_statement && !in_expression))
{
last_was_ldelimiter = false;
working_string += '{';
working_string += c;
line_position += length;
}
else
{
working_string += c;
mWorkingLineContent += line[line_position];
line_position++;
}
}
if (!working_string.empty())
if (!mWorkingLineContent.empty())
{
mWorkingTextBody->addLine(working_string);
mWorkingTextSpan->addLine(mWorkingLineContent);
mWorkingLineContent.clear();
}
}
void TemplateFile::onFoundStatement(const std::string& statement_string)
{
const auto statement_elements = StringUtils::split(statement_string);
if (statement_elements.size() == 0)
{
return;
@ -163,7 +170,13 @@ void TemplateFile::onFoundStatement(const std::string& statement_string)
{
onFoundExtends(statement_elements);
}
}
void TemplateFile::onFoundExpression(const std::string& expression_string)
{
const auto stripped = StringUtils::stripSurroundingWhitepsace(expression_string);
auto expression = std::make_unique<TemplateExpression>(mWorkingNode, stripped);
mWorkingNode->addChild(std::move(expression));
}
void TemplateFile::onFoundBlock(const std::vector<std::string> args)
@ -192,15 +205,3 @@ void TemplateFile::onFoundExtends(const std::vector<std::string> args)
auto extends = std::make_unique<TemplateExtends>(mWorkingNode, args[1]);
mWorkingNode->addChild(std::move(extends));
}
void TemplateFile::onFoundExpression(const std::string& expression_string)
{
const auto stripped = StringUtils::stripSurroundingWhitepsace(expression_string);
auto expression = std::make_unique<TemplateExpression>(mWorkingNode, stripped);
mWorkingNode->addChild(std::move(expression));
}
std::string TemplateFile::dumpContent()
{
return mRootNode->getRawContent();
}

23
src/compiler/template_engine/TemplateFile.h
View file

@ -1,14 +1,14 @@
#pragma once
#include "File.h"
#include "TemplateNode.h"
#include <vector>
#include <string>
#include <filesystem>
class TemplateNode;
class TemplateTextBody;
using Path = std::filesystem::path;
class TemplateFile
{
public:
@ -27,27 +27,32 @@ public:
void loadContent();
private:
void onTextBodyFinished(std::string working_string = {});
unsigned checkForStatement(const std::string& lineSection);
void processLine(const std::string& line);
unsigned checkForExpression(const std::string& lineSection);
void onTextSpanFinished();
void onFoundStatement(const std::string& statement_string);
void onFoundExpression(const std::string& expression_string);
void onFoundBlock(const std::vector<std::string> args);
void onFoundEndBlock(const std::vector<std::string> args);
void onFoundExtends(const std::vector<std::string> args);
void onFoundExpression(const std::string& expression_string);
void processLine(const std::string& line);
Path mPath;
std::string mParentName;
std::vector<std::string> mRawContent;
bool mHasLoaded{false};
unsigned mWorkingLine{ 0 };
std::unique_ptr<TemplateNode> mRootNode;
TemplateNode* mWorkingNode{ nullptr };
std::unique_ptr<TemplateTextBody> mWorkingTextBody;
std::string mWorkingLineContent;
std::unique_ptr<TemplateTextBody> mWorkingTextSpan;
};

10
src/compiler/template_engine/TemplateNode.cpp
View file

@ -75,8 +75,18 @@ void TemplateNode::setExtensionParent(TemplateNode* parent)
mExtensionParent = parent;
}
void TemplateNode::setExtensionBase(TemplateNode* base)
{
mExtensionBase = base;
}
std::string TemplateNode::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
{
if (mExtensionBase)
{
return mExtensionBase->render(substitutions, this);
}
if (!parentContext && mExtensionParent)
{
parentContext = mExtensionParent;

3
src/compiler/template_engine/TemplateNode.h
View file

@ -42,10 +42,13 @@ public:
void setExtensionParent(TemplateNode* parent);
void setExtensionBase(TemplateNode* base);
protected:
std::vector<std::unique_ptr<TemplateNode> > mChildren;
TemplateNode* mParent{ nullptr };
TemplateNode* mExtensionParent{ nullptr };
TemplateNode* mExtensionBase{ nullptr };
};
using TemplateNodePtr = std::unique_ptr<TemplateNode>;

13
src/compiler/template_engine/TemplatingEngine.cpp
View file

@ -26,18 +26,16 @@ std::string TemplatingEngine::renderTemplate(const std::string& name, TemplateSu
if (!file->hasLoaded())
{
file->loadContent();
std::cout << file->dumpContent();
//std::cout << file->dumpContent();
processTemplate(file, nullptr);
}
return file->getContent()->render(substitutionContext, nullptr);
}
else
{
return {};
}
}
void TemplatingEngine::loadTemplateFiles()
@ -62,21 +60,20 @@ TemplateFile* TemplatingEngine::getTemplateFile(const std::string& name)
void TemplatingEngine::processTemplate(TemplateFile* file, TemplateNode* parent)
{
std::cout << "Processing file " << file->getName() << std::endl;
auto content = file->getContent();
if (parent)
{
std::cout << "Setting extension parent" << std::endl;
content->setExtensionParent(parent);
parent->setExtensionBase(content);
}
if (auto extension_node = dynamic_cast<TemplateExtends*>(content->getFirstChildShallow(TemplateNode::Type::EXTENDS)))
{
std::cout << "Found extension node " << std::endl;
if (auto extension_template = getTemplateFile(Path(extension_node->getPath())))
{
std::cout << "Found extension template " << std::endl;
extension_template->loadContent();
//std::cout << extension_template->dumpContent();
processTemplate(extension_template, content);
}
}

4
src/web/html/HtmlTextRun.h
View file

@ -17,7 +17,7 @@ public:
std::string toString(unsigned depth = 0, bool keepInline = false) const override
{
const auto prefix = std::string(2*depth, ' ');
return prefix + getText();
return getText();
}
};

11
src/web/markdown/MarkdownDocument.cpp
View file

@ -34,9 +34,18 @@ std::vector<MarkdownLink*> MarkdownDocument::getAllLinks() const
{
if (element->getType() == MarkdownElement::Type::PARAGRAPH)
{
auto para_links = dynamic_cast<MarkdownParagraph*>(element.get())->getAllLinks();
auto para_links = dynamic_cast<MarkdownElementWithChildren*>(element.get())->getAllLinks();
links.insert(links.end(), para_links.begin(), para_links.end());
}
else if (element->getType() == MarkdownElement::Type::BULLET_LIST)
{
auto bullet_list = dynamic_cast<MarkdownBulletList*>(element.get());
for(unsigned idx=0; idx<bullet_list->getNumChildren(); idx++)
{
auto para_links = bullet_list->getChild(idx)->getAllLinks();
links.insert(links.end(), para_links.begin(), para_links.end());
}
}
}
return links;
}

16
src/web/markdown/MarkdownParser.cpp
View file

@ -7,7 +7,6 @@
#include "StringUtils.h"
#include <sstream>
#include <iostream>
static constexpr char MULTILINE_QUOTE_DELIMITER[]{"```"};
static constexpr char HEADING_DELIMITER{'#'};
@ -151,12 +150,10 @@ void MarkdownParser::onTextSpanFinished()
{
if (mWorkingTextSpan)
{
std::cout << "Adding to existing text span: " << std::endl;
mWorkingTextSpan->appendTextContent(mWorkingLine);
}
else
{
std::cout << "Adding new text span: " << mWorkingLine << std::endl;
auto text_span = std::make_unique<MarkdownTextSpan>();
text_span->addLine(mWorkingLine);
mWorkingTextSpan = text_span.get();
@ -183,7 +180,6 @@ void MarkdownParser::processLine(const std::string& line)
if (!mWorkingElement)
{
std::cout << "Adding new paragraph " << std::endl;
auto paragraph = std::make_unique<MarkdownParagraph>();
mWorkingElement = paragraph.get();
mMarkdownDocument->addElement(std::move(paragraph));
@ -341,7 +337,6 @@ void MarkdownParser::onFoundBulletItem(const std::string& line)
}
else
{
std::cout << "Starting new bullet list" << std::endl;
auto bullet_list = std::make_unique<MarkdownBulletList>();
mWorkingBulletList = bullet_list.get();
@ -350,15 +345,13 @@ void MarkdownParser::onFoundBulletItem(const std::string& line)
auto bullet_item = std::make_unique<MarkdownBulletItem>();
mWorkingElement = bullet_item.get();
mWorkingBulletList->addChild(std::move(bullet_item));
processLine(StringUtils::removeUpTo(line, "*"));
}
processLine(StringUtils::removeUpTo(line, "*"));
}
}
void MarkdownParser::onSectionFinished()
{
std::cout << "Section is finished" << std::endl;
mWorkingElement = nullptr;
mWorkingBulletList = nullptr;
mWorkingTextSpan = nullptr;
@ -373,36 +366,29 @@ std::unique_ptr<MarkdownDocument> MarkdownParser::run(const std::string& content
while (std::getline(ss, line, '\n'))
{
std::cout << "Processing line " << line << std::endl;
if (StringUtils::isWhitespaceOnly(line))
{
std::cout << "Is whitespace only " << std::endl;
onEmptyLine();
continue;
}
else if (startsWithMultiLineQuote(line))
{
std::cout << "Found multiline quote" << std::endl;
onFoundMultiLineQuote(line);
}
else if (auto result = startsWithCustomMultilineBlock(line); result >= 0)
{
std::cout << "Found custom multiline" << std::endl;
onFoundCustomMultiLineBlock(line, result);
}
else if (startsWithHeading(line))
{
std::cout << "Found heading" << std::endl;
onFoundHeading(line);
}
else if(startsWithBulletItem(line))
{
std::cout << "Found bulletitem" << std::endl;
onFoundBulletItem(line);
}
else
{
std::cout << "Found nothing - process line" << std::endl;
processLine(line);
}
}

10
src/web/markdown/MarkdownParser.h
View file

@ -28,11 +28,6 @@ private:
bool isInMultilineBlock() const;
bool startsWithMultiLineQuote(const std::string& line) const;
int startsWithCustomMultilineBlock(const std::string& line) const;
bool startsWithHeading(const std::string& line) const;
bool startsWithBulletItem(const std::string& line) const;
void onFoundMultiLineQuote(const std::string& line);
void onFoundCustomMultiLineBlock(const std::string& line, unsigned blockSlot);
void onFoundHeading(const std::string& line);
@ -44,6 +39,11 @@ private:
void processLine(const std::string& line);
bool startsWithMultiLineQuote(const std::string& line) const;
int startsWithCustomMultilineBlock(const std::string& line) const;
bool startsWithHeading(const std::string& line) const;
bool startsWithBulletItem(const std::string& line) const;
unsigned mCustomDelimiterIndex{0};
std::vector<std::string> mCustomMultilineDelimiters;
std::vector<std::string> mCustomInlineDelimiters;

18
test/compiler/TestTemplatingEngine.cpp
View file

@ -1,15 +1,29 @@
#include "TemplatingEngine.h"
#include "TemplateSubstitutionContext.h"
#include "File.h"
#include "TestFramework.h"
#include "TestUtils.h"
TEST_CASE(TestTemplatingEngine, "compiler")
TEST_CASE(TestTemplatingEngine_BlockInherit, "compiler")
{
auto engine = TemplatingEngine(TestUtils::getTestDataDir());
const auto content = engine.renderTemplate("index", nullptr);
File outfile(TestUtils::getTestOutputDir() / "index.html");
File outfile(TestUtils::getTestOutputDir(__FILE__) / "BlockInherit.html");
outfile.writeText(content);
}
TEST_CASE(TestTemplatingEngine_Simple, "compiler")
{
auto engine = TemplatingEngine(TestUtils::getTestDataDir());
TemplateSubstitutionContext sub_context;
sub_context.addSubstitution("content", "<div><p>test</p></div>");
const auto content = engine.renderTemplate("simple_template", &sub_context);
File outfile(TestUtils::getTestOutputDir(__FILE__) / "Simple.html");
outfile.writeText(content);
}

25
test/data/simple_markdown.md
View file

@ -1,30 +1,7 @@
# I'm a level one header
I'm some text under level one
## I'm a level two header
I'm some text under level two
```
I'm a code block
```
I'm a line under the code block, with some `inline code`.
### I'm a level three header
I'm a bullet point list:
* First point
* Second point
* Third point
With a [hyperlink](www.imahyperlink.com) embedded.
# I'm another level one header
I'm some inline math $a = b + c$ and I'm some standalone math:
$$
d = e + f
$$
![This is an image](https://myoctocat.com/assets/images/base-octocat.svg)
With some test after.

10
test/data/simple_template.html Normal file
View file

@ -0,0 +1,10 @@
<!DOCTYPE html>
<html lang="en">
<head>
<link rel="stylesheet" href="style.css" />
<title>My title</title>
</head>
<body>
<div id="content">{{ content }}</div>
</body>
</html>