Clean project structure.

src/base/CMakeLists.txt

@@ -0,0 +1,6 @@
+add_subdirectory(compiler)
+add_subdirectory(compression)
+add_subdirectory(core)
+add_subdirectory(database)
+add_subdirectory(geometry)
+add_subdirectory(network)

src/base/compiler/CMakeLists.txt

@@ -0,0 +1,28 @@
+set(MODULE_NAME compiler)
+
+list(APPEND HEADERS 
+    Lexer.h
+    template_engine/TemplatingEngine.h
+    template_engine/TemplateFile.h
+    template_engine/TemplateNode.h
+    template_engine/TemplateElements.h
+    )
+
+list(APPEND SOURCES 
+    Lexer.cpp
+    template_engine/TemplatingEngine.cpp
+    template_engine/TemplateFile.cpp
+    template_engine/TemplateNode.cpp
+    template_engine/TemplateElements.cpp
+    )
+
+add_library(${MODULE_NAME} SHARED ${HEADERS} ${SOURCES})
+
+target_include_directories(${MODULE_NAME} PUBLIC
+                          ${CMAKE_CURRENT_SOURCE_DIR}
+                          ${CMAKE_CURRENT_SOURCE_DIR}/template_engine
+                          )
+target_link_libraries( ${MODULE_NAME} PUBLIC core)
+
+set_target_properties( ${MODULE_NAME} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON ) 
+set_property(TARGET ${MODULE_NAME} PROPERTY FOLDER src/base)

src/base/compiler/Lexer.cpp

@@ -0,0 +1,72 @@
+#include "Lexer.h"
+
+bool Lexer::matchPattern(const std::string& pattern, const std::string& checkString, char delimiter, std::vector<std::string>& hitSequence)
+{
+    if (checkString.empty())
+    {
+        return false;
+    }
+
+    if (pattern.empty())
+    {
+        return false;
+    }
+
+    bool found_pattern = true;
+    unsigned check_idx = 0;
+    unsigned pattern_idx = 0;
+
+    std::vector<std::string> hits;
+    std::string working_hit;
+    while(check_idx < checkString.size())
+    {
+        if (pattern_idx == pattern.size())
+        {
+            break;
+        }
+
+        auto check_char = checkString[check_idx];
+        auto pattern_char = pattern[pattern_idx];
+        if (pattern_char == delimiter)
+        {
+            if (pattern_idx + 1 < pattern.size())
+            {
+                if (check_char == pattern[pattern_idx + 1])
+                {
+                    hits.push_back(working_hit);
+                    working_hit.clear();
+                    pattern_idx++;
+                }
+                else
+                {
+                    working_hit+=check_char;
+                    check_idx++;
+                }
+            }
+            else
+            {
+                working_hit+=check_char;
+                check_idx++;
+            }
+        }
+        else
+        {
+            if (check_char == pattern_char)
+            {
+                check_idx++;
+                pattern_idx++;
+            }
+            else
+            {
+                found_pattern = false;
+                break;
+            }
+        }
+    }
+
+    if (found_pattern)
+    {
+        hitSequence = hits;
+    }
+    return found_pattern;
+}

src/base/compiler/Lexer.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+class Lexer
+{
+public:
+    // e.g. Pattern [@](@) returns <source, tag> for input: [source](tag) and delimiter @
+    static bool matchPattern(const std::string& pattern, const std::string& checkString, char delimiter, std::vector<std::string>& hitSequence);
+};

src/base/compiler/template_engine/TemplateElements.cpp

@@ -0,0 +1,172 @@
+#include "TemplateElements.h"
+
+#include "StringUtils.h"
+#include "TemplateSubstitutionContext.h"
+
+#include <iostream>
+
+TemplateExtends::TemplateExtends(TemplateNode* parent, const std::string& path)
+    : TemplateNode(parent)
+{
+    mPath = StringUtils::stripQuotes(path);
+};
+
+std::string TemplateExtends::getRawContent() const
+{
+    return "TemplateExtends: " + mPath;
+}
+
+std::string TemplateExtends::getPath() const
+{
+    return mPath;
+}
+
+TemplateBlock::TemplateBlock(TemplateNode* parent, const std::string& name)
+    : TemplateNode(parent),
+    mName(name)
+{
+
+}
+
+void TemplateBlock::addLine(const std::string& line)
+{
+    mBody.push_back(line);
+}
+
+std::string TemplateBlock::getRawContent() const
+{
+    std::string content = "TemplateBlock: " + mName + " - Start \n";
+    content += TemplateNode::getRawContent();
+    content += "TemplateBlock: " + mName + " - End";
+    return content;
+}
+
+std::string  TemplateBlock::getIdentifier() const
+{
+    return mName;
+}
+
+std::string TemplateBlock::renderAsParent(TemplateSubstitutionContext* substitutions, TemplateBlock* base)
+{
+    std::string content;
+    for (auto& child : mChildren)
+    {
+        if (child->getType() == Type::EXPRESSION)
+        {
+            auto expression = dynamic_cast<TemplateExpression*>(child.get());
+            if (expression->getContent() == "super()")
+            {
+                content += base->render(substitutions, nullptr);
+            }
+        }
+        else if(child->getType() == Type::TEXT_BODY)
+        {
+            content += child->render(substitutions, nullptr);
+        }
+    }
+    return content;
+}
+
+std::string TemplateBlock::renderAsLeaf(TemplateSubstitutionContext* substitutions)
+{
+    std::string content;
+    for (auto& child : mChildren)
+    {
+        if(child->getType() == Type::TEXT_BODY)
+        {
+            content += child->render(substitutions, nullptr);
+        }
+    }
+    return content;
+}
+
+std::string TemplateBlock::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
+{
+    std::string content;
+    if (parentContext)
+    {
+        if (auto parent_node = parentContext->getFirstChildShallow(Type::BLOCK, getIdentifier()))
+        {
+            content = dynamic_cast<TemplateBlock*>(parent_node)->renderAsParent(substitutions, this);
+        }
+        else
+        {
+            content= renderAsLeaf(substitutions);
+        }
+    }
+    else
+    {
+        content= renderAsLeaf(substitutions);
+    }
+    return content;
+}
+
+TemplateExpression::TemplateExpression(TemplateNode* parent, const std::string& content)
+    : TemplateNode(parent),
+    mContent(content)
+{
+
+}
+
+const std::string& TemplateExpression::getContent() const
+{
+    return mContent;
+}
+
+std::string TemplateExpression::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
+{
+    if (substitutions && substitutions->hasSubstitution(mContent))
+    {
+        return substitutions->getSubstitution(mContent);
+    }
+    return {};
+}
+
+std::string TemplateExpression::getRawContent() const
+{
+    return "TemplateExpression: " + mContent;
+}
+
+TemplateTextBody::TemplateTextBody(TemplateNode* parent)
+        : TemplateNode(parent)
+{
+
+}
+
+void TemplateTextBody::addLine(const std::string& content)
+{
+    mContent.push_back(content);
+}
+
+bool TemplateTextBody::hasContent() const
+{
+    return !mContent.empty();
+}
+
+std::string TemplateTextBody::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
+{
+    std::string content;
+    for(unsigned idx=0; idx<mContent.size(); idx++)
+    {
+        content += mContent[idx];
+        if(idx != mContent.size() - 1)
+        {
+            content += '\n';
+        }
+    }
+    return content;
+}
+
+std::string TemplateTextBody::getRawContent() const
+{
+    std::string content;
+    for(unsigned idx=0; idx<mContent.size(); idx++)
+    {
+        content += "Template Body L-" + std::to_string(idx) + ": " + mContent[idx];
+        if(idx != mContent.size() - 1)
+        {
+            content += '\n';
+        }
+    }
+    return content;
+}

src/base/compiler/template_engine/TemplateElements.h

@@ -0,0 +1,96 @@
+#pragma once
+
+#include "TemplateNode.h"
+
+class TemplateExtends : public TemplateNode
+{
+public:
+    TemplateExtends(TemplateNode* parent, const std::string& path);
+
+    virtual ~TemplateExtends() = default;
+
+    std::string getRawContent() const override;
+    std::string getPath() const;
+
+    Type getType() const override
+    {
+        return Type::EXTENDS;
+    }
+
+private:
+    std::string mPath;
+};
+
+class TemplateBlock : public TemplateNode
+{
+public:
+    TemplateBlock(TemplateNode* parent, const std::string& name);
+
+    virtual ~TemplateBlock() = default;
+
+    void addLine(const std::string& line);
+
+    std::string getRawContent() const override;
+
+    std::string getIdentifier() const override;
+
+    Type getType() const override
+    {
+        return Type::BLOCK;
+    }
+
+    std::string renderAsParent(TemplateSubstitutionContext* substitutions, TemplateBlock* base);
+
+    std::string renderAsLeaf(TemplateSubstitutionContext* substitutions);
+
+    std::string render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext) override;
+
+private:
+    std::string mName;
+    std::vector<std::string> mBody;
+};
+
+class TemplateExpression : public TemplateNode
+{
+public:
+    TemplateExpression(TemplateNode* parent, const std::string& content);
+
+    virtual ~TemplateExpression() = default;
+
+    std::string getRawContent() const override;
+
+    const std::string& getContent() const;
+
+    Type getType() const override
+    {
+        return Type::EXPRESSION;
+    }
+
+    std::string render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext) override;
+
+private:
+    std::string mContent;
+};
+
+class TemplateTextBody : public TemplateNode
+{
+public:
+    TemplateTextBody(TemplateNode* parent);
+
+    virtual ~TemplateTextBody() = default;
+
+    void addLine(const std::string& content);
+
+    std::string getRawContent() const override;
+
+    bool hasContent() const;
+
+    std::string render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext) override;
+
+    Type getType() const override
+    {
+        return Type::TEXT_BODY;
+    }
+private:
+    std::vector<std::string> mContent;
+};

src/base/compiler/template_engine/TemplateFile.cpp

@@ -0,0 +1,207 @@
+#include "TemplateFile.h"
+
+#include "TemplateElements.h"
+#include "TemplateNode.h"
+
+#include "Lexer.h"
+#include "File.h"
+#include "StringUtils.h"
+
+TemplateFile::TemplateFile(const Path& path)
+    : mPath(path),
+      mRootNode(std::make_unique<TemplateNode>(nullptr))
+{
+
+}
+
+TemplateFile::~TemplateFile()
+{
+
+}
+
+std::string TemplateFile::getName() const
+{
+    return mPath.stem().string();
+}
+
+bool TemplateFile::hasLoaded() const
+{
+    return mHasLoaded;
+}
+
+TemplateNode* TemplateFile::getContent() const
+{
+    return mRootNode.get();
+}
+
+void TemplateFile::loadContent()
+{
+    if (mHasLoaded)
+    {
+        return;
+    }
+    mRawContent = File(mPath).readLines();
+
+    mWorkingNode = mRootNode.get();
+    mWorkingTextSpan = std::make_unique<TemplateTextBody>(mWorkingNode);
+    for (const auto& line : mRawContent)
+    {
+        processLine(line);
+    }
+    onTextSpanFinished();
+
+    mHasLoaded = true;
+}
+
+std::string TemplateFile::dumpContent()
+{
+    return mRootNode->getRawContent();
+}
+
+void TemplateFile::onTextSpanFinished()
+{
+    if (!mWorkingLineContent.empty())
+    {
+        mWorkingTextSpan->addLine(mWorkingLineContent);
+    }
+
+    if (mWorkingTextSpan->hasContent())
+    {
+        mWorkingNode->addChild(std::move(mWorkingTextSpan));
+        mWorkingTextSpan = std::make_unique<TemplateTextBody>(mWorkingNode);
+    }
+    mWorkingLineContent.clear();
+}
+
+std::size_t TemplateFile::checkForStatement(const std::string& lineSection)
+{
+    if (lineSection.empty())
+    {
+        return 0;
+    }
+
+    std::vector<std::string> hits;
+    std::size_t hit_size{0};
+    if (Lexer::matchPattern("{%@%}", lineSection, '@', hits))
+    {
+        if (hits.size() == 1)
+        {
+            auto content = hits[0];
+
+            onTextSpanFinished();
+
+            onFoundStatement(content);
+            hit_size = 4 + content.size();
+        }
+    }
+    return hit_size;
+}
+
+std::size_t TemplateFile::checkForExpression(const std::string& lineSection)
+{
+    if (lineSection.empty())
+    {
+        return 0;
+    }
+
+    std::vector<std::string> hits;
+    std::size_t hit_size{0};
+    if (Lexer::matchPattern("{{@}}", lineSection, '@', hits))
+    {
+        if (hits.size() == 1)
+        {
+            auto content = hits[0];
+
+            onTextSpanFinished();
+
+            onFoundExpression(content);
+            hit_size = 4 + content.size();
+        }
+    }
+    return hit_size;
+}
+
+void TemplateFile::processLine(const std::string& line)
+{
+    std::size_t line_position = 0;
+    mWorkingLineContent.clear();
+    while(line_position < line.size())
+    {
+        const auto remaining = line.substr(line_position, line.size() - line_position);
+        if(auto length = checkForStatement(remaining))
+        {
+            line_position += length;
+        }
+        else if(auto length = checkForExpression(remaining))
+        {
+            line_position += length;
+        }
+        else
+        {
+            mWorkingLineContent += line[line_position];
+            line_position++;
+        }
+    }
+
+    if (!mWorkingLineContent.empty())
+    {
+        mWorkingTextSpan->addLine(mWorkingLineContent);
+        mWorkingLineContent.clear();
+    }
+}
+
+void TemplateFile::onFoundStatement(const std::string& statement_string)
+{
+    const auto statement_elements = StringUtils::split(statement_string);
+    if (statement_elements.size() == 0)
+    {
+        return;
+    }
+    const auto statement_type = statement_elements[0];
+    if (statement_type == "block" && statement_elements.size() == 2)
+    {
+        onFoundBlock(statement_elements);
+    }
+    else if (statement_type == "endblock")
+    {
+        onFoundEndBlock(statement_elements);
+    }
+    else if (statement_type == "extends")
+    {
+        onFoundExtends(statement_elements);
+    }
+}
+
+void TemplateFile::onFoundExpression(const std::string& expression_string)
+{
+    const auto stripped = StringUtils::stripSurroundingWhitepsace(expression_string);
+    auto expression = std::make_unique<TemplateExpression>(mWorkingNode, stripped);
+    mWorkingNode->addChild(std::move(expression));
+}
+
+void TemplateFile::onFoundBlock(const std::vector<std::string> args)
+{
+    if (args.size() != 2)
+    {
+        return;
+    }
+    auto block = std::make_unique<TemplateBlock>(mWorkingNode, args[1]);
+    auto temp = block.get();
+    mWorkingNode->addChild(std::move(block));
+    mWorkingNode = temp;
+}
+
+void TemplateFile::onFoundEndBlock(const std::vector<std::string> args)
+{
+    mWorkingNode = mWorkingNode->getParent();
+}
+
+void TemplateFile::onFoundExtends(const std::vector<std::string> args)
+{
+    if (args.size() != 2)
+    {
+        return;
+    }
+    auto extends = std::make_unique<TemplateExtends>(mWorkingNode, args[1]);
+    mWorkingNode->addChild(std::move(extends));
+}

src/base/compiler/template_engine/TemplateFile.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <filesystem>
+
+class TemplateNode;
+class TemplateTextBody;
+
+using Path = std::filesystem::path;
+
+class TemplateFile
+{
+public:
+    TemplateFile(const Path& path);
+
+    ~TemplateFile();
+
+    std::string dumpContent();
+
+    std::string getName() const;
+
+    TemplateNode* getContent() const;
+
+    bool hasLoaded() const;
+
+    void loadContent();
+
+private:
+    std::size_t checkForStatement(const std::string& lineSection);
+
+    std::size_t checkForExpression(const std::string& lineSection);
+
+    void onTextSpanFinished();
+
+    void onFoundStatement(const std::string& statement_string);
+
+    void onFoundExpression(const std::string& expression_string);
+
+    void onFoundBlock(const std::vector<std::string> args);
+
+    void onFoundEndBlock(const std::vector<std::string> args);
+
+    void onFoundExtends(const std::vector<std::string> args);
+
+    void processLine(const std::string& line);
+
+    Path mPath;
+    std::string mParentName;
+    std::vector<std::string> mRawContent;
+    bool mHasLoaded{false};
+
+    std::unique_ptr<TemplateNode> mRootNode;
+    TemplateNode* mWorkingNode{ nullptr };
+
+    std::string mWorkingLineContent;
+    std::unique_ptr<TemplateTextBody> mWorkingTextSpan;
+};

src/base/compiler/template_engine/TemplateNode.cpp

@@ -0,0 +1,101 @@
+#include "TemplateNode.h"
+
+#include "TemplateSubstitutionContext.h"
+
+TemplateNode::TemplateNode(TemplateNode* parent)
+        : mParent(parent)
+{
+
+}
+
+TemplateNode* TemplateNode::getParent() const
+{
+    return mParent;
+}
+
+void TemplateNode::addChild(std::unique_ptr<TemplateNode> child)
+{
+    mChildren.push_back(std::move(child));
+}
+
+std::size_t TemplateNode::getNumChildren() const
+{
+    return mChildren.size();
+}
+
+TemplateNode* TemplateNode::getFirstChildShallow(Type type, const std::string& identifier) const
+{
+    for (const auto& child : mChildren)
+    {
+        if (child->getType() == type)
+        {
+            if (!identifier.empty())
+            {
+                if (child->getIdentifier() == identifier)
+                {
+                    return child.get();
+                }
+            }
+            else
+            {
+                return child.get();
+            }
+        }
+    }
+    return nullptr;
+}
+
+TemplateNode::Type TemplateNode::getType() const
+{
+    return Type::NONE;
+}
+
+std::string TemplateNode::getIdentifier() const
+{
+    return {};
+}
+
+TemplateNode* TemplateNode::getChild(std::size_t index) const
+{
+    return mChildren[index].get();
+}
+
+std::string TemplateNode::getRawContent() const
+{
+    std::string content;
+    for (const auto& child : mChildren)
+    {
+        content += child->getRawContent() + "\n";
+    }
+    return content;
+}
+
+void TemplateNode::setExtensionParent(TemplateNode* parent)
+{
+    mExtensionParent = parent;
+}
+
+void TemplateNode::setExtensionBase(TemplateNode* base)
+{
+    mExtensionBase = base;
+}
+
+std::string TemplateNode::render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext)
+{
+    if (mExtensionBase)
+    {
+        return mExtensionBase->render(substitutions, this);
+    }
+
+    if (!parentContext && mExtensionParent)
+    {
+        parentContext = mExtensionParent;
+    }
+
+    std::string content;
+    for (size_t idx = 0; idx < mChildren.size(); idx++)
+    {
+        content += mChildren[idx]->render(substitutions, parentContext);
+    }
+    return content;
+}

src/base/compiler/template_engine/TemplateNode.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+class TemplateSubstitutionContext;
+
+class TemplateNode
+{
+public:
+    enum class Type
+    {
+        NONE,
+        BLOCK,
+        EXPRESSION,
+        EXTENDS,
+        TEXT_BODY
+    };
+
+    TemplateNode(TemplateNode* parent);
+
+    virtual ~TemplateNode() = default;
+
+    virtual void addChild(std::unique_ptr<TemplateNode> child);
+
+    TemplateNode* getChild(std::size_t index) const;
+
+    TemplateNode* getFirstChildShallow(Type type, const std::string& identifier = {}) const;
+
+    virtual std::string getIdentifier() const;
+
+    std::size_t getNumChildren() const;
+
+    TemplateNode* getParent() const;
+
+    virtual std::string getRawContent() const;
+
+    virtual Type getType() const;
+
+    virtual std::string render(TemplateSubstitutionContext* substitutions, TemplateNode* parentContext = nullptr);
+
+    void setExtensionParent(TemplateNode* parent);
+
+    void setExtensionBase(TemplateNode* base);
+
+protected:
+    std::vector<std::unique_ptr<TemplateNode> > mChildren;
+    TemplateNode* mParent{ nullptr };
+    TemplateNode* mExtensionParent{ nullptr };
+    TemplateNode* mExtensionBase{ nullptr };
+};
+
+using TemplateNodePtr = std::unique_ptr<TemplateNode>;

src/base/compiler/template_engine/TemplateSubstitutionContext.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+class TemplateSubstitutionContext
+{
+public:
+
+    void addSubstitution(const std::string& key, const std::string& value)
+    {
+        mSubstitutions[key] = value;
+    }
+
+    bool hasSubstitution(const std::string& key) const
+    {
+        return mSubstitutions.find(key) != mSubstitutions.end();
+    }
+
+    std::string getSubstitution(const std::string& key) const
+    {
+        auto iter = mSubstitutions.find(key);
+        if(iter != mSubstitutions.end())
+        {
+            return iter->second;
+        }
+        else
+        {
+            return {};
+        }
+    }
+
+private:
+    std::unordered_map<std::string, std::string> mSubstitutions;
+};

src/base/compiler/template_engine/TemplatingEngine.cpp

@@ -0,0 +1,80 @@
+#include "TemplatingEngine.h"
+
+#include "Directory.h"
+#include "FileLogger.h"
+
+#include "TemplateElements.h"
+#include "TemplateSubstitutionContext.h"
+
+#include <iostream>
+
+TemplatingEngine::TemplatingEngine(const Path& workingDirectory)
+    : mWorkingDirectory(workingDirectory)
+{
+
+}
+
+std::string TemplatingEngine::renderTemplate(const std::string& name, TemplateSubstitutionContext* substitutionContext)
+{
+    if (mTemplateFiles.empty())
+    {
+        loadTemplateFiles();
+    }
+
+    if (auto file = getTemplateFile(name))
+    {
+        if (!file->hasLoaded())
+        {
+            file->loadContent();
+            //std::cout << file->dumpContent();
+
+            processTemplate(file, nullptr);
+        }
+        return file->getContent()->render(substitutionContext, nullptr);
+    }
+    else
+    {
+        return {};
+    }
+}
+
+void TemplatingEngine::loadTemplateFiles()
+{
+    const auto files = Directory::getFilesWithExtension(mWorkingDirectory, mTemplateExtension);
+    for (const auto& path : files)
+    {
+        mTemplateFiles[path.stem().string()] = std::make_unique<TemplateFile>(path);
+    }
+    MLOG_INFO("Found: " << mTemplateFiles.size() << " templates in " << mWorkingDirectory);
+}
+
+TemplateFile* TemplatingEngine::getTemplateFile(const Path& path)
+{
+    return getTemplateFile(path.stem().string());
+}
+
+TemplateFile* TemplatingEngine::getTemplateFile(const std::string& name)
+{
+    return mTemplateFiles[name].get();
+}
+
+void TemplatingEngine::processTemplate(TemplateFile* file, TemplateNode* parent)
+{
+    auto content = file->getContent();
+    if (parent)
+    {
+        content->setExtensionParent(parent);
+        parent->setExtensionBase(content);
+    }
+
+    if (auto extension_node = dynamic_cast<TemplateExtends*>(content->getFirstChildShallow(TemplateNode::Type::EXTENDS)))
+    {
+        if (auto extension_template = getTemplateFile(Path(extension_node->getPath())))
+        {
+            extension_template->loadContent();
+            //std::cout << extension_template->dumpContent();
+
+            processTemplate(extension_template, content);
+        }
+    }
+}

src/base/compiler/template_engine/TemplatingEngine.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include "File.h"
+#include "TemplateFile.h"
+
+#include <vector>
+#include <string>
+#include <memory>
+#include <unordered_map>
+
+class TemplateSubstitutionContext;
+
+class TemplatingEngine
+{
+public:
+    TemplatingEngine(const Path& workingDirectory);
+
+    std::string renderTemplate(const std::string& name, TemplateSubstitutionContext* substitutionContext);
+
+private:
+    TemplateFile* getTemplateFile(const std::string& name);
+    TemplateFile* getTemplateFile(const Path& path);
+
+    void loadTemplateFiles();
+    void processTemplate(TemplateFile* file, TemplateNode* parent = nullptr);
+
+    std::unordered_map<std::string, std::unique_ptr<TemplateFile> > mTemplateFiles;
+    Path mWorkingDirectory;
+    std::string mTemplateExtension{ ".html" };
+};

src/base/compression/AbstractEncoder.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "AbstractChecksumCalculator.h"
+
+#include <vector>
+
+class BitStream;
+
+class AbstractEncoder
+{
+public:
+    AbstractEncoder(BitStream* inputStream, BitStream* outputStream)
+        : mInputStream(inputStream),
+          mOutputStream(outputStream)
+    {
+
+    }
+
+    virtual ~AbstractEncoder() = default;
+
+    void addChecksumCalculator(AbstractChecksumCalculator* calculator)
+    {
+        mChecksumCalculators.push_back(calculator);
+    }
+
+    virtual bool encode() = 0;
+    virtual bool decode() = 0;
+
+protected:
+
+    std::vector<AbstractChecksumCalculator*> mChecksumCalculators;
+    BitStream* mInputStream{nullptr};
+    BitStream* mOutputStream{nullptr};
+};

src/base/compression/Adler32Checksum.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "AbstractChecksumCalculator.h"
+
+class Adler32Checksum : public AbstractChecksumCalculator
+{
+public:
+    void addValue(unsigned char val) override
+    {
+        mSum1 = (mSum1 + val) % MOD_ADLER32;
+        mSum2 = (mSum2 + mSum1) % MOD_ADLER32;
+    }
+
+    uint32_t getChecksum() const override
+    {
+        return (mSum2 << 16) | mSum1;
+    }
+
+    void reset() override
+    {
+        mSum1 = 1;
+        mSum2 = 0;
+    }
+
+private:
+    static constexpr unsigned MOD_ADLER32{65536};
+    uint32_t mSum1{1};
+    uint32_t mSum2{0};
+};

src/base/compression/CMakeLists.txt

@@ -0,0 +1,27 @@
+set(MODULE_NAME compression)
+
+list(APPEND SOURCES 
+    StreamCompressor.cpp
+    huffman/HuffmanEncoder.cpp
+    huffman/HuffmanStream.cpp
+    huffman/HuffmanCodeLengthTable.cpp
+    huffman/HuffmanTree.cpp
+    RunLengthEncoder.cpp
+    ZlibEncoder.cpp
+    deflate/DeflateEncoder.cpp
+    deflate/DeflateBlock.cpp
+    Lz77Encoder.cpp
+    CyclicRedundancyChecker.cpp
+    )
+
+add_library(${MODULE_NAME} SHARED ${SOURCES})
+
+target_include_directories(${MODULE_NAME} PUBLIC
+                          ${CMAKE_CURRENT_SOURCE_DIR}
+                          ${CMAKE_CURRENT_SOURCE_DIR}/deflate
+                          ${CMAKE_CURRENT_SOURCE_DIR}/huffman
+                          )
+                          
+target_link_libraries(${MODULE_NAME} PUBLIC core)
+set_property(TARGET ${MODULE_NAME} PROPERTY FOLDER src/base)
+set_target_properties( ${MODULE_NAME} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON )

src/base/compression/CyclicRedundancyChecker.cpp

@@ -0,0 +1,43 @@
+#include "CyclicRedundancyChecker.h"
+
+void CyclicRedundancyChecker::createTable()
+{
+    mTable = std::vector<unsigned long>(TABLE_SIZE, 0);
+    unsigned long c{0};
+    for (int n = 0; n < TABLE_SIZE; n++)
+    {
+        c = (unsigned long) n;
+        for (int k = 0; k < 8; k++)
+        {
+            if (c & 1)
+            {
+                c = 0xedb88320L ^ (c >> 1);
+            }
+            else
+            {
+                c = c >> 1;
+            }
+        }
+        mTable[n] = c;
+    }
+    mTableComputed = true;
+}
+
+void CyclicRedundancyChecker::addValue(unsigned char val)
+{
+    if (!mTableComputed)
+    {
+        createTable();
+    }
+    mLastValue = mTable[(mLastValue ^ val) & 0xff] ^ (mLastValue >> 8);
+}
+
+uint32_t CyclicRedundancyChecker::getChecksum() const
+{
+    return mLastValue ^ 0xffffffffL;
+}
+
+void CyclicRedundancyChecker::reset()
+{
+    mLastValue = 0xffffffffL;
+}

src/base/compression/CyclicRedundancyChecker.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "AbstractChecksumCalculator.h"
+
+#include <vector>
+
+class CyclicRedundancyChecker : public AbstractChecksumCalculator
+{
+public:
+    void addValue(unsigned char val) override;
+
+    uint32_t getChecksum() const override;
+
+    void reset() override;
+
+private:
+    void createTable();
+    bool mTableComputed{false};
+
+    uint32_t mLastValue{0xffffffffL};
+
+    static const std::size_t TABLE_SIZE{ 256 };
+    std::vector<unsigned long> mTable;
+};

src/base/compression/Lz77Encoder.cpp

@@ -0,0 +1,330 @@
+#include "Lz77Encoder.h"
+
+#include "StringUtils.h"
+#include "BitStream.h"
+#include "ByteUtils.h"
+#include "HuffmanEncoder.h"
+
+#include <iostream>
+
+Lz77Encoder::Lz77Encoder(BitStream* inputStream, BitStream* outputStream)
+    : AbstractEncoder(inputStream, outputStream),
+      mSearchBuffer(mSearchBufferSize),
+      mLookaheadBuffer(mLookAheadBufferSize)
+{
+
+}
+
+void Lz77Encoder::setPrefixCodeGenerator(std::unique_ptr<PrefixCodeGenerator> generator)
+{
+    mCodeGenerator = std::move(generator);
+}
+
+bool Lz77Encoder::hitBufferFull() const
+{
+    return mHitBuffer.size() == mMaxHitBufferSize;
+}
+
+void Lz77Encoder::populateSearchBuffer(const Hit& hit)
+{
+    const auto& [length, distance, next_char] = hit;
+
+    if (length == 0)
+    {
+        mSearchBuffer.addItem(next_char);
+    }
+    else
+    {
+        std::vector<unsigned char> new_items(distance, 0);
+        for(unsigned idx=0 ;idx<distance; idx++)
+        {
+            new_items[idx] = getSearchBufferItem(idx);
+        }
+        for(auto item : new_items)
+        {
+            mSearchBuffer.addItem(item);
+        }
+
+        int difference = int(length) - distance;
+        if (difference > 0)
+        {
+            for(unsigned idx=0; idx<unsigned(difference); idx++)
+            {
+                mSearchBuffer.addItem(mLookaheadBuffer.getItem(idx));
+            }
+        }
+    }
+}
+
+unsigned char Lz77Encoder::getSearchBufferItem(unsigned index) const
+{
+    return mSearchBuffer.getItem(mSearchBuffer.getNumItems() - 1 - index);
+}
+
+unsigned Lz77Encoder::lookAheadForMatchingChars(unsigned distance)
+{
+    unsigned length{0};
+    for(unsigned idx=0; idx<unsigned(mMaxLookAheadBufferIndex + 1); idx++)
+    {
+        int search_offset = int(distance-1) - idx;
+        unsigned char search_char{0};
+        if (search_offset < 0)
+        {
+            search_char = mLookaheadBuffer.getItem(-search_offset - 1);
+        }
+        else
+        {
+            search_char = getSearchBufferItem(static_cast<unsigned>(search_offset));
+        }
+        unsigned char lookahead_char = mLookaheadBuffer.getItem(idx);
+        if ((lookahead_char != search_char) || (idx == mMaxLookAheadBufferIndex))
+        {
+            if (idx + 1>= mMinLengthMatchSize)
+            {
+                length = idx + 1;
+            }
+            break;
+        }
+    }
+    return length;
+}
+
+void Lz77Encoder::lookForMatches(unsigned char searchChar, unsigned& hitLength, unsigned& hitOffset)
+{
+    for (unsigned idx = 0; idx< mSearchBuffer.getNumItems(); idx++)
+    {
+        if (mSearchBuffer.getItem(mSearchBuffer.getNumItems() - 1 - idx) == searchChar)
+        {
+            auto num_hits = lookAheadForMatchingChars(idx + 1);
+            if (num_hits > 0 && num_hits >= hitLength)
+            {
+                hitLength = num_hits;
+                hitOffset = idx + 1;
+            }
+        }
+    }
+}
+
+bool Lz77Encoder::lookAheadSourceEmpty() const
+{
+    if (mLookaheadBuffer.getNumItems() < mLookAheadBufferSize)
+    {
+        return true;
+    }
+
+    if (mMaxLookAheadBufferIndex < int(mLookAheadBufferSize) - 1)
+    {
+        return true;
+    }
+
+    return false;
+}
+
+void Lz77Encoder::populateLookaheadBuffer(unsigned size, bool firstPass)
+{
+    if (!firstPass && lookAheadSourceEmpty())
+    {
+        for(unsigned idx=0; idx<size; idx++)
+        {
+            mLookaheadBuffer.addItem(0);
+            mMaxLookAheadBufferIndex--;
+        }
+        return;
+    }
+
+    bool stream_finished{false};
+    unsigned stream_end_id{0};
+    for(unsigned idx=0; idx<size; idx++)
+    {
+        if (!stream_finished)
+        {
+            auto byte = mInputStream->readNextByte();
+            if (!byte)
+            {
+                stream_finished = true;
+                stream_end_id = idx -1;
+                mLookaheadBuffer.addItem(0);
+                mMaxLookAheadBufferIndex--;
+                continue;
+            }
+            else
+            {
+                mLookaheadBuffer.addItem(*byte);
+            }
+        }
+        else
+        {
+            mLookaheadBuffer.addItem(0);
+            mMaxLookAheadBufferIndex--;
+        }
+    }
+
+    if (stream_finished && firstPass)
+    {
+        mMaxLookAheadBufferIndex = stream_end_id;
+    }
+}
+
+bool Lz77Encoder::encode()
+{
+    if (!mCodeGenerator)
+    {
+        mCodeGenerator = std::make_unique<HuffmanEncoder>();
+    }
+
+    // Fill the lookahead buffer
+    mMaxLookAheadBufferIndex = mLookAheadBufferSize - 1;
+    populateLookaheadBuffer(mLookAheadBufferSize, true);
+    if(mMaxLookAheadBufferIndex < 0)
+    {
+        return true;
+    }
+
+    bool input_stream_ended{false};
+    while(!hitBufferFull())
+    {
+        if (mMaxLookAheadBufferIndex < 0)
+        {
+            input_stream_ended = true;
+            break;
+        }
+
+        const auto working_byte = mLookaheadBuffer.getItem(0);
+        unsigned hit_length{0};
+        unsigned hit_distance{0};
+        lookForMatches(working_byte, hit_length, hit_distance);
+
+        const Hit hit{hit_length, hit_distance, working_byte};
+        mHitBuffer.push_back(hit);
+
+        populateSearchBuffer(hit);
+        if (hit_length == 0)
+        {
+            populateLookaheadBuffer(1);
+        }
+        else
+        {
+            populateLookaheadBuffer(hit_length);
+        }
+    }
+
+    return input_stream_ended;
+}
+
+const std::vector<Lz77Encoder::Hit>& Lz77Encoder::getHitBuffer() const
+{
+    return mHitBuffer;
+}
+
+/*
+void Lz77Encoder::flushHitBuffer()
+{
+    // If dynamic huffman build trees
+    if (!mCodeGenerator)
+    {
+        mCodeGenerator = std::make_unique<HuffmanEncoder>();
+    }
+
+    // Convert hit buffer to prefix codes and write to output stream
+    for (const auto& hit : mHitBuffer)
+    {
+        const auto& [length, distance, next_char] = hit;
+
+        PrefixCode code;
+        if (length == 0)
+        {
+            code = *mCodeGenerator->getLiteralValue(next_char);
+            std::cout << "Writing symbol " << static_cast<int>(next_char) << " with code " << ByteUtils::toString(code.getData(), code.getLength()) << "\n";
+
+            mOutputStream->writeNBits(code.getData(), code.getLength());
+        }
+        else
+        {
+            code = *mCodeGenerator->getLengthValue(length);
+            const auto distance_code = mCodeGenerator->getDistanceValue(distance);
+
+            std::cout << "Writing length " << length << " with code " << ByteUtils::toString(code.getData(), code.getLength()) << "\n";
+            mOutputStream->writeNBits(code.getData(), code.getLength());
+
+            std::cout << "Writing distance " << distance << " with code " << ByteUtils::toString(distance_code.getData(), distance_code.getLength()) << "\n";
+            mOutputStream->writeNBits(distance_code.getData(), distance_code.getLength());
+        }
+    }
+
+    auto eos_code = mCodeGenerator->getEndOfStreamValue();
+    std::cout << "Writing EOS value with code " << ByteUtils::toString(eos_code->getData(), eos_code->getLength()) << "\n";
+
+    mOutputStream->writeNBits(eos_code->getData(), eos_code->getLength());
+}
+*/
+
+bool Lz77Encoder::decode()
+{
+    /*
+    std::string ret;
+
+    unsigned loc{0};
+    while(loc < stream.size())
+    {
+        auto working_char = stream[loc];
+        if (working_char == '@')
+        {
+            unsigned loc_working = loc;
+
+            auto remainder = stream.size() - loc;
+            std::string offset;
+
+            unsigned length_loc{0};
+            for(unsigned jdx=0; jdx< remainder; jdx++)
+            {
+                loc++;
+
+                auto offset_char = stream[loc];
+                if (offset_char == 'L')
+                {
+                    loc++;
+                    break;
+                }
+                else
+                {
+                    offset += offset_char;
+                }
+            }
+            unsigned offset_amount = std::stoul(offset);
+
+            std::string length;
+            remainder = stream.size() - loc;
+
+            for(unsigned jdx=0; jdx< remainder; jdx++)
+            {
+                auto length_char = stream[loc];
+                if (StringUtils::IsAlphabetical(length_char) || length_char == '@')
+                {
+                    break;
+                }
+                else
+                {
+                    loc++;
+                    length += length_char;
+                }
+            }
+
+            unsigned length_amount = std::stoul(length);
+
+            auto buffer_index = ret.size() - offset_amount;
+            for(unsigned jdx=buffer_index;jdx<buffer_index+length_amount; jdx++)
+            {
+                ret += ret[jdx];
+            }
+        }
+        else
+        {
+            loc++;
+            ret += working_char;
+        }
+    }
+    return ret;
+    */
+    return false;
+}
+

src/base/compression/Lz77Encoder.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include "AbstractEncoder.h"
+#include "HuffmanEncoder.h"
+#include "CircleBuffer.h"
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <tuple>
+
+class PrefixCodeGenerator;
+
+class Lz77Encoder : public AbstractEncoder
+{
+public:
+    using Hit = std::tuple<unsigned, unsigned, unsigned char>;
+
+    Lz77Encoder(BitStream* inputStream, BitStream* outputStream);
+
+    bool encode() override;
+
+    bool decode() override;
+
+    const std::vector<Hit>& getHitBuffer() const;
+
+    void setSearchBufferSize(unsigned size);
+
+    void setLookAheadBufferSize(unsigned size);
+
+    void setPrefixCodeGenerator(std::unique_ptr<PrefixCodeGenerator> generator);
+
+    bool hitBufferFull() const;
+
+private:
+
+    bool lookAheadSourceEmpty() const;
+    unsigned char getSearchBufferItem(unsigned index) const;
+
+    unsigned lookAheadForMatchingChars(unsigned searchIndex);
+
+    void lookForMatches(unsigned char searchChar, unsigned& hitLength, unsigned& hitOffset);
+
+    void populateLookaheadBuffer(unsigned size, bool firstPass = false);
+
+    void populateSearchBuffer(const Hit& hit);
+
+    unsigned mMaxHitBufferSize{32000};
+    std::vector<Hit> mHitBuffer;
+
+    unsigned mSearchBufferSize{32000};
+    CircleBuffer<unsigned char> mSearchBuffer;
+
+    unsigned mLookAheadBufferSize{256};
+    int mMaxLookAheadBufferIndex{0};
+    unsigned mMinLengthMatchSize{1};
+
+    CircleBuffer<unsigned char>  mLookaheadBuffer;
+    std::unique_ptr<PrefixCodeGenerator> mCodeGenerator;
+};

src/base/compression/RunLengthEncoder.cpp

@@ -0,0 +1,54 @@
+#include "RunLengthEncoder.h"
+
+std::vector<RunLengthEncoder::Hit> RunLengthEncoder::encode(const std::vector<unsigned char>& input)
+{
+    std::vector<RunLengthEncoder::Hit> ret;
+    if (input.empty())
+    {
+        return ret;
+    }
+
+    char working_char{0};
+    unsigned count = 1;
+    for(unsigned idx=0; idx<input.size(); idx++)
+    {
+        auto c = input[idx];
+        if (idx == 0)
+        {
+            working_char = c;
+            continue;
+        }
+
+        if (c == working_char)
+        {
+            count++;
+        }
+        else
+        {
+            ret.push_back({working_char, count});
+            working_char = c;
+            count = 1;
+        }
+    }
+    ret.push_back({working_char, count});
+    return ret;
+}
+
+std::vector<unsigned char> RunLengthEncoder::decode(const std::vector<RunLengthEncoder::Hit>& input)
+{
+    std::vector<unsigned char> ret;
+    if (input.empty())
+    {
+        return ret;
+    }
+
+    for (const auto& hit : input)
+    {
+        for(unsigned idx=0; idx< hit.second; idx++)
+        {
+            ret.push_back(hit.first);
+        }
+    }
+    return ret;
+}
+

src/base/compression/RunLengthEncoder.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <vector>
+
+class RunLengthEncoder
+{
+public:
+    using Hit = std::pair<unsigned char, unsigned>;
+
+    std::vector<Hit> encode(const std::vector<unsigned char>& input);
+
+    std::vector<unsigned char> decode(const std::vector<Hit>& input);
+
+private:
+};

src/base/compression/StreamCompressor.h

@@ -0,0 +1,6 @@
+#pragma once
+
+class StreamCompressor
+{
+
+};

src/base/compression/ZlibEncoder.cpp

@@ -0,0 +1,170 @@
+#include "ZlibEncoder.h"
+
+#include "ByteUtils.h"
+#include "DeflateEncoder.h"
+#include "FileLogger.h"
+#include "BitStream.h"
+
+#include "Adler32Checksum.h"
+
+#include <math.h>
+#include <iostream>
+#include <sstream>
+
+ZlibEncoder::ZlibEncoder(BitStream* inputStream, BitStream* outputStream)
+: AbstractEncoder(inputStream, outputStream)
+{
+    mChecksumCalculator = std::make_unique<Adler32Checksum>();
+}
+
+ZlibEncoder::~ZlibEncoder()
+{
+
+}
+
+void ZlibEncoder::setWindowSize(unsigned size)
+{
+    mWindowSize = size;
+}
+
+std::string ZlibEncoder::toString(CompressionLevel level) const
+{
+    switch(level)
+    {
+    case CompressionLevel::FASTEST:
+        return "FASTEST";
+    case CompressionLevel::FAST:
+        return "FAST";
+    case CompressionLevel::DEFAULT:
+        return "DEFAULT";
+    case CompressionLevel::MAX_COMPRESSION:
+        return "MAX_COMPRESSION";
+    default:
+        return "UNKNOWN";
+    }
+}
+
+std::string ZlibEncoder::toString(CompressionMethod method) const
+{
+    return method == CompressionMethod::DEFLATE ? "DEFLATE" : "UNKNOWN";
+}
+
+void ZlibEncoder::parseCompressionMethod(unsigned char method)
+{
+    //std::cout << "Got compression input " << static_cast<int>(method) << std::endl;
+    mCompressionMethod = static_cast<CompressionMethod>(ByteUtils::getLowerNBits(method, 4));
+    auto compression_info = ByteUtils::getHigherNBits(method, 4);
+
+    if (mCompressionMethod == CompressionMethod::DEFLATE)
+    {
+        mWindowSize = static_cast<unsigned>(pow(2, compression_info + 8));
+    }
+}
+
+void ZlibEncoder::parseExtraFlags(unsigned char extraFlags, unsigned char compression_byte)
+{
+    //std::cout << "Got flags  " << static_cast<int>(extraFlags) << std::endl;
+
+    auto mod = ((static_cast<unsigned>(compression_byte) << 8) | extraFlags) % 31;
+    if (mod != 0)
+    {
+        //std::cout << "Invalid header. Mod is " << mod << std::endl;
+    }
+
+    mFlagCheck = ByteUtils::getLowerNBits(extraFlags, 5);
+    mUseDictionary = bool(ByteUtils::getBitN(extraFlags, 5));
+    mFlagLevel = static_cast<CompressionLevel>(ByteUtils::getHigherNBits(extraFlags, 2));
+}
+
+std::string ZlibEncoder::getData() const
+{
+    std::stringstream sstream;
+    sstream << "ZlibEncoder data \n";
+    sstream << "Compression method: " << toString(mCompressionMethod) << '\n';
+    sstream << "Window size: " << mWindowSize << '\n';
+    sstream << "Flag check: " << static_cast<int>(mFlagCheck) << '\n';
+    sstream << "Use dictionary: " << mUseDictionary << '\n';
+    sstream << "Flag level: " << toString(mFlagLevel) << '\n';
+    return sstream.str();
+}
+
+bool ZlibEncoder::encode()
+{
+    DeflateEncoder* deflate_encoder{nullptr};
+    if (!mWorkingEncoder)
+    {
+        if (mCompressionMethod == CompressionMethod::DEFLATE)
+        {
+            auto uq_deflate_encoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
+            deflate_encoder = uq_deflate_encoder.get();
+            mWorkingEncoder = std::move(uq_deflate_encoder);
+            mWorkingEncoder->addChecksumCalculator(mChecksumCalculator.get());
+        }
+        else
+        {
+            MLOG_ERROR("Zib requested decoder not recognized: " << static_cast<int>(mCompressionMethod) << " aborting encode");
+            return false;
+        }
+    }
+    deflate_encoder->setCompressionMethod(mDeflateCompressionMethod);
+
+    auto compression_info = static_cast<unsigned char>(log2(mWindowSize) - 8);
+    const unsigned char compression_byte = (compression_info << 4) | static_cast<unsigned char>(mCompressionMethod);
+
+    //std::cout << "ZlibEncoder Writing compression byte " << static_cast<int>(compression_byte) << " with info " << static_cast<int>(compression_info) << std::endl;
+    mOutputStream->writeByte(compression_byte);
+
+    unsigned char flag_byte{0};
+    flag_byte |= (static_cast<unsigned char>(mUseDictionary) << 5);
+    flag_byte |= (static_cast<unsigned char>(mFlagLevel) << 6);
+
+    const auto mod = (unsigned(compression_byte)*256 + flag_byte) % 31;
+    flag_byte += (31 - mod);
+
+    //std::cout << "ZlibEncoder Writing Flag byte " << static_cast<int>(flag_byte) << std::endl;
+    mOutputStream->writeByte(flag_byte);
+
+    if(!mWorkingEncoder->encode())
+    {
+        MLOG_ERROR("Sub-Encoder failed - aborting zlib encode");
+        //return false;
+    }
+
+    const auto checksum = mChecksumCalculator->getChecksum();
+    //std::cout << "ZlibEncoder Writing Adler32 Checksum " << checksum << std::endl;
+    mOutputStream->write(checksum);
+    return true;
+}
+
+bool ZlibEncoder::decode()
+{
+    auto compression_byte = *mInputStream->readNextByte();
+
+    parseCompressionMethod(compression_byte);
+    parseExtraFlags(*mInputStream->readNextByte(), compression_byte);
+
+    if (!mWorkingEncoder)
+    {
+        if (mCompressionMethod == CompressionMethod::DEFLATE)
+        {
+            mWorkingEncoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
+        }
+        else
+        {
+            MLOG_ERROR("Zib requested decoder not recognized: " << static_cast<int>(mCompressionMethod) << " aborting decode");
+            return false;
+        }
+    }
+
+    auto valid = mWorkingEncoder->decode();
+
+    unsigned char byte0 = *mInputStream->readNextByte();
+    unsigned char byte1 = *mInputStream->readNextByte();
+    unsigned char byte2 = *mInputStream->readNextByte();
+    unsigned char byte3 = *mInputStream->readNextByte();
+
+    uint32_t adler32 = (byte0 << 24) | (byte1 << 16) | (byte2 << 8) | byte3;
+    //std::cout << "Got adler 32 checksum " << adler32 << std::endl;
+
+    return valid;
+}

src/base/compression/ZlibEncoder.h

@@ -0,0 +1,59 @@
+#pragma once
+
+#include "AbstractEncoder.h"
+
+#include "DeflateElements.h"
+
+#include <memory>
+#include <vector>
+
+class AbstractChecksumCalculator;
+
+class ZlibEncoder : public AbstractEncoder
+{
+public:
+
+    enum class CompressionMethod : unsigned char
+    {
+        DEFLATE = 8,
+    };
+
+    enum class CompressionLevel : unsigned char
+    {
+        FASTEST,
+        FAST,
+        DEFAULT,
+        MAX_COMPRESSION
+    };
+
+    ZlibEncoder(BitStream* inputStream, BitStream* outputStream);
+    ~ZlibEncoder();
+
+    void setWindowSize(unsigned size);
+    void setDeflateCompressionMethod(Deflate::CompressionMethod method)
+    {
+        mDeflateCompressionMethod = method;
+    }
+
+    bool encode() override;
+    bool decode() override;
+
+    std::string getData() const;
+    std::string toString(CompressionLevel level) const;
+    std::string toString(CompressionMethod method) const;
+
+private:
+    void parseCompressionMethod(unsigned char method);
+    void parseExtraFlags(unsigned char extraFlags, unsigned char compression_byte);
+
+    CompressionMethod mCompressionMethod{CompressionMethod::DEFLATE};
+    Deflate::CompressionMethod mDeflateCompressionMethod{Deflate::CompressionMethod::NONE};
+    unsigned mWindowSize{32768}; // Window size, n in 2^(n+8) bytes
+
+    unsigned char mFlagCheck{0};
+    bool mUseDictionary{false};
+    CompressionLevel mFlagLevel{CompressionLevel::DEFAULT};
+
+    std::unique_ptr<AbstractChecksumCalculator> mChecksumCalculator;
+    std::unique_ptr<AbstractEncoder> mWorkingEncoder;
+};

src/base/compression/deflate/DeflateBlock.cpp

@@ -0,0 +1,157 @@
+#include "DeflateBlock.h"
+
+#include "ByteUtils.h"
+#include "AbstractChecksumCalculator.h"
+
+#include <algorithm>
+#include <iostream>
+#include <sstream>
+
+DeflateBlock::DeflateBlock(BitStream* inputStream, BitStream* outputStream)
+    : mInputStream(inputStream),
+      mOutputStream(outputStream)
+{
+
+}
+
+std::string DeflateBlock::getMetaData() const
+{
+    std::stringstream sstr;
+    sstr << "DeflateBlock Metadata \n";
+
+    sstr << "Final block: " << mInFinalBlock << '\n';
+    sstr << "Compression method: " << Deflate::toString(mCompressionMethod) << '\n';
+    sstr << "Uncompressed block length: " << mUncompressedBlockLength << '\n';
+    return sstr.str();
+}
+
+void DeflateBlock::setIsFinalBlock(bool isFinal)
+{
+    mInFinalBlock = isFinal;
+}
+
+bool DeflateBlock::isFinalBlock() const
+{
+    return mInFinalBlock;
+}
+
+bool DeflateBlock::read()
+{
+    auto working_byte = *mInputStream->readNextByte();
+
+    //std::cout << mInputStream->logNextNBytes(60);
+    //std::cout << "DeflateBlock::read location " << mInputStream->logLocation();
+
+    unsigned char final_block{0};
+    mInputStream->readNextNBits(1, final_block);
+    mInFinalBlock = bool(final_block);
+
+    unsigned char compression_type{0};
+    mInputStream->readNextNBits(2, compression_type);
+    mCompressionMethod = static_cast<Deflate::CompressionMethod>(compression_type);
+
+    if (mCompressionMethod == Deflate::CompressionMethod::NONE)
+    {
+        return readUncompressedStream();
+    }
+    else if(mCompressionMethod == Deflate::CompressionMethod::FIXED_HUFFMAN)
+    {
+        return readFixedHuffmanStream();
+    }
+    else if(mCompressionMethod == Deflate::CompressionMethod::DYNAMIC_HUFFMAN)
+    {
+        return readDynamicHuffmanStream();
+    }
+    return false;
+}
+
+bool DeflateBlock::readUncompressedStream()
+{
+    auto byte0 = *mInputStream->readNextByte();
+    auto byte1 = *mInputStream->readNextByte();
+    mUncompressedBlockLength = (byte0 << 8) | byte1;
+
+    std::cout << "Check block 0: " << ByteUtils::toString(byte0) << std::endl;
+    std::cout << "Check block 1: " << ByteUtils::toString(byte1) << std::endl;
+
+    auto byte2 = *mInputStream->readNextByte();
+    auto byte3 = *mInputStream->readNextByte();
+    uint16_t len_check = (byte2 << 8) | byte3;
+
+    //std::cout << "Check block 2: " << ByteUtils::toString(byte2) << std::endl;
+    //std::cout << "Check block 3: " << ByteUtils::toString(byte3) << std::endl;
+    //if (!(byte0 ==(~byte2) && byte1 ==(~byte3)))
+    //{
+        //std::cout << "Uncompressed block length check failed - aborting." << std::endl;
+        //return false;
+    //}
+    //else
+    //{
+        for(unsigned idx=0; idx<mUncompressedBlockLength;idx++)
+        {
+            mOutputStream->writeByte(*mInputStream->readNextByte());
+        }
+    //}
+    return true;
+}
+
+bool DeflateBlock::readFixedHuffmanStream()
+{
+    //std::cout << "Reading fixed huffman stream" << std::endl;
+    mHuffmanStream = std::make_unique<HuffmanStream>(mInputStream, mOutputStream);
+
+    mHuffmanStream->generateFixedCodeMapping();
+    return mHuffmanStream->decode();
+}
+
+bool DeflateBlock::readDynamicHuffmanStream()
+{
+    mHuffmanStream = std::make_unique<HuffmanStream>(mInputStream, mOutputStream);
+    return mHuffmanStream->decode();
+}
+
+void DeflateBlock::write(uint16_t datalength)
+{
+    mUncompressedBlockLength = datalength;
+
+    unsigned char working_block{0};
+    working_block |= static_cast<unsigned char>(mInFinalBlock);
+    working_block |= (static_cast<unsigned char>(mCompressionMethod) << 1);
+
+    if (mCompressionMethod == Deflate::CompressionMethod::NONE)
+    {
+        writeUncompressedStream(working_block, datalength);
+    }
+    else if (mCompressionMethod == Deflate::CompressionMethod::FIXED_HUFFMAN)
+    {
+        mOutputStream->writeNBits(working_block, 3);
+        while(auto byte = mInputStream->readNextByte())
+        {
+            mOutputStream->writeByte(*byte);
+        }
+
+        if (const auto& remaining_bits = mInputStream->getRemainingBits(); remaining_bits.second > 0)
+        {
+            mOutputStream->writeNBits(remaining_bits.first, remaining_bits.second);
+        }
+    }
+}
+
+void DeflateBlock::writeUncompressedStream(unsigned char working_byte, uint16_t datalength)
+{
+    //std::cout << "Writing compression block header " << ByteUtils::toString(working_byte) << std::endl;
+    mOutputStream->writeByte(working_byte);
+
+    //std::cout << "Writing data length " << mUncompressedBlockLength << " " << ByteUtils::toString(mUncompressedBlockLength) << std::endl;
+    mOutputStream->writeWord(datalength);
+
+    //std::cout << "Writing iverse data length " << ~mUncompressedBlockLength << " " << ByteUtils::toString(~mUncompressedBlockLength) << std::endl;
+    mOutputStream->writeWord(static_cast<uint16_t>(~mUncompressedBlockLength));
+
+    for(unsigned idx=0; idx<mUncompressedBlockLength;idx++)
+    {
+        auto byte = *mInputStream->readNextByte();
+        //std::cout << "Writing next byte " << static_cast<int>(byte) << std::endl;
+        mOutputStream->writeByte(byte);
+    }
+}

src/base/compression/deflate/DeflateBlock.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "DeflateElements.h"
+#include "HuffmanStream.h"
+
+#include "BitStream.h"
+
+#include <memory>
+
+class AbstractChecksumCalculator;
+
+class DeflateBlock
+{
+public:
+    DeflateBlock(BitStream* inputStream, BitStream* outputStream);
+
+    std::string getMetaData() const;
+
+    bool isFinalBlock() const;
+
+    bool read();
+
+    void setIsFinalBlock(bool isFinal);
+    void setCompressionMethod(Deflate::CompressionMethod method)
+    {
+        mCompressionMethod = method;
+    }
+
+    void write(uint16_t datalength);
+
+private:
+    bool readUncompressedStream();
+    bool readFixedHuffmanStream();
+    bool readDynamicHuffmanStream();
+
+    void writeUncompressedStream(unsigned char working_byte, uint16_t datalength);
+
+    BitStream* mInputStream;
+    BitStream* mOutputStream;
+
+    std::unique_ptr<HuffmanStream> mHuffmanStream;
+    uint16_t mUncompressedBlockLength{0};
+
+    bool mInFinalBlock{false};
+    Deflate::CompressionMethod mCompressionMethod{Deflate::CompressionMethod::NONE};
+};

src/base/compression/deflate/DeflateElements.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <string>
+
+namespace Deflate
+{
+    enum class CompressionMethod
+    {
+        NONE,
+        FIXED_HUFFMAN,
+        DYNAMIC_HUFFMAN,
+        ERROR
+    };
+
+    inline std::string toString(CompressionMethod method)
+    {
+        switch (method)
+        {
+        case CompressionMethod::NONE:
+                return "NONE";
+        case CompressionMethod::FIXED_HUFFMAN:
+                return "FIXED_HUFFMAN";
+        case CompressionMethod::DYNAMIC_HUFFMAN:
+                return "DYNAMIC_HUFFMAN";
+        case CompressionMethod::ERROR:
+                return "ERROR";
+        default:
+            return "UNKNOWN";
+        }
+    }
+
+
+}

src/base/compression/deflate/DeflateEncoder.cpp

@@ -0,0 +1,88 @@
+#include "DeflateEncoder.h"
+
+#include "BitStream.h"
+#include "ByteUtils.h"
+#include "DeflateBlock.h"
+#include "BufferBitStream.h"
+
+#include <iostream>
+
+DeflateEncoder::DeflateEncoder(BitStream* inputStream, BitStream* outputStream)
+    : AbstractEncoder(inputStream, outputStream)
+{
+
+}
+
+DeflateEncoder::~DeflateEncoder()
+{
+
+}
+
+bool DeflateEncoder::encode()
+{
+    uint16_t count = 0;
+    BufferBitStream stream;
+    std::unique_ptr<DeflateBlock> working_block = std::make_unique<DeflateBlock>(&stream, mOutputStream);
+    working_block->setCompressionMethod(mCompressionMethod);
+
+    if (mChecksumCalculators.size() > 0)
+    {
+       //std::cout << "Setting checksum calculator " << std::endl;
+       mOutputStream->setChecksumCalculator(mChecksumCalculators[0]);
+    }
+
+    while(true)
+    {
+        if (count == mMaxBlockSize)
+        {
+            //std::cout << working_block->getMetaData();
+            working_block->write(count);
+
+            working_block = std::make_unique<DeflateBlock>(&stream, mOutputStream);
+            working_block->setCompressionMethod(mCompressionMethod);
+            stream.reset();
+        }
+
+        if (auto byte = mInputStream->readNextByte())
+        {
+            //std::cout << "Adding byte " << ByteUtils::toString(*byte) << " to deflate block input" << std::endl;
+            stream.writeByte(*byte);
+        }
+        else
+        {
+            if (const auto& remaining_bits = mInputStream->getRemainingBits(); remaining_bits.second > 0)
+            {
+                stream.writeNBits(remaining_bits.first, remaining_bits.second);
+            }
+
+            stream.resetOffsets();
+            working_block->setIsFinalBlock(true);
+
+            //std::cout << working_block->getMetaData();
+            working_block->write(count);
+            break;
+        }
+        count++;
+    }
+    mOutputStream->flushRemainingBits();
+    mOutputStream->clearChecksumCalculator();
+    return true;
+}
+
+bool DeflateEncoder::decode()
+{
+    auto working_block = std::make_unique<DeflateBlock>(mInputStream, mOutputStream);
+    working_block->read();
+
+    //std::cout << working_block->getMetaData() << std::endl;
+
+    DeflateBlock* raw_block = working_block.get();
+
+    while(!raw_block->isFinalBlock())
+    {
+        break;
+    }
+
+    return true;
+}
+

src/base/compression/deflate/DeflateEncoder.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "AbstractEncoder.h"
+#include "DeflateElements.h"
+
+#include <vector>
+#include <memory>
+
+class DeflateBlock;
+
+class DeflateEncoder : public AbstractEncoder
+{
+public:
+    DeflateEncoder(BitStream* inputStream, BitStream* outputStream);
+
+    ~DeflateEncoder();
+
+    bool encode() override;
+
+    bool decode() override;
+
+    void setCompressionMethod(Deflate::CompressionMethod method)
+    {
+        mCompressionMethod = method;
+    }
+
+private:
+    uint16_t mMaxBlockSize{65535};
+    Deflate::CompressionMethod mCompressionMethod{Deflate::CompressionMethod::NONE};
+    std::unique_ptr<DeflateBlock > mLastBlock;
+};
+
+

src/base/compression/huffman/HuffmanCodeLengthTable.cpp

@@ -0,0 +1,253 @@
+#include "HuffmanCodeLengthTable.h"
+
+#include "ByteUtils.h"
+#include "RunLengthEncoder.h"
+#include "BitStream.h"
+
+#include <algorithm>
+#include <sstream>
+#include <iostream>
+
+void HuffmanCodeLengthTable::buildCompressedLengthSequence()
+{
+    RunLengthEncoder rl_encoder;
+    auto rle_encoded = rl_encoder.encode(mInputLengthSequence);
+
+    for (const auto& entry : rle_encoded)
+    {
+        //std::cout << "Got rle " << static_cast<int>(entry.first) << " | " << entry.second << std::endl;
+    }
+    mCompressedLengthSequence.clear();
+
+    for (const auto& entry : rle_encoded)
+    {
+        const auto length = entry.first;
+        const auto count = entry.second;
+        if (count < 3)
+        {
+            for(std::size_t idx=0; idx<count; idx++)
+            {
+                mCompressedLengthSequence.push_back({length, 0});
+            }
+        }
+        else if (length == 0)
+        {
+            std::size_t num_big = count / 138;
+            for(std::size_t idx=0; idx<num_big; idx++)
+            {
+                mCompressedLengthSequence.push_back({18, 127});
+            }
+            auto remainder_big = count % 138;
+            if (remainder_big > 10)
+            {
+                mCompressedLengthSequence.push_back({18, remainder_big-11});
+            }
+            else if(remainder_big > 2)
+            {
+                mCompressedLengthSequence.push_back({17, remainder_big-3});
+            }
+            else
+            {
+                for(std::size_t idx=0; idx<remainder_big; idx++)
+                {
+                    mCompressedLengthSequence.push_back({0, 0});
+                }
+            }
+        }
+        else
+        {
+            mCompressedLengthSequence.push_back({length, 0});
+            auto num_blocks_of_six = (count-1)/6;
+            for(std::size_t idx=0; idx<num_blocks_of_six; idx++)
+            {
+                mCompressedLengthSequence.push_back({16, 3});
+            }
+            auto remaining_counts = (count-1) % 6;
+            if (remaining_counts >= 3)
+            {
+                mCompressedLengthSequence.push_back({16, remaining_counts - 3});
+            }
+            else
+            {
+                for(std::size_t idx=0; idx<remaining_counts; idx++)
+                {
+                    mCompressedLengthSequence.push_back({length, 0});
+                }
+            }
+        }
+    }
+
+    mCompressedLengthCounts = std::vector<std::size_t>(19, 0);
+    for (const auto& entry : mCompressedLengthSequence)
+    {
+        mCompressedLengthCounts[entry.first]++;
+    }
+}
+
+const std::vector<HuffmanCodeLengthTable::CompressedSequenceEntry>& HuffmanCodeLengthTable::getCompressedLengthSequence() const
+{
+    return mCompressedLengthSequence;
+}
+
+const std::vector<std::size_t> HuffmanCodeLengthTable::getCompressedLengthCounts() const
+{
+    return mCompressedLengthCounts;
+}
+
+std::optional<PrefixCode> HuffmanCodeLengthTable::getCodeForSymbol(unsigned symbol) const
+{
+    return mTree.getCode(symbol);
+}
+
+bool HuffmanCodeLengthTable::readNextSymbol(unsigned& result, BitStream* stream)
+{
+    if (getNumCodeLengths() == 0)
+    {
+        return false;
+    }
+
+    std::size_t working_index{0};
+    auto length = getCodeLength(working_index);
+    auto delta = length;
+
+    bool found{false};
+    unsigned char buffer{0};
+    uint32_t working_bits{0};
+    unsigned working_symbol{0};
+
+    while(!found)
+    {
+        auto valid = stream->readNextNBits(delta, buffer);
+        //std::cout << "Got buffer " << ByteUtils::toString(buffer) << std::endl;;
+
+        unsigned hold = buffer;
+        working_bits = working_bits | (hold << (length - delta));
+        //std::cout << "Read " << delta << " bits with length " << length << " and value " << ByteUtils::toString(working_bits) << std::endl;
+
+        if (const auto symbol = findMatch(working_index, working_bits))
+        {
+            found = true;
+            working_symbol = *symbol;
+        }
+        else
+        {
+            working_index++;
+            if (working_index >= getNumCodeLengths())
+            {
+                break;
+            }
+
+            auto new_length = getCodeLength(working_index);
+            delta = new_length - length;
+            length = new_length;
+        }
+    }
+
+    if (found)
+    {
+        result = working_symbol;
+        // std::cout << "Found symbol " << working_symbol << " with bits " << ByteUtils::toString(working_bits) << std::endl;
+        // std::cout << "At Byte offset " << stream->getCurrentByteOffset() << " and bit offset " << stream->getCurrentBitOffset() << std::endl;
+        return true;
+    }
+    else
+    {
+        //std::cout << "SYMBOL NOT FOUND " << " with bits " << ByteUtils::toString(working_bits) << " and index " << working_index << std::endl;
+        return false;
+    }
+}
+
+void HuffmanCodeLengthTable::buildPrefixCodes()
+{
+    if(mInputLengthSequence.empty())
+    {
+        return;
+    }
+
+    unsigned char max_length = *std::max_element(mInputLengthSequence.begin(), mInputLengthSequence.end());
+    std::vector<unsigned> counts(max_length+1, 0);
+    for (const auto length : mInputLengthSequence)
+    {
+        counts[length]++;
+    }
+    counts[0] = 0;
+
+    uint32_t code{0};
+    std::vector<uint32_t> next_code(max_length + 1, 0);
+    for (unsigned bits = 1; bits <= max_length; bits++)
+    {
+        code = (code + counts[bits-1]) << 1;
+        //std::cout << "Start code for bit " << bits << " is " << ByteUtils::toString(code) << " | dec " << code << " count " << counts[bits-1] << std::endl;
+        next_code[bits] = code;
+    }
+
+    for(std::size_t idx=0; idx<mInputLengthSequence.size(); idx++)
+    {
+        if (const auto length = mInputLengthSequence[idx]; length != 0)
+        {
+            const auto code = next_code[length];
+            next_code[length]++;
+            auto prefix_code = PrefixCode(code, length);
+            mTree.addCodeLengthEntry(length, {PrefixCode(code, length), static_cast<unsigned>(idx)});
+            mCodes.push_back(prefix_code);
+        }
+    }
+    mTree.sortTable();
+    //std::cout << dumpPrefixCodes();
+}
+
+const PrefixCode& HuffmanCodeLengthTable::getCode(std::size_t index) const
+{
+    return mCodes[index];
+}
+
+std::string HuffmanCodeLengthTable::dumpPrefixCodes() const
+{
+    return mTree.dump();
+}
+
+std::size_t HuffmanCodeLengthTable::mapToDeflateIndex(std::size_t index) const
+{
+    if (index>= DEFLATE_PERMUTATION_SIZE)
+    {
+        return 0;
+    }
+    else
+    {
+        return DEFLATE_PERMUTATION[index];
+    }
+}
+
+std::size_t HuffmanCodeLengthTable::getNumCodeLengths() const
+{
+    return mTree.getNumCodeLengths();
+}
+
+std::optional<HuffmanTree::Symbol> HuffmanCodeLengthTable::findMatch(std::size_t treeIndex, uint32_t code) const
+{
+    return mTree.findMatch(treeIndex, code);
+}
+
+unsigned HuffmanCodeLengthTable::getCodeLength(std::size_t index) const
+{
+    return mTree.getCodeLength(index);
+}
+
+void HuffmanCodeLengthTable::setInputLengthSequence(const std::vector<unsigned char>& sequence, bool targetDeflate)
+{
+    mTargetDeflate = targetDeflate;
+
+    if (targetDeflate)
+    {
+        mInputLengthSequence = std::vector<unsigned char>(DEFLATE_PERMUTATION_SIZE, 0);
+        for(std::size_t idx=0; idx<sequence.size(); idx++)
+        {
+            mInputLengthSequence[mapToDeflateIndex(idx)] = sequence[idx];
+            //std::cout << "Got code length for " << mapToDeflateIndex(idx) << " of " << static_cast<unsigned>(sequence[idx]) << std::endl;
+        }
+    }
+    else
+    {
+        mInputLengthSequence = sequence;
+    }
+}

src/base/compression/huffman/HuffmanCodeLengthTable.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "HuffmanTree.h"
+
+#include <vector>
+#include <string>
+#include <optional>
+
+class BitStream;
+
+class HuffmanCodeLengthTable
+{
+public:
+    void buildPrefixCodes();
+
+    void buildCompressedLengthSequence();
+
+    std::string dumpPrefixCodes() const;
+
+    std::optional<HuffmanTree::Symbol> findMatch(std::size_t treeIndex, uint32_t code) const;
+
+    const HuffmanTree& getTree() const;
+
+    const PrefixCode& getCode(std::size_t index) const;
+
+    std::optional<PrefixCode> getCodeForSymbol(unsigned symbol) const;
+
+    using CompressedSequenceEntry = std::pair<unsigned, unsigned>;
+    const std::vector<CompressedSequenceEntry>& getCompressedLengthSequence() const;
+
+    const std::vector<std::size_t> getCompressedLengthCounts() const;
+
+    std::size_t getNumCodeLengths() const;
+
+    unsigned getCodeLength(std::size_t treeIndex) const;
+
+    std::size_t mapToDeflateIndex(std::size_t index) const;
+
+    void setInputLengthSequence(const std::vector<unsigned char>& sequence, bool targetDeflate = true);
+
+    bool readNextSymbol(unsigned& buffer, BitStream* stream);
+
+private:
+
+    HuffmanTree mTree;
+    bool mTargetDeflate{true};
+
+    std::vector<unsigned char> mInputLengthSequence;
+    std::vector<PrefixCode> mCodes;
+
+    std::vector<CompressedSequenceEntry> mCompressedLengthSequence;
+    std::vector<std::size_t> mCompressedLengthCounts;
+
+    static constexpr unsigned DEFLATE_PERMUTATION_SIZE{19};
+    static constexpr unsigned DEFLATE_PERMUTATION[DEFLATE_PERMUTATION_SIZE]{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
+};

src/base/compression/huffman/HuffmanEncoder.cpp

@@ -0,0 +1,173 @@
+#include "HuffmanEncoder.h"
+
+#include "RawTree.h"
+
+#include "HuffmanFixedCodes.h"
+
+#include <unordered_map>
+#include <queue>
+#include <tuple>
+#include <iostream>
+
+void HuffmanEncoder::dumpNode(RawNode<CountPair>* node, unsigned depth) const
+{
+    if (!node)
+    {
+        return;
+    }
+
+    auto data = node->getData();
+
+    std::string prefix(depth, '_');
+
+    if (node->isLeaf())
+    {
+        //std::cout << prefix << "Leaf with value: " << data.first << " and sum " << data.second << std::endl;
+    }
+    else
+    {
+        //std::cout << prefix << "Intermediate with sum " << data.second << std::endl;
+        //std::cout << prefix << "Doing Left.." << std::endl;
+        dumpNode(node->getLeftChild(), depth+1);
+
+        //std::cout << prefix << "Doing Right.." << std::endl;
+        dumpNode(node->getRightChild(), depth+1);
+
+        //std::cout << prefix << "*****" << std::endl;
+    }
+}
+
+void HuffmanEncoder::dumpTree(const RawTree<CountPair>& tree) const
+{
+    dumpNode(tree.getRootNode(), 0);
+}
+
+void HuffmanEncoder::encode(const std::vector<unsigned>& counts)
+{
+    auto cmp = [](RawNode<CountPair>* left, RawNode<CountPair>* right)
+        {
+            return left->getData().second > right->getData().second;
+        };
+
+    std::priority_queue<RawNode<CountPair>*, std::vector<RawNode<CountPair>* >, decltype(cmp)> q(cmp);
+    unsigned offset{0};
+    for (auto count : counts)
+    {
+        if (count > 0)
+        {
+            q.push(new RawNode<CountPair>({offset, count}));
+        }
+        offset++;
+    }
+
+    while(q.size() > 1)
+    {
+        auto node0 = q.top();
+        q.pop();
+
+        auto node1 = q.top();
+        q.pop();
+
+        const auto sum = node0->getData().second + node1->getData().second;
+        auto new_node = new RawNode<CountPair>(CountPair{0, sum});
+
+        new_node->addChild(node0);
+        new_node->addChild(node1);
+        q.push(new_node);
+    }
+
+    auto root = q.top();
+    q.pop();
+
+    RawTree<CountPair> tree;
+    tree.addRootNode(root);
+
+    //using TableEntry = std::tuple<>
+
+    //dumpTree(tree);
+
+    //std::cout << "********" << std::endl;
+}
+
+void HuffmanEncoder::encode(const std::unordered_map<unsigned char, unsigned>& counts)
+{
+    std::vector<unsigned> just_counts;
+    for (const auto& data: counts)
+    {
+        mSymbolMapping.push_back(data.first);
+        just_counts.push_back(data.second);
+    }
+
+    encode(just_counts);
+}
+
+void HuffmanEncoder::setUseFixedCode(bool useFixed)
+{
+    mUseFixedCode = useFixed;
+}
+
+uint32_t HuffmanEncoder::getLengthValue(unsigned length)
+{
+    return 0;
+}
+
+std::optional<PrefixCode> HuffmanEncoder::getLiteralValue(unsigned char value) const
+{
+    return mLiteralLengthTable.getCodeForSymbol(value);
+}
+
+std::optional<PrefixCode> HuffmanEncoder::getLengthValue(unsigned length) const
+{
+    return mLiteralLengthTable.getCodeForSymbol(length);
+}
+
+std::optional<PrefixCode> HuffmanEncoder::getDistanceValue(unsigned distance) const
+{
+    return mDistanceTable.getCodeForSymbol(distance);
+}
+
+std::optional<PrefixCode> HuffmanEncoder::getEndOfStreamValue() const
+{
+    return mLiteralLengthTable.getCodeForSymbol(256);
+}
+
+void HuffmanEncoder::initializeTrees(const std::vector<Hit>& hits)
+{
+    initializeLiteralLengthTable(hits);
+}
+
+void HuffmanEncoder::initializeLiteralLengthTable(const std::vector<Hit>& hits)
+{
+    if(mUseFixedCode)
+    {
+        mLiteralLengthTable.setInputLengthSequence(HuffmanFixedCodes::getDeflateFixedHuffmanCodes(), false);
+        mLiteralLengthTable.buildPrefixCodes();
+        return;
+    }
+
+    std::vector<unsigned> counts(285, 0);
+    counts[256] = 1;
+    for (const auto& hit : hits)
+    {
+        const auto& [length, distance, next_char] = hit;
+        if (length > 0 )
+        {
+            const auto& [code, extra_bits, num_extra_bits] = HuffmanFixedCodes::getCodeForLength(length);
+            counts[code]++;
+        }
+        else
+        {
+            counts[next_char]++;
+        }
+    }
+
+    for(unsigned idx=0; idx<counts.size(); idx++)
+    {
+        if (counts[idx]>0)
+        {
+            //std::cout << "Count for " << idx << " is " << counts[idx] << std::endl;
+        }
+    }
+
+    encode(counts);
+}

src/base/compression/huffman/HuffmanEncoder.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "RawTree.h"
+
+#include "HuffmanCodeLengthTable.h"
+#include "HuffmanFixedCodes.h"
+
+#include <vector>
+#include <tuple>
+#include <unordered_map>
+
+class PrefixCodeGenerator
+{
+public:
+    virtual ~PrefixCodeGenerator() = default;
+    virtual std::optional<PrefixCode> getLiteralValue(unsigned char symbol) const = 0;
+    virtual std::optional<PrefixCode> getLengthValue(unsigned length) const = 0;
+    virtual std::optional<PrefixCode> getDistanceValue(unsigned distance) const = 0;
+
+    virtual std::optional<PrefixCode> getEndOfStreamValue() const = 0;
+};
+
+class HuffmanEncoder : public PrefixCodeGenerator
+{
+    using CountPair = std::pair<unsigned, unsigned>;
+    using Hit = std::tuple<unsigned, unsigned, unsigned char>;
+
+public:
+    void encode(const std::vector<unsigned>& counts);
+    void encode(const std::unordered_map<unsigned char, unsigned>& counts);
+
+    uint32_t getLengthValue(unsigned length);
+
+    std::optional<PrefixCode> getLiteralValue(unsigned char symbol) const override;
+
+    std::optional<PrefixCode> getLengthValue(unsigned length) const override;
+
+    std::optional<PrefixCode> getDistanceValue(unsigned distance) const override;
+
+    std::optional<PrefixCode> getEndOfStreamValue() const override;
+
+    void initializeTrees(const std::vector<Hit>& hits);
+
+    void setUseFixedCode(bool useFixed);
+private:
+    void initializeLiteralLengthTable(const std::vector<Hit>& hits);
+
+    void dumpTree(const RawTree<CountPair>& tree) const;
+    void dumpNode(RawNode<CountPair>* node, unsigned depth) const;
+
+    bool mUseFixedCode{false};
+    bool mTableIsInitialized{false};
+
+    std::vector<unsigned char> mSymbolMapping;
+    HuffmanCodeLengthTable mLiteralLengthTable;
+    HuffmanCodeLengthTable mDistanceTable;
+};

src/base/compression/huffman/HuffmanFixedCodes.h

@@ -0,0 +1,145 @@
+#pragma once
+
+#include <vector>
+#include <tuple>
+
+namespace HuffmanFixedCodes
+{
+    inline std::vector<unsigned char> getDeflateFixedHuffmanCodes()
+    {
+        std::vector<std::pair<unsigned, unsigned char> > mappings {{144, 8}, {112, 9}, {24, 7}, {8 ,8}};
+        std::vector<unsigned char> sequence;
+        for(const auto& entry : mappings)
+        {
+            for(unsigned idx=0;idx<entry.first;idx++)
+            {
+                sequence.push_back(entry.second);
+            }
+        }
+        return sequence;
+    }
+
+    inline std::tuple<unsigned, unsigned char, unsigned char> getCodeForLength(unsigned length)
+    {
+        if (length <= 10)
+        {
+            return {length - 3, 0, 0};
+        }
+
+        unsigned base = 2;
+        unsigned last_offset = 10;
+        for (unsigned n = 1; n < 5; n++)
+        {
+            const auto diff = length - last_offset;
+            if (diff <= 4*base)
+            {
+                auto extra = diff/base + diff % base;
+                return {last_offset + diff/base, extra, n};
+            }
+
+            last_offset += 4*n;
+            base = base*2;
+        }
+        return {258, 0, 0};
+    }
+
+    /*
+    inline std::pair<unsigned, unsigned char> getCodeForLength(unsigned length)
+    {
+        if (length <= 10)
+        {
+            return {257 + length - 3, 0};
+        }
+        else if(length <= 18)
+        {
+            auto offset = length - 10;
+            auto extra = offset/2 + offset % 2;
+            return {265 + offset/2, extra};
+        }
+        else if(length <= 34)
+        {
+            auto offset = length - 19;
+            auto extra = offset/4 + offset % 4;
+            return {269 + offset/4, extra};
+        }
+        else if(length <= 66)
+        {
+            auto offset = length - 35;
+            auto extra = offset/8 + offset % 8;
+            return {273 + offset/8, extra};
+        }
+        else if(length <= 114)
+        {
+            auto offset = length - 67;
+            auto extra = offset/16 + offset % 16;
+            return {277 + offset/16, extra};
+        }
+        else if(length <= 257)
+        {
+            auto offset = length - 115;
+            auto extra = offset/32 + offset % 32;
+            return {281 + offset/32, extra};
+        }
+        else
+        {
+            return {258, 0};
+        }
+    }
+
+    inline unsigned getLengthForCode(unsigned symbol, unsigned extra)
+    {
+        if (symbol <= 264)
+        {
+            return 3 + symbol - 257;
+        }
+        else if (symbol <= 268)
+        {
+            return 11 + 2*(symbol - 265) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 272)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(2, extra);
+
+            auto length = 19 + 4*(symbol - 269) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 276)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(3, extra);
+
+            auto length = 35 + 8*(symbol - 273) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 280)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(4, extra);
+
+            auto length = 67 + 16*(symbol - 277) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 284)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(5, extra);
+
+            auto length = 131 + 32*(symbol - 281) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol == 285)
+        {
+            auto length = 258;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+    }
+    */
+}

src/base/compression/huffman/HuffmanStream.cpp

@@ -0,0 +1,346 @@
+#include "HuffmanStream.h"
+
+#include "ByteUtils.h"
+#include "HuffmanFixedCodes.h"
+
+#include <iostream>
+#include <algorithm>
+#include <unordered_map>
+#include <sstream>
+
+std::vector<unsigned> DISTANCE_OFFSETS
+{
+    5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193,
+    258, 385, 513, 769, 1025, 1537, 2049, 3073, 4097,
+    6145, 8193, 12289, 16385, 24577
+};
+
+HuffmanStream::HuffmanStream(BitStream* inputStream, BitStream* outputStream)
+    : mInputStream(inputStream),
+      mOutputStream(outputStream)
+{
+
+}
+
+void HuffmanStream::generateFixedCodeMapping()
+{
+    mUsingFixedCodes = true;
+    mCodeLengthTable.setInputLengthSequence(HuffmanFixedCodes::getDeflateFixedHuffmanCodes(), false);
+    mCodeLengthTable.buildPrefixCodes();
+}
+
+bool HuffmanStream::readNextCodeLengthSymbol(unsigned& buffer)
+{
+    return mCodeLengthTable.readNextSymbol(buffer, mInputStream);
+}
+
+bool HuffmanStream::readNextLiteralSymbol(unsigned& buffer)
+{
+    return mLiteralTable.readNextSymbol(buffer, mInputStream);
+}
+
+bool HuffmanStream::readNextDistanceSymbol(unsigned& buffer)
+{
+    unsigned base_symbol{0};
+    unsigned char extra_bits{0};
+    const auto valid = mDistanceTable.readNextSymbol(base_symbol, mInputStream);
+    if (!valid)
+    {
+        return false;
+    }
+
+    //std::cout << "Got distance base symbol " << base_symbol << std::endl;
+
+    if (base_symbol <= 3)
+    {
+        buffer = 1 + base_symbol;
+    }
+    else
+    {
+        const auto num_extra_bits = (base_symbol - 3 - 1)/2 + 1;
+        unsigned extra_sum{0};
+        if (num_extra_bits > 8)
+        {
+            auto byte_val = *mInputStream->readNextByte();
+            mInputStream->readNextNBits(num_extra_bits-8, extra_bits);
+            extra_sum = extra_bits;
+            extra_sum = extra_sum << (num_extra_bits - 8);
+            extra_sum |= byte_val;
+        }
+        else
+        {
+            mInputStream->readNextNBits(num_extra_bits, extra_bits);
+            extra_sum = extra_bits;
+        }
+        buffer = DISTANCE_OFFSETS[base_symbol - 4] + extra_sum;
+    }
+    return true;
+}
+
+void HuffmanStream::addValue(unsigned value, unsigned& count, unsigned& lastValue, std::vector<unsigned char>& literals, unsigned numLiterals, std::vector<unsigned char>& distances)
+{
+    if (count < mNumLiterals)
+    {
+        literals[count] = value;
+    }
+    else
+    {
+        distances[count - mNumLiterals] = value;
+    }
+    lastValue = value;
+    count++;
+}
+
+void HuffmanStream::readCodeLengths()
+{
+    std::vector<unsigned char> literal_lengths(288, 0);
+    std::vector<unsigned char> distance_lengths(32, 0);
+    unsigned symbol{0};
+    unsigned count{0};
+
+    unsigned last_value{0};
+
+    while(count < mNumLiterals + mNumDistances)
+    {
+        bool valid = readNextCodeLengthSymbol(symbol);
+
+        if (!valid)
+        {
+            //std::cout << "Hit unknown symbol - bailing out" << std::endl;
+            break;
+        }
+
+        if (symbol < 16)
+        {
+            addValue(symbol, count, last_value, literal_lengths, mNumLiterals, distance_lengths);
+        }
+        else if(symbol == 16)
+        {
+            unsigned char num_reps{0};
+            mInputStream->readNextNBits(2, num_reps);
+
+            //std::cout << "Got val 16 doing " << 3 + num_reps << std::endl;
+            for(unsigned char idx=0; idx< 3 + num_reps; idx++)
+            {
+                addValue(last_value, count, last_value, literal_lengths, mNumLiterals, distance_lengths);
+            }
+        }
+        else if(symbol == 17)
+        {
+            unsigned char num_reps{0};
+            mInputStream->readNextNBits(3, num_reps);
+
+            //std::cout << "Got val 17 doing " << 3 + num_reps << std::endl;
+            for(unsigned char idx=0; idx< 3 + num_reps; idx++)
+            {
+                addValue(0, count, last_value, literal_lengths, mNumLiterals, distance_lengths);
+            }
+        }
+        else if(symbol == 18)
+        {
+            unsigned char num_reps{0};
+            mInputStream->readNextNBits(7, num_reps);
+
+            //std::cout << "Got val 18 doing " << 11 + num_reps << std::endl;
+            for(unsigned idx=0; idx< 11 + unsigned(num_reps); idx++)
+            {
+                addValue(0, count, last_value, literal_lengths, mNumLiterals, distance_lengths);
+            }
+        }
+    }
+
+    //std::cout << "Got final literal length sequence " << std::endl;
+    for(unsigned idx=0; idx<literal_lengths.size(); idx++)
+    {
+        //std::cout << static_cast<int>(literal_lengths[idx]) << "," ;
+    }
+    //std::cout << std::endl;
+
+    //std::cout << "Got final distance length sequence " << std::endl;
+    for(unsigned idx=0; idx<distance_lengths.size(); idx++)
+    {
+        //std::cout << static_cast<int>(distance_lengths[idx]) << "," ;
+    }
+    //std::cout << std::endl;
+
+    mLiteralTable.setInputLengthSequence(literal_lengths, false);
+    mLiteralTable.buildPrefixCodes();
+
+    mDistanceTable.setInputLengthSequence(distance_lengths, false);
+    mDistanceTable.buildPrefixCodes();
+}
+
+void HuffmanStream::copyFromBuffer(unsigned length, unsigned distance)
+{
+    std::size_t offset = mBuffer.size() - 1 - distance;
+    for(unsigned idx=0; idx<length; idx++)
+    {
+        auto symbol = mBuffer[offset + idx];
+
+        mOutputStream->writeByte(symbol);
+        mBuffer.push_back(symbol);
+    }
+}
+
+void HuffmanStream::readSymbols()
+{
+    bool hit_end_stream{false};
+    unsigned symbol{0};
+    unsigned distance{0};
+    while(!hit_end_stream)
+    {
+        const auto valid = readNextLiteralSymbol(symbol);
+        if (!valid)
+        {
+            //std::cout << "Hit unknown symbol - bailing out" << std::endl;
+            break;
+        }
+
+        //std::cout << "Got symbol " << symbol << std::endl;
+
+        if(symbol <= 255)
+        {
+            mOutputStream->writeByte(symbol);
+            mBuffer.push_back(symbol);
+        }
+        else if(symbol == 256)
+        {
+            hit_end_stream = true;
+            break;
+        }
+        else if (symbol <= 264)
+        {
+            auto length = 3 + symbol - 257;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 268)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(1, extra);
+
+            auto length = 11 + 2*(symbol - 265) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 272)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(2, extra);
+
+            auto length = 19 + 4*(symbol - 269) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 276)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(3, extra);
+
+            auto length = 35 + 8*(symbol - 273) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 280)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(4, extra);
+
+            auto length = 67 + 16*(symbol - 277) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol <= 284)
+        {
+            unsigned char extra{0};
+            mInputStream->readNextNBits(5, extra);
+
+            auto length = 131 + 32*(symbol - 281) + extra;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+        else if (symbol == 285)
+        {
+            auto length = 258;
+            const auto valid_dist = readNextDistanceSymbol(distance);
+            copyFromBuffer(length, distance);
+        }
+    }
+
+    if (hit_end_stream)
+    {
+        //std::cout << "Found end of stream ok" << std::endl;
+    }
+}
+
+bool HuffmanStream::decode()
+{
+    if (!mUsingFixedCodes)
+    {
+        readCodingsTable();
+
+        readSymbols();
+
+        //std::cout << "Got final buffer size " << mBuffer.size() << std::endl;
+        for(unsigned idx=0; idx< 100; idx++)
+        {
+            //std::cout << idx << " |  " << mBuffer[idx] << std::endl;
+        }
+    }
+    else
+    {
+        bool found_end_seq{false};
+        unsigned symbol{0};
+        while(!found_end_seq)
+        {
+            bool valid = readNextCodeLengthSymbol(symbol);
+
+            if (!valid)
+            {
+                //std::cout << "Hit unknown symbol - bailing out" << std::endl;
+                break;
+            }
+
+            if (symbol == 256)
+            {
+                found_end_seq = true;
+                break;
+            }
+        }
+    }
+
+    return false;
+}
+
+void HuffmanStream::readCodingsTable()
+{
+    unsigned char h_lit{0};
+    mInputStream->readNextNBits(5, h_lit);
+    mNumLiterals = h_lit + 257;
+    //std::cout << "Got HLIT " << mNumLiterals << std::endl;
+
+    unsigned char h_dist{0};
+    mInputStream->readNextNBits(5, h_dist);
+    mNumDistances = h_dist + 1;
+    //std::cout << "Got HDIST " << mNumDistances << std::endl;
+
+    unsigned char h_clen{0};
+    mInputStream->readNextNBits(4, h_clen);
+
+    unsigned num_code_lengths = h_clen + 4;
+    //std::cout << "Got HCLEN " << num_code_lengths << std::endl;
+
+    auto sequence = std::vector<unsigned char>(num_code_lengths, 0);
+    unsigned char buffer{0};
+    for(unsigned idx = 0; idx< num_code_lengths; idx++)
+    {
+        mInputStream->readNextNBits(3, buffer);
+        //std::cout << "Got coding table value " << idx << " | " << static_cast<int>(buffer) << " | " << ByteUtils::toString(buffer) << std::endl;
+        sequence[idx] = buffer;
+    }
+
+    mCodeLengthTable.setInputLengthSequence(sequence, true);
+    mCodeLengthTable.buildPrefixCodes();
+
+    readCodeLengths();
+}

src/base/compression/huffman/HuffmanStream.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include "BitStream.h"
+#include "HuffmanCodeLengthTable.h"
+
+#include <vector>
+#include <string>
+
+
+class HuffmanStream
+{
+
+public:
+    HuffmanStream(BitStream* inputStream, BitStream* outputStream);
+
+    bool decode();
+
+    void generateFixedCodeMapping();
+
+    void setCodeLengthAlphabetLengths(const std::vector<unsigned char>& lengths);
+
+private:
+    void readCodingsTable();
+
+    void readCodeLengths();
+
+    void readSymbols();
+
+    void copyFromBuffer(unsigned length, unsigned distance);
+
+    bool readNextLiteralSymbol(unsigned& buffer);
+
+    bool readNextDistanceSymbol(unsigned& buffer);
+
+    bool readNextCodeLengthSymbol(unsigned& buffer);
+
+    void addValue(unsigned value, unsigned& count, unsigned& lastValue, std::vector<unsigned char>& literals, unsigned numLiterals, std::vector<unsigned char>& distances);
+
+    BitStream* mInputStream;
+    BitStream* mOutputStream;
+
+    std::vector<unsigned> mBuffer;
+
+    unsigned mNumLiterals{0}; // HLIT + 257
+    unsigned mNumDistances{0}; // HDIST + 1
+
+    bool mUsingFixedCodes{false};
+    HuffmanCodeLengthTable mCodeLengthTable;
+    HuffmanCodeLengthTable mLiteralTable;
+    HuffmanCodeLengthTable mDistanceTable;
+};

src/base/compression/huffman/HuffmanTree.cpp

@@ -0,0 +1,121 @@
+#include "HuffmanTree.h"
+
+#include "ByteUtils.h"
+
+#include <sstream>
+#include <algorithm>
+#include <iostream>
+
+PrefixCode::PrefixCode(uint32_t data, unsigned length)
+    : mLength(length)
+{
+    mData = ByteUtils::mirror(data, length);
+}
+
+bool PrefixCode::matches(unsigned length, uint32_t code) const
+{
+    return (mLength == length) && (mData == code);
+}
+
+std::string PrefixCode::toString(bool bitsAsRightToLeft) const
+{
+    if (bitsAsRightToLeft)
+    {
+        if (mLength <=8 )
+        {
+            return ByteUtils::toString(mData).substr(8 - mLength, mLength);
+        }
+        else
+        {
+            return ByteUtils::toString(mData, mLength);
+        }
+    }
+    else
+    {
+        if (mLength <=8 )
+        {
+            return ByteUtils::toString(ByteUtils::mirror(mData, mLength)).substr(0, mLength);
+        }
+        else
+        {
+            return ByteUtils::toString(mData, mLength);
+        }
+    }
+}
+
+void HuffmanTree::addCodeLengthEntry(unsigned length, const CodeSymbolPair& data)
+{
+    bool found{false};
+    for (auto& entry : mTable)
+    {
+        if (entry.first == length)
+        {
+            entry.second.push_back(data);
+            found = true;
+            break;
+        }
+    }
+
+    if (!found)
+    {
+        mTable.push_back({length, {data}});
+    }
+}
+
+void HuffmanTree::sortTable()
+{
+    std::sort(mTable.begin(), mTable.end(), [](CodeLengthData a, CodeLengthData b){return a.first < b.first;});
+}
+
+std::optional<HuffmanTree::Symbol> HuffmanTree::findMatch(std::size_t treeIndex, uint32_t code) const
+{
+    const auto& legth_data = mTable[treeIndex];
+    for(const auto& entry : legth_data.second)
+    {
+        //std::cout << "Checking if " << entry.second << " matches code " << ByteUtils::toString(code) << std::endl;;
+        if (entry.first.matches(legth_data.first, code))
+        {
+            return entry.second;
+        }
+    }
+    return std::nullopt;
+}
+
+std::optional<PrefixCode> HuffmanTree::getCode(Symbol symbol) const
+{
+    for(const auto& entry : mTable)
+    {
+        for(const auto& data : entry.second)
+        {
+            if (data.second == symbol)
+            {
+                return data.first;
+            }
+        }
+    }
+    return std::nullopt;
+}
+
+std::size_t HuffmanTree::getNumCodeLengths() const
+{
+    return mTable.size();
+}
+
+unsigned HuffmanTree::getCodeLength(std::size_t idx) const
+{
+    return mTable[idx].first;
+}
+
+std::string HuffmanTree::dump(bool bitsAsRightToLeft) const
+{
+    std::stringstream sstr;
+    for (const auto& code_length_data : mTable)
+    {
+        sstr << "Prefix table for Code Length " << code_length_data.first << " has vals: \n";
+        for (const auto& entry : code_length_data.second)
+        {
+            sstr << "Code " << entry.first.toString(bitsAsRightToLeft) << " Symbol: " << entry.second << '\n';
+        }
+    }
+    return sstr.str();
+}

src/base/compression/huffman/HuffmanTree.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <optional>
+
+class PrefixCode
+{
+public:
+    PrefixCode(uint32_t data, unsigned length);
+
+    std::string toString(bool bitsAsRightToLeft = true) const;
+
+    bool matches(unsigned length, uint32_t code) const;
+
+    uint32_t getData() const
+    {
+        return mData;
+    }
+
+    unsigned getLength() const
+    {
+        return mLength;
+    }
+
+private:
+    unsigned mLength{0};
+    uint32_t mData{0};
+};
+
+class HuffmanTree
+{
+public:
+    using Symbol = unsigned;
+    using CodeLength = unsigned;
+    using CodeSymbolPair = std::pair<PrefixCode, Symbol>;
+    using CodeLengthData = std::pair<CodeLength, std::vector<CodeSymbolPair> >;
+
+    void addCodeLengthEntry(unsigned length, const CodeSymbolPair& data);
+
+    std::string dump(bool bitsAsRightToLeft = true) const;
+
+    std::optional<HuffmanTree::Symbol> findMatch(std::size_t treeIndex, uint32_t code) const;
+
+    std::size_t getNumCodeLengths() const;
+
+    unsigned getCodeLength(std::size_t idx) const;
+
+    std::optional<PrefixCode> getCode(Symbol symbol) const;
+
+    void sortTable();
+private:
+    std::vector<CodeLengthData> mTable;
+};

src/base/core/AbstractApp.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <memory>
+
+class IApplicationContext
+{
+public:
+    IApplicationContext() = default;
+    virtual ~IApplicationContext() = default;
+};
+
+class AbstractApp
+{
+public:
+    AbstractApp() = default;
+
+    virtual ~AbstractApp() = default;
+    IApplicationContext* GetApplicationContext() const
+    {
+        return mApplicationContext.get();
+    }
+protected:
+    std::unique_ptr<IApplicationContext> mApplicationContext{ nullptr };
+};
+
+using AbstractAppPtr = std::unique_ptr<AbstractApp>;

src/base/core/AbstractNamedItem.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+
+class AbstractNamedItem
+{
+public:
+	~AbstractNamedItem() = default;
+protected:
+	std::string mName;
+	std::string mSymbol;
+};

src/base/core/AbstractWebApp.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "AbstractApp.h"
+
+class HttpRequest;
+class HttpResponse;
+
+class AbstractWebApp : public AbstractApp
+{
+public:
+	virtual HttpResponse onHttpRequest(const HttpRequest& request) = 0;
+};

src/base/core/CMakeLists.txt

@@ -0,0 +1,70 @@
+set(MODULE_NAME core)
+
+list(APPEND HEADERS
+    AbstractApp.h
+	AbstractNamedItem.h
+	AbstractWebApp.h
+    Dictionary.h
+    Event.h
+    Color.h
+    CommandLineArgs.h
+	data_structures/RawTree.h
+	data_structures/List.h
+    data_structures/Tree.h
+	encoding/ByteUtils.h
+	encoding/StringUtils.h
+	encoding/UnicodeUtils.h
+    loggers/FileLogger.h
+    file_utilities/Directory.h
+    file_utilities/File.h
+    file_utilities/FileFormats.h
+    file_utilities/PathUtils.h
+    http/HttpResponse.h
+    http/HttpHeader.h
+    http/HttpRequest.h
+    serializers/TomlReader.h
+	Win32BaseIncludes.h
+)   
+
+list(APPEND SOURCES 
+    Event.cpp
+    Dictionary.cpp
+    Color.cpp
+    CommandLineArgs.cpp
+    data_structures/RawTree.cpp
+    data_structures/Tree.cpp
+    loggers/FileLogger.cpp
+    file_utilities/Directory.cpp
+    file_utilities/File.cpp
+    file_utilities/FileFormats.cpp
+    file_utilities/PathUtils.cpp
+    memory/SharedMemory.cpp
+    RandomUtils.cpp
+	encoding/StringUtils.cpp
+    encoding/ByteUtils.cpp
+	encoding/UnicodeUtils.cpp
+    streams/BinaryStream.cpp
+    streams/BitStream.cpp
+    streams/InputBitStream.cpp
+    streams/OutputBitStream.cpp
+    streams/BufferBitStream.cpp
+    http/HttpResponse.cpp
+    http/HttpHeader.cpp
+    http/HttpRequest.cpp
+    serializers/TomlReader.cpp)
+
+add_library(${MODULE_NAME} SHARED ${SOURCES} ${HEADERS})
+
+target_include_directories(${MODULE_NAME} PUBLIC
+                          ${CMAKE_CURRENT_SOURCE_DIR}
+						  ${CMAKE_CURRENT_SOURCE_DIR}/encoding
+                          ${CMAKE_CURRENT_SOURCE_DIR}/file_utilities
+                          ${CMAKE_CURRENT_SOURCE_DIR}/loggers
+                          ${CMAKE_CURRENT_SOURCE_DIR}/memory
+                          ${CMAKE_CURRENT_SOURCE_DIR}/streams
+                          ${CMAKE_CURRENT_SOURCE_DIR}/http
+                          ${CMAKE_CURRENT_SOURCE_DIR}/data_structures
+                          ${CMAKE_CURRENT_SOURCE_DIR}/serializers
+                          )
+set_target_properties( ${MODULE_NAME} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON )
+set_property(TARGET ${MODULE_NAME} PROPERTY FOLDER src/base)

src/base/core/Color.cpp

@@ -0,0 +1,55 @@
+#include "Color.h"
+
+Color::Color(unsigned char r, unsigned char g, unsigned char b, double a)
+    : mR(r),
+      mG(g),
+      mB(b),
+      mAlpha(a)
+{
+
+}
+
+std::unique_ptr<Color> Color::Create(unsigned char r, unsigned char g, unsigned char b, double a )
+{
+    return std::make_unique<Color>(r, g, b, a);
+}
+
+std::unique_ptr<Color> Color::Create(const Color& color)
+{
+    return std::make_unique<Color>(color);
+}
+
+unsigned char Color::getR() const
+{
+    return mR;
+}
+
+unsigned char Color::getG() const
+{
+    return mG;
+}
+
+unsigned char Color::getB() const
+{
+    return mB;
+}
+
+double Color::getAlpha() const
+{
+    return mAlpha;
+}
+
+std::vector<double> Color::getAsVectorDouble() const
+{
+    return { mR / 255.0, mG / 255.0, mB / 255.0, mAlpha };
+}
+
+uint32_t Color::getAsUInt32() const
+{
+    return static_cast<uint32_t>(mB + (mG << 8) + (mR << 16));
+}
+
+std::string Color::toString() const
+{
+    return std::to_string(static_cast<int>(mR)) + "," + std::to_string(static_cast<int>(mG)) + "," + std::to_string(static_cast<int>(mB));
+}

src/base/core/Color.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <memory>
+#include <vector>
+#include <string>
+
+class Color
+{
+public:
+    Color(unsigned char r = 0, unsigned char g = 0, unsigned char b = 0, double a = 1.0);
+
+    static std::unique_ptr<Color> Create(unsigned char r, unsigned char g, unsigned char b, double a = 1.0);
+    static std::unique_ptr<Color> Create(const Color& color);
+
+    unsigned char getR() const;
+    unsigned char getG() const;
+    unsigned char getB() const;
+    double getAlpha() const;
+
+    std::vector<double> getAsVectorDouble() const;
+
+    uint32_t getAsUInt32() const;
+
+    std::string toString() const;
+
+    bool operator==(const Color& rhs) const
+    {
+       return (mR == rhs.mR)
+       && (mG == rhs.mG)
+       && (mB == rhs.mB)
+       && (mAlpha == rhs.mAlpha);
+    }
+
+    bool operator!=(const Color& rhs) const
+    {
+      return !operator==(rhs);
+    }
+
+private:
+    unsigned char mR{0};
+    unsigned char mG{0};
+    unsigned char mB{0};
+    double mAlpha{0.0};
+};
+
+using ColorPtr = std::shared_ptr<Color>;
+using ColorUPtr = std::unique_ptr<Color>;

src/base/core/CommandLineArgs.cpp

@@ -0,0 +1,76 @@
+#include "CommandLineArgs.h"
+
+#include "UnicodeUtils.h"
+
+#include "Win32BaseIncludes.h"
+
+CommandLineArgs::CommandLineArgs()
+    : mArugments(),
+      mLaunchPath()
+{
+
+}
+
+void CommandLineArgs::initialize(CommandLineArgs* args)
+{
+#ifdef _WIN32
+    int nArgs{ 0 };
+    auto szArglist = ::CommandLineToArgvW(::GetCommandLineW(), &nArgs);
+    if (szArglist == nullptr)
+    {
+        return;
+    }
+
+    std::vector<std::string> windowsArgs(nArgs);
+    for (int idx = 0; idx < nArgs; idx++)
+    {
+        windowsArgs[idx] = UnicodeUtils::utf16ToUtf8String(szArglist[idx]);
+    }
+    ::LocalFree(szArglist);
+
+    args->process(windowsArgs);
+#endif
+}
+
+std::unique_ptr<CommandLineArgs> CommandLineArgs::Create()
+{
+    return std::make_unique<CommandLineArgs>();
+}
+
+std::filesystem::path CommandLineArgs::getLaunchPath()
+{
+    return mLaunchPath;
+}
+
+void CommandLineArgs::recordLaunchPath()
+{
+    mLaunchPath = std::filesystem::current_path();
+}
+
+void CommandLineArgs::process(int argc, char *argv[])
+{
+    for(int idx=0; idx<argc; idx++)
+    {
+        mArugments.push_back(std::string(argv[idx]));
+    }
+}
+
+void CommandLineArgs::process(const std::vector<std::string>& args)
+{
+    mArugments = args;
+}
+
+std::vector<std::string> CommandLineArgs::getUserArgs() const
+{
+    std::vector<std::string> user_args;
+    for(unsigned idx=1; idx<mArugments.size(); idx++)
+    {
+        user_args.push_back(mArugments[idx]);
+    }
+    return user_args;
+}
+
+const std::vector<std::string> CommandLineArgs::getArgs() const
+{
+    return mArugments;
+}

src/base/core/CommandLineArgs.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <memory>
+#include <vector>
+#include <string>
+#include <filesystem>
+
+class CommandLineArgs
+{
+public:
+    CommandLineArgs();
+
+    static std::unique_ptr<CommandLineArgs> Create();
+
+    std::filesystem::path getLaunchPath();
+
+    const std::vector<std::string> getArgs() const;
+
+    std::vector<std::string> getUserArgs() const;
+
+    static void initialize(CommandLineArgs* args);
+
+    void process(int argc, char *argv[]);
+
+    void process(const std::vector<std::string>& args);
+
+    void recordLaunchPath();
+private:
+    std::vector<std::string> mArugments;
+    std::filesystem::path mLaunchPath;
+};
+
+using CommandLineArgsUPtr = std::unique_ptr<CommandLineArgs>;

src/base/core/Dictionary.cpp

@@ -0,0 +1,56 @@
+#include "Dictionary.h"
+
+bool Dictionary::hasKey(const std::string& key) const
+{
+    return hasStringKey(key) || hasDictKey(key);
+}
+
+bool Dictionary::hasStringKey(const std::string& key) const
+{
+    return mStringData.count(key) > 0;
+}
+
+bool Dictionary::hasDictKey(const std::string& key) const
+{
+    return mDictData.count(key) > 0;
+}
+
+std::vector<std::string> Dictionary::getStringKeys() const
+{
+    std::vector<std::string> keys;
+    for (const auto& item : mStringData)
+    {
+        keys.push_back(item.first);
+    }
+    return keys;
+}
+
+std::vector<std::string> Dictionary::getDictKeys() const
+{
+    std::vector<std::string> keys;
+    for (const auto& item : mDictData)
+    {
+        keys.push_back(item.first);
+    }
+    return keys;
+}
+
+Dictionary* Dictionary::getDict(const std::string& key) const
+{
+    return mDictData.at(key).get();
+}
+
+std::string Dictionary::getItem(const std::string& key) const
+{
+    return mStringData.at(key);
+}
+
+void Dictionary::addStringItem(const std::string& key, const std::string& item)
+{
+    mStringData[key] = item;
+}
+
+void Dictionary::addDictItem(const std::string& key, std::unique_ptr<Dictionary> dict)
+{
+    mDictData[key] = std::move(dict);
+}

src/base/core/Dictionary.h

@@ -0,0 +1,34 @@
+#pragma once
+#include <string>
+#include <vector>
+#include <map>
+#include <memory>
+
+class Dictionary
+{
+public:
+    Dictionary() = default;
+    virtual ~Dictionary() = default;
+
+    void addStringItem(const std::string& key, const std::string& item);
+
+    void addDictItem(const std::string& key, std::unique_ptr<Dictionary> dict);
+
+    Dictionary* getDict(const std::string& key) const;
+
+    std::vector<std::string> getDictKeys() const;
+
+    std::vector<std::string> getStringKeys() const;
+
+    std::string getItem(const std::string& key) const;
+
+    bool hasKey(const std::string& key) const;
+
+    bool hasStringKey(const std::string& key) const;
+
+    bool hasDictKey(const std::string& key) const;
+protected:
+
+    std::map<std::string, std::string> mStringData;
+    std::map<std::string, std::unique_ptr<Dictionary> > mDictData;
+};

src/base/core/Event.cpp

@@ -0,0 +1,11 @@
+#include "Event.h"
+
+Event::Event()
+{
+
+}
+
+Event::~Event()
+{
+
+}

src/base/core/Event.h

@@ -0,0 +1,8 @@
+class Event
+{
+public:
+
+	Event();
+
+	~Event();
+};

src/base/core/RandomUtils.cpp

@@ -0,0 +1,22 @@
+#include "RandomUtils.h"
+
+#include <random>
+#include <algorithm>
+#include <iterator>
+
+std::vector<unsigned> RandomUtils::getRandomVecUnsigned(std::size_t size)
+{
+    std::random_device rnd_device;
+
+    std::mt19937 mersenne_engine {rnd_device()};
+    std::uniform_int_distribution<unsigned> dist {0, 9};
+
+    auto generator = [&dist, &mersenne_engine]()
+    		{
+    	return dist(mersenne_engine);
+    		};
+
+    std::vector<unsigned> vec(size);
+    std::generate(std::begin(vec), std::end(vec), generator);
+    return vec;
+}

src/base/core/RandomUtils.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <vector>
+
+class RandomUtils
+{
+public:
+
+	static std::vector<unsigned> getRandomVecUnsigned(std::size_t size);
+};

src/base/core/Win32BaseIncludes.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#ifndef UNICODE
+#define UNICODE
+#endif
+
+//#ifndef WIN32_LEAN_AND_MEAN
+//#define WIN32_LEAN_AND_MEAN
+//#endif
+
+#ifdef _WIN32
+#include "Windows.h"
+#endif

src/base/core/data_structures/CircleBuffer.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <vector>
+
+#include <iostream>
+
+template<typename T>
+class CircleBuffer
+{
+public:
+    CircleBuffer(std::size_t size)
+        : mData(size)
+    {
+
+    }
+
+    void addItem(const T& item)
+    {
+        if (mEndPointer < mData.size())
+        {
+            mData[mEndPointer] = item;
+            mEndPointer++;
+        }
+        else
+        {
+            mData[mStartPointer] = item;
+            if (mStartPointer < mData.size() - 1)
+            {
+                mStartPointer++;
+            }
+            else
+            {
+                mStartPointer = 0;
+            }
+        }
+    }
+
+    std::size_t getNumItems() const
+    {
+        return mEndPointer;
+    }
+
+    const T& getItem(std::size_t index) const
+    {
+        if (mEndPointer < mData.size())
+        {
+            return mData[index];
+        }
+        else
+        {
+            auto offset = mStartPointer + index;
+            if (offset >= mData.size())
+            {
+                offset -= mData.size();
+            }
+            return mData[offset];
+        }
+    }
+
+private:
+    std::size_t mStartPointer{0};
+    std::size_t mEndPointer{0};
+    std::vector<T> mData;
+};

src/base/core/data_structures/List.h

@@ -0,0 +1,69 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <stdexcept>  
+
+class AbstractList
+{
+public:
+	virtual ~AbstractList() = default;
+
+	virtual std::size_t getLength() const = 0;
+};
+
+template<typename T>
+class List : public AbstractList
+{
+public:
+	List() = default;
+
+	void initializeTo(std::size_t size, T value)
+	{
+		mData = std::vector<T>(size, value);
+	}
+
+	const T* getDataPtr() const
+	{
+		return mData.data();
+	}
+
+	const std::vector<T>& getData() const
+	{
+		return mData;
+	}
+
+	T getItem(std::size_t index) const
+	{
+		if (index < mData.size())
+		{
+			return mData[index];
+		}
+		else
+		{
+			const auto msg = "Tried to access out of range index: " + std::to_string(index) + " with size " + std::to_string(mData.size());
+			throw std::out_of_range(msg);
+		}
+	}
+
+	void setData(const std::vector<T>& data)
+	{
+		mData = data;
+	}
+
+	void setItem(std::size_t index, T item)
+	{
+		if (index < mData.size())
+		{
+			mData[index] = item;
+		}
+	}
+
+	std::size_t getLength() const override
+	{
+		return mData.size();
+	}
+
+private:
+	std::vector<T> mData;
+};

src/base/core/data_structures/RawTree.h

@@ -0,0 +1,97 @@
+#pragma once
+
+#include <memory>
+
+template<typename T>
+class RawNode
+{
+public:
+    RawNode(T data)
+        : mData(data)
+    {
+
+
+    }
+
+    ~RawNode()
+    {
+        if(mLeftChild)
+        {
+            delete mLeftChild;
+        }
+
+        if(mRightChild)
+        {
+            delete mRightChild;
+        }
+    }
+
+    void addChild(RawNode* child)
+    {
+        if(!mLeftChild)
+        {
+            mLeftChild = child;
+        }
+        else
+        {
+            mRightChild = child;
+        }
+    }
+
+    bool isLeaf() const
+    {
+        return !mLeftChild && !mRightChild;
+    }
+
+    T getData() const
+    {
+        return mData;
+    }
+
+    RawNode* getLeftChild() const
+    {
+        return mLeftChild;
+    }
+
+    RawNode* getRightChild() const
+    {
+        return mRightChild;
+    }
+
+private:
+    T mData;
+    unsigned char mTag{0};
+    RawNode* mLeftChild{nullptr};
+    RawNode* mRightChild{nullptr};
+};
+
+template<typename T>
+class RawTree
+{
+public:
+    RawTree()
+    {
+
+    }
+
+    ~RawTree()
+    {
+        if (mRootNode)
+        {
+            delete mRootNode;
+        }
+    }
+
+    void addRootNode(RawNode<T>* root)
+    {
+        mRootNode = root;
+    }
+
+    RawNode<T>* getRootNode() const
+    {
+        return mRootNode;
+    }
+
+private:
+    RawNode<T>* mRootNode{nullptr};
+};

src/base/core/data_structures/Tree.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include <memory>
+
+template<typename T>
+class Node
+{
+public:
+    Node(T data)
+        : mData(data)
+    {}
+
+    void addChild(std::unique_ptr<Node> child)
+    {
+        if(!mLeftChild)
+        {
+            mLeftChild = std::move(child);
+        }
+        else
+        {
+            mRightChild = std::move(child);
+        }
+    }
+
+    bool isLeaf() const
+    {
+        return !mLeftChild && !mRightChild;
+    }
+
+    T getData() const
+    {
+        return mData;
+    }
+
+    Node* getLeftChild() const
+    {
+        return mLeftChild.get();
+    }
+
+    Node* getRightChild() const
+    {
+        return mRightChild.get();
+    }
+
+private:
+    T mData;
+    unsigned char mTag{0};
+    std::unique_ptr<Node> mLeftChild;
+    std::unique_ptr<Node> mRightChild;
+};
+
+template<typename T>
+using NodePtr = std::unique_ptr<Node<T> >;
+
+template<typename T>
+class Tree
+{
+public:
+    Tree()
+    {
+
+    }
+
+    void addRootNode(NodePtr<T> root)
+    {
+        mRootNode = std::move(root);
+    }
+
+    Node<T>* getRootNode() const
+    {
+        return mRootNode.get();
+    }
+
+private:
+    NodePtr<T> mRootNode;
+};

src/base/core/encoding/ByteUtils.cpp

@@ -0,0 +1,197 @@
+#include "ByteUtils.h"
+
+
+bool ByteUtils::MostSignificantBitIsOne(char c)
+{
+    return c & (1 << 7);
+}
+
+ByteUtils::Word ByteUtils::GetWordFirstBit(const Word word)
+{
+    return word & ByteUtils::WORD_FIRST_BIT;
+};
+
+ByteUtils::Word ByteUtils::GetWordLastByte(const Word word)
+{
+    return word & ByteUtils::WORD_LAST_BYTE;
+}
+
+unsigned char ByteUtils::getHigherNBits(unsigned char input, unsigned num)
+{
+    return input >> (8 - num);
+}
+
+unsigned char ByteUtils::getByteN(uint32_t input, unsigned n)
+{
+    return (input << 8*n) >> 24;
+}
+
+uint32_t ByteUtils::mirror(uint32_t byte, unsigned length)
+{
+    uint32_t ret{0};
+    for(unsigned idx=0; idx<length; idx++)
+    {
+        if (getBitN(byte, length - 1 - idx))
+        {
+            ret |= (0x01 << idx);
+        }
+    }
+    return ret;
+}
+
+unsigned char ByteUtils::getLowerNBits(uint32_t input, unsigned num)
+{
+    switch (num)
+    {
+    case 1:
+        return input & 0x01;
+    case 2:
+        return input & 0x03;
+    case 3:
+        return input & 0x07;
+    case 4:
+        return input & 0x0F;
+    case 5:
+        return input & 0x1F;
+    case 6:
+        return input & 0x3F;
+    case 7:
+        return input & 0x7F;
+    case 8:
+        return static_cast<unsigned char>(input);
+    default:
+        return 0;
+    }
+}
+
+unsigned char ByteUtils::getTwoBitsAtN(unsigned char input, unsigned n)
+{
+    return (input & (0x03 << n)) >> n;
+}
+
+unsigned char ByteUtils::getMBitsAtN(unsigned char input, unsigned m, unsigned n)
+{
+    switch (m)
+    {
+    case 1:
+        return (input & (0x01 << n)) >> n;
+    case 2:
+        return (input & (0x03 << n)) >> n;
+    case 3:
+        return (input & (0x07 << n)) >> n;
+    case 4:
+        return (input & (0x0F << n)) >> n;
+    case 5:
+        return (input & (0x1F << n)) >> n;
+    case 6:
+        return (input & (0x3F << n)) >> n;
+    case 7:
+        return (input & (0x7F << n)) >> n;
+    case 8:
+        return input;
+    default:
+        return 0;
+    }
+}
+
+bool ByteUtils::getBitN(uint32_t input, unsigned n)
+{
+    return input & (1 << n);
+}
+
+unsigned char ByteUtils::getFromString(const std::string& string)
+{
+    unsigned char ret{0};
+
+    if (string.length() < 8)
+    {
+        return ret;
+    }
+
+    for(unsigned idx=0; idx<8; idx++)
+    {
+        if (string[idx] == '1')
+        {
+            ret |= (0x01 << (7 - idx));
+        }
+    }
+    return ret;
+}
+
+std::string ByteUtils::toString(uint32_t input, unsigned length)
+{
+    std::string ret;
+    if (length > 8)
+    {
+        unsigned overshoot = length - 8;
+        for(unsigned idx=0; idx<overshoot; idx++)
+        {
+            ret += getBitN(input, length - 1 - idx) ? '1' : '0';
+        }
+        ret += "-";
+        for(unsigned idx=0; idx<8; idx++)
+        {
+            ret += getBitN(input, 7 - idx) ? '1' : '0';
+        }
+    }
+    else
+    {
+        for(unsigned idx=0; idx<length; idx++)
+        {
+            ret += getBitN(input, 7 - idx) ? '1' : '0';
+        }
+    }
+    return ret;
+}
+
+void ByteUtils::ReverseBuffer(char* buffer, char* reverse, unsigned size, unsigned targetSize)
+{
+    for(unsigned idx=0; idx<targetSize; idx++)
+    {
+        if (idx < size)
+        {
+            reverse[idx] = buffer[size - 1 -idx];
+        }
+        else
+        {
+            reverse[idx] = 0;
+        }
+    }
+}
+
+ByteUtils::Word ByteUtils::ToWord(char* buffer, bool reverse)
+{
+    return ToType<Word>(buffer, reverse);
+}
+
+ByteUtils::DWord ByteUtils::ToDWord(char* buffer, bool reverse)
+{
+    return ToType<DWord>(buffer, reverse);
+}
+
+ByteUtils::QWord ByteUtils::ToQWord(char* buffer, bool reverse)
+{
+    return ToType<QWord>(buffer, reverse);
+}
+
+bool ByteUtils::Compare(char* buffer, const char* tag, unsigned size)
+{
+    for(unsigned idx=0; idx<size; idx++)
+    {
+        if(tag[idx] != buffer[idx])
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+bool ByteUtils::CompareDWords(char* buffer, const char* tag)
+{
+    return Compare(buffer, tag, sizeof(DWord));
+}
+
+bool ByteUtils::CompareWords(char* buffer, const char* tag)
+{
+    return Compare(buffer, tag, sizeof(Word));
+}

src/base/core/encoding/ByteUtils.h

@@ -0,0 +1,72 @@
+#pragma once
+
+#include <cstring>
+#include <stdint.h>
+#include <string>
+
+class ByteUtils
+{
+public:
+    using Word = int16_t;
+    using DWord = int32_t;
+    using QWord = int64_t;
+
+    static bool MostSignificantBitIsOne(char c);
+
+    static Word GetWordFirstBit(const Word word);
+
+    static Word GetWordLastByte(const Word word);
+
+    static unsigned char getByteN(uint32_t input, unsigned n);
+
+    static unsigned char getHigherNBits(unsigned char input, unsigned num);
+
+    static unsigned char getLowerNBits(uint32_t input, unsigned num);
+
+    static unsigned char getTwoBitsAtN(unsigned char input, unsigned n);
+
+    static unsigned char getMBitsAtN(unsigned char input, unsigned m, unsigned n);
+
+    static bool getBitN(uint32_t input, unsigned n);
+
+    static unsigned char getFromString(const std::string& string);
+
+    static std::string toString(uint32_t input, unsigned length = 8);
+
+    static uint32_t mirror(uint32_t input, unsigned length=0);
+
+    static void ReverseBuffer(char* buffer, char* reverse, unsigned size, unsigned targetSize);
+
+    template<typename T>
+    static T ToType(char* buffer, bool reverse = true)
+    {
+        T result {0};
+        if(reverse)
+        {
+            char reversed[sizeof(T)];
+            ReverseBuffer(buffer, reversed, sizeof(T), sizeof(T));
+            std::memcpy(&result, reversed, sizeof(T));
+        }
+        else
+        {
+            std::memcpy(&result, buffer, sizeof(T));
+        }
+        return result;
+    }
+
+    static Word ToWord(char* buffer, bool reverse = true);
+
+    static DWord ToDWord(char* buffer, bool reverse = true);
+
+    static QWord ToQWord(char* buffer, bool reverse = true);
+
+    static bool Compare(char* buffer, const char* tag, unsigned size);
+
+    static bool CompareDWords(char* buffer, const char* tag);
+
+    static bool CompareWords(char* buffer, const char* tag);
+
+    static const int BYTE_FIRST_BIT = 0x40; // 1000 0000
+    static const Word WORD_FIRST_BIT = static_cast<Word>(0x8000); // 1000 0000 - 0000 0000
+    static const Word WORD_LAST_BYTE = 0x00FF; // 0000 0000 - 1111 1111
+};

src/base/core/encoding/StringUtils.cpp

@@ -0,0 +1,209 @@
+#include "StringUtils.h"
+
+#include <locale>
+#include <algorithm>
+#include <sstream>
+#include <iomanip>
+
+bool StringUtils::isAlphaNumeric(char c)
+{
+    return std::isalnum(c);
+}
+
+bool StringUtils::isSpace(char c)
+{
+    return std::isspace(c);
+}
+
+bool StringUtils::isAlphabetical(char c)
+{
+    return std::isalpha(c);
+}
+
+std::vector<unsigned char> StringUtils::toBytes(const std::string& input)
+{
+    return {input.begin(), input.end()};
+}
+
+std::string StringUtils::toString(const std::vector<unsigned char>& bytes)
+{
+    return {bytes.begin(), bytes.end()};
+}
+
+std::vector<std::string> StringUtils::toLines(const std::string& input)
+{
+    auto result = std::vector<std::string>{};
+    auto ss = std::stringstream{input};
+    for (std::string line; std::getline(ss, line, '\n');)
+    {
+        result.push_back(line);
+    }
+    return result;
+}
+
+bool StringUtils::isWhitespaceOnly(const std::string& input)
+{
+    if (input.empty())
+    {
+        return true;
+    }
+    else
+    {
+        return std::all_of(input.cbegin(), input.cend(), [](char c){ return std::isspace(c); });
+    }
+}
+
+std::size_t StringUtils::countFirstConsecutiveHits(const std::string& input, char c)
+{
+    auto found_id = input.find(c);
+    if(found_id == std::string::npos)
+    {
+        return 0;
+    }
+    else
+    {
+        std::size_t count = 1;
+        for(std::size_t idx=found_id+1; idx<input.size(); idx++)
+        {
+            if(input[idx] == c)
+            {
+                count++;
+            }
+            else
+            {
+                return count;
+            }
+        }
+        return count;
+    }
+}
+
+std::string StringUtils::stripSurroundingWhitepsace(const std::string& input)
+{
+    if (input.empty())
+    {
+        return {};
+    }
+
+    std::size_t first_nonspace = 0;
+    std::size_t last_nonspace = input.size() - 1;
+    for (std::size_t idx = 0; idx < input.size(); idx++)
+    {
+        if (!std::isspace(input[idx]))
+        {
+            first_nonspace = idx;
+            break;
+        }
+    }
+
+    if (first_nonspace == last_nonspace)
+    {
+        return {};
+    }
+
+    for (std::size_t idx = last_nonspace; idx > 0; idx--)
+    {
+        if (!std::isspace(input[idx]))
+        {
+            last_nonspace = idx;
+            break;
+        }
+    }
+    return input.substr(first_nonspace, last_nonspace-first_nonspace + 1);
+}
+
+std::vector<std::string> StringUtils::split(const std::string& input)
+{
+    std::vector<std::string> substrings;
+    std::string working_string;
+    std::locale loc;
+    bool last_was_nonspace{ false };
+    for (auto c : input)
+    {
+        if (std::isspace(c, loc))
+        {
+            if (last_was_nonspace)
+            {
+                substrings.push_back(working_string);
+                working_string = "";
+                last_was_nonspace = false;
+            }
+        }
+        else
+        {
+            last_was_nonspace = true;
+            working_string += c;
+        }
+    }
+    if (!working_string.empty())
+    {
+        substrings.push_back(working_string);
+    }
+    return substrings;
+}
+
+std::string StringUtils::toLower(const std::string& s)
+{
+    std::string ret;
+    std::transform(s.begin(), s.end(), ret.begin(), [](unsigned char c){ return std::tolower(c); });
+    return ret;
+}
+
+std::string StringUtils::toPaddedString(unsigned numBytes, unsigned entry)
+{
+    std::stringstream sstr;
+    sstr << std::setfill('0') << std::setw(numBytes) << entry;
+    return sstr.str();
+}
+
+std::string StringUtils::stripQuotes(const std::string& input)
+{
+    if (input.size() < 3)
+    {
+        return input;
+    }
+    std::size_t start_index = 0;
+    std::size_t end_index = input.size()-1;
+    if (input[start_index] == '"')
+    {
+        start_index = 1;
+    }
+    if (input[end_index] == '"')
+    {
+        end_index = end_index - 1;
+    }
+    return input.substr(start_index, end_index - start_index + 1);
+}
+
+std::string StringUtils::removeUpTo(const std::string& input, const std::string& prefix)
+{
+    std::size_t found = input.find(prefix);
+    if (found != std::string::npos)
+    {
+        return input.substr(found + prefix.size(), input.size()-found);
+    }
+    else
+    {
+        return input;
+    }
+}
+
+bool StringUtils::startsWith(const std::string& input, const std::string& prefix, bool ignoreWhitespace)
+{
+    if(ignoreWhitespace)
+    {
+        const auto loc = input.find(prefix);
+        if (loc == std::string::npos)
+        {
+            return false;
+        }
+        else
+        {
+            return isWhitespaceOnly(input.substr(0, loc));
+        }
+    }
+    else
+    {
+        return input.find(prefix) == 0;
+    }
+}

src/base/core/encoding/StringUtils.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+class StringUtils
+{
+public:
+    static constexpr char LEFT_BRACKET = '<';
+    static constexpr char RIGHT_BRACKET = '>';
+    static constexpr char FORWARD_SLASH = '/';
+    static constexpr char BACK_SLASH = '\\';
+    static constexpr char QUESTION_MARK = '?';
+    static constexpr char EQUALS = '=';
+    static constexpr char DOUBLE_QUOTE = '"';
+    static constexpr char SINGLE_QUOTE = '\'';
+    static constexpr char COLON = ':';
+
+    static std::size_t countFirstConsecutiveHits(const std::string& input, char c);
+
+    static bool isAlphaNumeric(char c);
+
+    static bool isAlphabetical(char c);
+
+    static bool isSpace(char c);
+
+    static bool isWhitespaceOnly(const std::string& input);
+
+    static std::string removeUpTo(const std::string& input, const std::string& prefix);
+
+    static std::vector<std::string> split(const std::string& input);
+
+    static bool startsWith(const std::string& input, const std::string& prefix, bool ignoreWhitespace = false);
+
+    static std::string stripSurroundingWhitepsace(const std::string& input);
+
+    static std::string stripQuotes(const std::string& input);
+
+    static std::vector<unsigned char> toBytes(const std::string& input);
+
+    static std::string toLower(const std::string& s);
+
+    static std::vector<std::string> toLines(const std::string& input);
+
+    static std::string toPaddedString(unsigned numBytes, unsigned entry);
+
+    static std::string toString(const std::vector<unsigned char>& bytes);
+};

src/base/core/encoding/UnicodeUtils.cpp

@@ -0,0 +1,41 @@
+#include "UnicodeUtils.h"
+
+#include "Win32BaseIncludes.h"
+
+#include <vector>
+#include <stdexcept>
+
+std::string UnicodeUtils::utf16ToUtf8String(const std::wstring& input)
+{
+    if (input.empty())
+    {
+        return std::string();
+    }
+
+#ifdef _WIN32
+    const auto size = ::WideCharToMultiByte(CP_UTF8, 0, &input[0], static_cast<int>(input.size()), nullptr, 0, nullptr, nullptr);
+
+    std::string result(size, 0);
+    ::WideCharToMultiByte(CP_UTF8, 0, &input[0], static_cast<int>(input.size()), &result[0], size, nullptr, nullptr);
+    return result;
+#else
+    throw std::logic_error("Not implemented");
+#endif
+}
+
+std::wstring UnicodeUtils::utf8ToUtf16WString(const std::string& input)
+{
+    if (input.empty())
+    {
+        return std::wstring();
+    }
+
+#ifdef _WIN32
+    const auto charsNeeded = ::MultiByteToWideChar(CP_UTF8, 0, input.data(), static_cast<int>(input.size()), nullptr, 0);
+    std::vector<wchar_t> buffer(charsNeeded);
+    const auto charsConverted = ::MultiByteToWideChar(CP_UTF8, 0, input.data(), static_cast<int>(input.size()), &buffer[0], static_cast<int>(buffer.size()));
+    return std::wstring(&buffer[0], charsConverted);
+#else
+    throw std::logic_error("Not implemented");
+#endif
+}

src/base/core/encoding/UnicodeUtils.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+
+class UnicodeUtils
+{
+public:
+	static std::string utf16ToUtf8String(const std::wstring& input);
+
+	static std::wstring utf8ToUtf16WString(const std::string& input);
+};

src/base/core/file_utilities/Directory.cpp

@@ -0,0 +1,22 @@
+#include "Directory.h"
+
+std::vector<Path> Directory::getFilesWithExtension(const Path& path, const std::string& extension, bool recursive)
+{
+    std::vector<Path> paths;
+    if (std::filesystem::is_directory(path))
+    {
+        for (const auto& entry : std::filesystem::directory_iterator(path))
+        {
+            if (std::filesystem::is_regular_file(entry) && entry.path().extension() == extension)
+            {
+                paths.push_back(entry.path());
+            }
+            else if(recursive && std::filesystem::is_directory(entry))
+            {
+                const auto child_paths = getFilesWithExtension(entry, extension, recursive);
+                paths.insert(paths.end(), child_paths.begin(), child_paths.end());
+            }
+        }
+    }
+    return paths;
+}

src/base/core/file_utilities/Directory.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <vector>
+#include <filesystem>
+
+using Path = std::filesystem::path;
+
+class Directory
+{
+public:
+
+    static std::vector<Path> getFilesWithExtension(const Path& path, const std::string& extension, bool recursive=false);
+};

src/base/core/file_utilities/File.cpp

@@ -0,0 +1,220 @@
+#include "File.h"
+
+#include "FileLogger.h"
+#include "ByteUtils.h"
+
+#include <streambuf>
+#include <sstream>
+
+File::File(std::filesystem::path path)
+    : mFullPath(path),
+      mInHandle(),
+      mOutHandle()
+{
+
+}
+
+File::~File()
+{
+    close();
+}
+
+std::string File::getExtension() const
+{
+    return mFullPath.extension().string();
+}
+
+std::string File::dumpBinary()
+{
+    open(AccessMode::Read);
+
+    std::stringstream sstr;
+    sstr << "Count | Binary | Decimal | ASCII \n";
+    unsigned count = 0;
+    while(mInHandle->peek() != EOF)
+    {
+        const unsigned char val = static_cast<unsigned char>(mInHandle->get());
+        const unsigned char ascii_val = std::isalpha(val) ? val : '.';
+        sstr << count << " | " << ByteUtils::toString(val) << " | " << static_cast<int>(val) << " | " << ascii_val << '\n';
+        if (count < 0 && count % 10 == 0)
+        {
+            sstr << "\n";
+        }
+        count++;
+    }
+    const auto out = sstr.str();
+    close();
+    return out;
+}
+
+std::ifstream* File::getInHandle() const
+{
+    return mInHandle.get();
+}
+
+std::ofstream* File::getOutHandle() const
+{
+    return mOutHandle.get();
+}
+
+std::optional<unsigned char> File::readNextByte()
+{
+    if (!mInHandle)
+    {
+        if (!open(AccessMode::Read))
+        {
+            return std::nullopt;
+        }
+    }
+
+    if (mInHandle->good())
+    {
+        if (auto val = mInHandle->get(); val == EOF)
+        {
+            return std::nullopt;
+        }
+        else
+        {
+            return val;
+        }
+    }
+    else
+    {
+        return std::nullopt;
+    }
+}
+
+bool File::open(AccessMode accessMode)
+{
+    if (mFullPath.is_absolute() && !std::filesystem::exists(mFullPath.parent_path()))
+    {
+        std::filesystem::create_directories(mFullPath.parent_path());
+    }
+
+    if(accessMode == AccessMode::Read)
+    {
+        auto flags = std::ifstream::in;
+        mInHandle = std::make_unique<std::ifstream>();
+        mInHandle->open(mFullPath, flags);
+        if (!mInHandle->is_open())
+        {
+            MLOG_ERROR("Failed to open file at :" << mFullPath);
+            return false;
+        }
+    }
+    else
+    {
+        auto flags = std::ofstream::out;
+        mOutHandle = std::make_unique<std::ofstream>();
+        mOutHandle->open(mFullPath, flags);
+        if (!mOutHandle->is_open())
+        {
+            MLOG_ERROR("Failed to open file at :" << mFullPath);
+            return false;
+        }
+    }
+    return true;
+}
+
+void File::close()
+{
+    if(mOutHandle)
+    {
+        mOutHandle->close();
+    }
+    if(mInHandle)
+    {
+        mInHandle->close();
+    }
+}
+
+FileFormat::Format File::inferFormat() const
+{
+    const auto extension = getExtension();
+    return FileFormat::inferFormat(extension);
+}
+
+void File::writeText(const std::string& text)
+{
+    bool had_to_open{false};
+    if (!mOutHandle)
+    {
+        had_to_open = true;
+        if (!open(File::AccessMode::Write))
+        {
+            return;
+        }
+    }
+
+    (*mOutHandle) << text;
+
+    if (had_to_open)
+    {
+        close();
+    }
+}
+
+std::vector<std::string> File::readLines()
+{
+    std::vector<std::string> content;
+
+    if (!pathExists())
+    {
+        return {};
+    }
+
+    if(!open(AccessMode::Read))
+    {
+        return {};
+    }
+
+    std::string str;
+    while(std::getline(*mInHandle, str))
+    {
+        content.push_back(str);
+    }
+
+    close();
+    return content;
+}
+
+std::string File::read()
+{
+    if (!pathExists())
+    {
+        return {};
+    }
+
+    if(!open(AccessMode::Read))
+    {
+        return {};
+    }
+
+    std::stringstream buffer;
+    buffer << mInHandle->rdbuf();
+
+    close();
+    return buffer.str();
+}
+
+std::string File::readText()
+{
+    if (!pathExists())
+    {
+        return {};
+    }
+
+    if(!open(AccessMode::Read))
+    {
+        return {};
+    }
+
+    std::string str((std::istreambuf_iterator<char>(*mInHandle)),
+                     std::istreambuf_iterator<char>());
+    return str;
+}
+
+bool File::pathExists() const
+{
+    return std::filesystem::exists(mFullPath);
+}

src/base/core/file_utilities/File.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "FileFormats.h"
+
+#include <filesystem>
+#include <string>
+#include <fstream>
+#include <memory>
+#include <vector>
+#include <optional>
+
+using Path = std::filesystem::path;
+
+class File
+{
+public:
+    enum class AccessMode{
+        Read,
+        Write
+    };
+
+public:
+    File(std::filesystem::path fullPath);
+
+    ~File();
+
+    void close();
+
+    std::string dumpBinary();
+
+    std::string getExtension() const;
+
+    std::ifstream* getInHandle() const;
+
+    std::ofstream* getOutHandle() const;
+
+    FileFormat::Format inferFormat() const;
+
+    std::string readText();
+
+    std::vector<std::string> readLines();
+
+    std::string read();
+
+    bool pathExists() const;
+
+    bool open(AccessMode mode);
+
+    std::optional<unsigned char> readNextByte();
+
+    void writeText(const std::string& text);
+
+private:
+    std::filesystem::path mFullPath;
+    std::unique_ptr<std::ifstream> mInHandle;
+    std::unique_ptr<std::ofstream> mOutHandle;
+};

src/base/core/file_utilities/FileFormats.cpp

@@ -0,0 +1,34 @@
+#include "FileFormats.h"
+
+#include "StringUtils.h"
+
+FileFormat::ExtensionMap FileFormat::mExtensions = []
+{
+    ExtensionMap ret;
+    ret[Format::Markdown] = ".md";
+    ret[Format::Html] = ".html";
+    ret[Format::Wav] = ".wav";
+    return ret;
+}();
+
+bool FileFormat::isFormat(const std::string& extension, Format format)
+{
+    return StringUtils::toLower(extension) == mExtensions[format];
+}
+
+FileFormat::Format FileFormat::inferFormat(const std::string& query)
+{
+    for(const auto& extension : mExtensions)
+    {
+        if(extension.second == query)
+        {
+            return extension.first;
+        }
+    }
+    return Format::Unknown;
+}
+
+std::string FileFormat::getExtension(Format format)
+{
+    return mExtensions[format];
+}

src/base/core/file_utilities/FileFormats.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <map>
+#include <string>
+
+class FileFormat{
+
+public:
+    enum class Encoding{
+        Ascii,
+        UTF8,
+        Binary
+    };
+
+    enum class Format{
+        Html,
+        Xml,
+        Markdown,
+        Pdf,
+        Json,
+        Png,
+        Wav,
+        Midi,
+        Unknown
+    };
+
+    using ExtensionMap = std::map<Format, std::string>;
+public:
+    static bool isFormat(const std::string& extension, Format format);
+
+    static Format inferFormat(const std::string& query);
+
+    static std::string getExtension(Format format);
+
+private:
+    static ExtensionMap mExtensions;
+};
+

src/base/core/file_utilities/PathUtils.cpp

@@ -0,0 +1,33 @@
+#include "PathUtils.h"
+
+
+std::string PathUtils::getBaseFilename(const Path& path)
+{
+    return path.stem().string();
+}
+
+Path PathUtils::getRelativePath(const Path& input, const Path& relativeTo)
+{
+    return std::filesystem::relative(input, relativeTo);
+}
+
+std::string PathUtils::getPathDelimited(const Path& path, char delimiter)
+{
+    std::string name;
+
+    unsigned count = 0;
+    for(const auto& element : path)
+    {
+        if (count == 0)
+        {
+            name += element.stem().string();
+        }
+        else
+        {
+            name += delimiter + element.stem().string();
+        }
+        count++;
+    }
+
+    return name;
+}

src/base/core/file_utilities/PathUtils.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <filesystem>
+#include <string>
+#include <vector>
+
+using Path = std::filesystem::path;
+
+class PathUtils
+{
+public:
+    static std::string getBaseFilename(const Path& path);
+
+    static Path getRelativePath(const Path& path, const Path& relativeTo);
+
+    static std::string getPathDelimited(const Path& path, char delimiter='-');
+};

src/base/core/http/HttpHeader.cpp

@@ -0,0 +1,97 @@
+#include "HttpHeader.h"
+#include "StringUtils.h"
+
+
+HttpHeader::HttpHeader()
+    :   mContentType("text / html"),
+        mHttpVersion("1.1")
+{
+
+}
+
+std::string HttpHeader::getContentType() const
+{
+    return mContentType;
+}
+
+std::string HttpHeader::getHttpVersion() const
+{
+    return mHttpVersion;
+}
+
+void HttpHeader::parse(const std::vector<std::string >& message)
+{
+    std::string tag;
+    std::string value;
+    bool foundDelimiter{false};
+    for (const auto& line : message)
+    {
+        for(std::size_t idx = 0; idx< line.size(); idx++)
+        {
+            const auto c = line[idx];
+            if (c == StringUtils::COLON)
+            {
+                foundDelimiter = true;
+            }
+            else if(foundDelimiter)
+            {
+                value.push_back(c);
+            }
+            else
+            {
+                tag.push_back(c);
+            }
+        }
+    }
+
+    if (tag.empty() || value.empty())
+    {
+        return;
+    }
+
+    if (tag == "Host")
+    {
+        mHost = value;
+    }
+    else if (tag == "User-Agent")
+    {
+        mUserAgent = value;
+    }
+    else if (tag == "Accept")
+    {
+        mAccept = value;
+    }
+    else if (tag == "Accept-Language")
+    {
+        mAcceptLanguage = value;
+    }
+    else if (tag == "Accept-Encoding")
+    {
+        mAcceptEncoding = value;
+    }
+    else if (tag == "Connection")
+    {
+        mConnection = value;
+    }
+    else if (tag == "Referer")
+    {
+        mReferer = value;
+    }
+    else if (tag == "Sec-Fetch-Dest")
+    {
+        mSecFetchDest = value;
+    }
+    else if (tag == "Sec-Fetch-Mode")
+    {
+        mSecFetchMode = value;
+    }
+    else if (tag == "Sec-Fetch-Site")
+    {
+        mSecFetchSite = value;
+    }
+    else
+    {
+        mOtherFields[tag] = value;
+    }
+
+}

src/base/core/http/HttpHeader.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <map>
+
+class HttpHeader
+{
+public:
+    HttpHeader();
+    void parse(const std::vector<std::string >& message);
+
+    std::string getContentType() const;
+
+    std::string getHttpVersion() const;
+
+private:
+    std::string mHttpVersion;
+    std::string mContentType;
+    std::string mHost;
+    std::string mUserAgent;
+    std::string mAccept;
+    std::string mAcceptLanguage;
+    std::string mAcceptEncoding;
+    std::string mConnection;
+    std::string mReferer;
+    std::string mSecFetchDest;
+    std::string mSecFetchMode;
+    std::string mSecFetchSite;
+
+    std::map<std::string, std::string> mOtherFields;
+};

src/base/core/http/HttpRequest.cpp

@@ -0,0 +1,85 @@
+#include "HttpRequest.h"
+
+#include "StringUtils.h"
+
+#include <sstream>
+
+HttpRequest::HttpRequest(Verb verb, const std::string& path)
+    :   mVerb(verb),
+        mPath(path)
+{
+
+}
+
+HttpRequest::Verb HttpRequest::getVerb() const
+{
+    return mVerb;
+}
+
+std::string HttpRequest::getPath() const
+{
+    return mPath;
+}
+
+void HttpRequest::parseMessage(const std::string& message)
+{
+    std::stringstream ss(message);
+
+    std::string buffer;
+    bool firstLine {true};
+
+    std::vector<std::string> headers;
+    while(std::getline(ss, buffer, '\n'))
+    {
+        if (firstLine)
+        {
+            parseFirstLine(buffer);
+            firstLine = false;
+        }
+        else
+        {
+            headers.push_back(buffer);
+        }
+    }
+    mHeader.parse(headers);
+}
+
+void HttpRequest::parseFirstLine(const std::string& line)
+{
+    bool inPath{false};
+    bool inMethod{true};
+    bool inProtocol{false};
+
+    for (std::size_t idx=0; idx<line.size();idx++)
+    {
+        const auto c = line[idx];
+        if (inPath)
+        {
+            if (StringUtils::isSpace(c))
+            {
+                inPath = false;
+                inMethod = true;
+            }
+            else
+            {
+                mMethod.push_back(c);
+            }
+        }
+        else if (inMethod)
+        {
+            if (StringUtils::isSpace(c))
+            {
+                inMethod = false;
+                inProtocol = true;
+            }
+            else
+            {
+                mPath.push_back(c);
+            }
+        }
+        else if (inProtocol)
+        {
+            mProtocolVersion.push_back(c);
+        }
+    }
+}

src/base/core/http/HttpRequest.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "HttpHeader.h"
+
+#include <string>
+
+class HttpRequest
+{
+public:
+
+    enum class Verb
+    {
+        GET,
+        PUT,
+        POST,
+        PATCH,
+        _DELETE,
+        UNKNOWN
+    };
+
+    HttpRequest() = default;
+
+    HttpRequest(Verb verb, const std::string& path);
+
+    Verb getVerb() const;
+
+    std::string getPath() const;
+
+    void parseMessage(const std::string& message);
+
+private:
+    void parseFirstLine(const std::string& line);
+
+    Verb mVerb = Verb::UNKNOWN;
+    HttpHeader mHeader;
+    std::string mMethod;
+    std::string mPath;
+    std::string mProtocolVersion;
+};

src/base/core/http/HttpResponse.cpp

@@ -0,0 +1,67 @@
+#include "HttpResponse.h"
+
+HttpResponse::HttpResponse()
+    :   mStatusCode(200),
+        mResponseReason("OK"),
+        mBody()
+{
+
+}
+
+HttpResponse::~HttpResponse()
+{
+
+}
+
+const HttpHeader& HttpResponse::getHeader() const
+{
+    return mHeader;
+}
+
+unsigned short HttpResponse::getStatusCode() const
+{
+    return mStatusCode;
+}
+
+const std::string& HttpResponse::getBody() const
+{
+    return mBody;
+}
+
+void HttpResponse::setBody(const std::string& body)
+{
+    mBody = body;
+}
+
+unsigned HttpResponse::getBodyLength() const
+{
+    return unsigned(mBody.length());
+}
+
+std::string HttpResponse::getHeaderString() const
+{
+    std::string header = "HTTP/" + mHeader.getHttpVersion() + " " + std::to_string(mStatusCode) + " " + mResponseReason + "\n";
+    header += "Content-Type: " + mHeader.getContentType() + "\n";
+    header += "Content-Length: " + std::to_string(getBodyLength()) + "\n";
+    return header;
+}
+
+const std::string& HttpResponse::getResponseReason() const
+{
+    return mResponseReason;
+}
+
+std::string HttpResponse::toString() const
+{
+    return getHeaderString() + "\n\n" + mBody;
+}
+
+void HttpResponse::setStatusCode(unsigned short code)
+{
+    mStatusCode = code;
+}
+
+void HttpResponse::setResponseReason(const std::string& reason)
+{
+    mResponseReason = reason;
+}

src/base/core/http/HttpResponse.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "HttpHeader.h"
+
+#include <string>
+
+class HttpResponse
+{
+public:
+    HttpResponse();
+
+    ~HttpResponse();
+
+    unsigned getBodyLength() const;
+
+    const std::string& getBody() const;
+
+    const HttpHeader& getHeader() const;
+
+    std::string getHeaderString() const;
+
+    std::string toString() const;
+
+    unsigned short getStatusCode() const;
+
+    const std::string& getResponseReason() const;
+
+    void setStatusCode(unsigned short code);
+
+    void setResponseReason(const std::string& reason);
+
+    void setBody(const std::string& body);
+
+private:
+    HttpHeader mHeader;
+    unsigned short mStatusCode{ 200 };
+    std::string mResponseReason{ };
+    std::string mBody;
+};

src/base/core/loggers/FileLogger.cpp

@@ -0,0 +1,73 @@
+#include "FileLogger.h"
+
+#include <time.h>
+#include <iomanip>
+#include <iostream>
+#include <filesystem>
+
+FileLogger::~FileLogger()
+{
+
+}
+
+void FileLogger::SetWorkDirectory(const std::string& workDir)
+{
+    mWorkDirectory = workDir;
+}
+
+void FileLogger::SetFileName(const std::string& fileName)
+{
+    mFileName = fileName;
+}
+
+void FileLogger::Open()
+{
+    if (mWorkDirectory.empty())
+    {
+        mWorkDirectory = std::filesystem::current_path().string();
+    }
+    mFileStream.open(mWorkDirectory + "/" + mFileName);
+}
+
+void FileLogger::Close()
+{
+    mFileStream.close();
+}
+
+void FileLogger::LogLine(const std::ostringstream& line)
+{
+    if (mDisabled)
+    {
+        return;
+    }
+
+    if (!mFileStream.is_open())
+    {
+        Open();
+    }
+    mFileStream << line.str() << std::endl;
+}
+
+void FileLogger::disable()
+{
+    mDisabled = true;
+}
+
+void FileLogger::LogLine(const std::string& logType, const std::ostringstream& line, const std::string& fileName, const std::string& functionName, int lineNumber)
+{
+    if (mDisabled)
+    {
+        return;
+    }
+
+    std::time_t t = std::time(nullptr);
+    const std::string cleanedFileName = fileName.substr(fileName.find_last_of("/\\") + 1);
+    std::tm time_buf = { 0 };
+#ifdef WIN32
+    gmtime_s(&time_buf, &t);
+#else
+    gmtime_r(&t, &time_buf);
+#endif
+    std::cout << logType << "|" << std::put_time(&time_buf, "%T") << "|" << cleanedFileName << "::" << functionName << "::" << lineNumber << "|" << line.str() << std::endl;
+    mFileStream << logType << "|" << std::put_time(&time_buf, "%T") << "|" << cleanedFileName << "::" << functionName << "::" << lineNumber << "|" << line.str() <<  std::endl;
+}

src/base/core/loggers/FileLogger.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#define MLOG_ALL(msg, level) {std::ostringstream mt_logstream;\
+    mt_logstream << msg; \
+    FileLogger::GetInstance().LogLine(level, mt_logstream, __FILE__, __FUNCTION__, __LINE__);};
+
+#define MLOG_INFO(msg) MLOG_ALL(msg, "Info");
+#define MLOG_ERROR(msg) MLOG_ALL(msg, "Error");
+
+#include <memory>
+#include <string>
+#include <fstream>
+#include <sstream>
+
+class FileLogger
+{
+    FileLogger()
+        :mWorkDirectory(),
+         mFileName("MT_Log.txt"),
+         mFileStream()
+    {
+
+    }
+
+public:
+    static FileLogger& GetInstance()
+    {
+        static FileLogger instance;
+        return instance;
+    }
+
+    FileLogger(FileLogger const&) = delete;
+    void operator=(FileLogger const&)  = delete;
+
+    ~FileLogger();
+
+    void disable();
+
+    void SetWorkDirectory(const std::string& workDir);
+
+    void SetFileName(const std::string& fileName);
+
+    void Open();
+
+    void Close();
+
+    void LogLine(const std::ostringstream& line);
+
+    void LogLine(const std::string& logType, const std::ostringstream& line, const std::string& fileName = "", const std::string& functionName = "", int lineNumber=-1);
+private:
+    bool mDisabled{false};
+    std::string mWorkDirectory;
+    std::string mFileName;
+    std::ofstream mFileStream;
+};
+
+using FileLoggerPtr = std::shared_ptr<FileLogger>;

src/base/core/memory/SharedMemory.cpp

@@ -0,0 +1,78 @@
+#include "SharedMemory.h"
+
+#include "RandomUtils.h"
+
+#ifdef __linux__
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#endif
+
+void SharedMemory::allocate(const std::string& namePrefix, std::size_t size)
+{
+    createFile(namePrefix);
+
+    if (!mIsValid)
+    {
+        return;
+    }
+
+#ifdef __linux__
+    int ret{-1};
+    do {
+        ret = ftruncate(mFileDescriptor, size);
+    } while (ret < 0 && errno == EINTR);
+
+    if (ret < 0)
+    {
+        close(mFileDescriptor);
+        mIsValid = false;
+    }
+#else
+    (void)(size);
+#endif
+}
+
+int SharedMemory::getFileDescriptor() const
+{
+    return mFileDescriptor;
+}
+
+bool SharedMemory::isValid() const
+{
+    return mIsValid;
+}
+
+void SharedMemory::createFile(const std::string& namePrefix)
+{
+#ifdef __linux__
+    unsigned retries = 100;
+    do {
+        const auto name = getRandomName(namePrefix);
+        --retries;
+
+        const int fd = shm_open(name.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600);
+        if (fd >= 0)
+        {
+            shm_unlink(name.c_str());
+            mFileDescriptor = fd;
+            mIsValid = true;
+            break;
+        }
+    } while (retries > 0 && errno == EEXIST);
+#else
+    (void)(namePrefix);
+#endif 
+}
+
+std::string SharedMemory::getRandomName(const std::string& namePrefix) const
+{
+    std::string randomSuffix;
+    for (const auto entry : RandomUtils::getRandomVecUnsigned(6))
+    {
+        randomSuffix += std::to_string(entry);
+    }
+    return namePrefix + randomSuffix;
+}

src/base/core/memory/SharedMemory.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <string>
+
+class SharedMemory
+{
+public:
+    void allocate(const std::string& namePrefix, std::size_t size);
+
+    int getFileDescriptor() const;
+
+    bool isValid() const;
+
+private:
+
+    void createFile(const std::string& namePrefix);
+
+    std::string getRandomName(const std::string& namePrefix) const;
+
+    int mFileDescriptor{0};
+    bool mIsValid{false};
+};

src/base/core/serializers/TomlReader.cpp

@@ -0,0 +1,151 @@
+#include "TomlReader.h"
+
+#include <iostream>
+#include <locale>
+#include <algorithm>
+
+TomlTable::TomlTable(const std::string& header)
+    : mHeader(header)
+{
+
+}
+
+void TomlTable::addComment(const Comment& comment)
+{
+    mComments.push_back(comment);
+}
+
+void TomlTable::addTable(std::unique_ptr<TomlTable> table)
+{
+    mTables[table->getHeader()] = std::move(table);
+}
+
+void TomlTable::addKeyValuePair(const std::string& key, const std::string& value)
+{
+    mMap[key] = value;
+}
+
+std::string TomlTable::getHeader() const
+{
+    return mHeader;
+}
+
+TomlTable* TomlTable::getTable(const std::string& path)
+{
+    return mTables[path].get();
+}
+
+TomlTable::KeyValuePairs TomlTable::getKeyValuePairs() const
+{
+    return mMap;
+}
+
+
+TomlContent::TomlContent()
+    : mRootTable(std::make_unique<TomlTable>("root"))
+{
+
+}
+
+TomlTable* TomlContent::getRootTable() const
+{
+    return mRootTable.get();
+}
+
+TomlTable* TomlContent::getTable(const std::string& path) const
+{
+    return mRootTable->getTable(path);
+}
+
+TomlReader::TomlReader()
+    : mContent(std::make_unique<TomlContent>())
+{
+
+}
+
+TomlContent* TomlReader::getContent() const
+{
+    return mContent.get();
+}
+
+void TomlReader::read(const Path& input_path)
+{
+    const auto lines = File(input_path).readLines();
+    mLastSectionOffset = 0;
+    mWorkingTable = mContent->getRootTable();
+
+    for (const auto& line : lines)
+    {
+        processLine(line);
+        mLastSectionOffset++;
+    }
+
+    mWorkingTable = nullptr;
+}
+
+void TomlReader::processLine(const std::string& line)
+{
+    bool in_comment{ false };
+    bool in_header{ false };
+    bool found_key{ false };
+    std::string working_string;
+    std::string key_string;
+    for (auto c : line)
+    {
+        if (c == '#' && !in_comment)
+        {
+            in_comment = true;
+        }
+        else if (c == '[' && !in_comment && !in_header)
+        {
+            in_header = true;
+        }
+        else if (c == '=' && !in_comment && !in_header)
+        {
+            found_key = true;
+            key_string = working_string;
+            working_string = "";
+        }
+        else if (c == ']' && in_header)
+        {
+            break;
+        }
+        else
+        {
+            working_string += c;
+        }
+    }
+
+    if (in_comment)
+    {
+        mWorkingTable->addComment({ mLastSectionOffset, working_string });
+    }
+    else if (in_header)
+    {
+        onHeader(working_string);
+    }
+    else if (found_key)
+    {
+        key_string.erase(std::remove_if(key_string.begin(), key_string.end(), [](char c) {return std::isspace(c); }), key_string.end());
+        working_string.erase(std::remove_if(working_string.begin(), working_string.end(), [](char c) {return std::isspace(c); }), working_string.end());
+        if (working_string.size()>2 && working_string[0] == '"' && working_string[working_string.size() - 1] == '"')
+        {
+            working_string = working_string.substr(1, working_string.size() - 2);
+        }
+
+        onKeyValuePair(key_string, working_string);
+    }
+}
+
+void TomlReader::onHeader(const std::string& header)
+{
+    auto new_table = std::make_unique<TomlTable>(header);
+    auto table_temp = new_table.get();
+    mWorkingTable->addTable(std::move(new_table));
+    mWorkingTable = table_temp;
+}
+
+void TomlReader::onKeyValuePair(const std::string key, const std::string value)
+{
+    mWorkingTable->addKeyValuePair(key, value);
+}

src/base/core/serializers/TomlReader.h

@@ -0,0 +1,72 @@
+#pragma once
+
+#include "File.h"
+
+#include <filesystem>
+#include <memory>
+#include <unordered_map>
+#include <string>
+
+using Path = std::filesystem::path;
+
+class TomlTable
+{
+public:
+    using Comment = std::pair<unsigned, std::string>;
+    using KeyValuePairs = std::unordered_map<std::string, std::string>;
+
+    TomlTable(const std::string& header);
+
+    void addComment(const Comment& comment);
+
+    void addTable(std::unique_ptr<TomlTable> table);
+
+    void addKeyValuePair(const std::string& key, const std::string& value);
+
+    std::string getHeader() const;
+
+    TomlTable* getTable(const std::string& path);
+
+    KeyValuePairs getKeyValuePairs() const;
+
+private:
+    unsigned mLineOffset{ 0 };
+    std::string mHeader;
+    std::unordered_map<std::string, std::unique_ptr<TomlTable> > mTables;
+    KeyValuePairs mMap;
+    std::vector<Comment> mComments;
+};
+
+class TomlContent
+{
+public:
+    TomlContent();
+
+    TomlTable* getRootTable() const;
+
+    TomlTable* getTable(const std::string& path) const;
+
+private:
+    std::unique_ptr<TomlTable> mRootTable;
+};
+
+class TomlReader
+{
+public:
+    TomlReader();
+
+    TomlContent* getContent() const;
+
+    void read(const Path& input_path);
+
+    void processLine(const std::string& line);
+
+    void onHeader(const std::string& header);
+
+    void onKeyValuePair(const std::string key, const std::string value);
+
+private:
+    unsigned mLastSectionOffset{ 0 };
+    std::unique_ptr<TomlContent> mContent;
+    TomlTable* mWorkingTable{nullptr};
+};

src/base/core/streams/AbstractChecksumCalculator.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "stdint.h"
+
+class AbstractChecksumCalculator
+{
+public:
+    virtual ~AbstractChecksumCalculator() = default;
+
+    virtual void addValue(unsigned char val) = 0;
+    virtual uint32_t getChecksum() const = 0;
+
+    virtual void reset()
+    {
+    }
+};

src/base/core/streams/BinaryStream.cpp

@@ -0,0 +1,97 @@
+#include "BinaryStream.h"
+
+std::optional<int> BinaryStream::getNextByteAsInt(std::basic_istream<char>* stream)
+{
+    return stream->get();
+}
+
+bool BinaryStream::getNextNBytes(std::basic_istream<char>* stream, char* buffer, unsigned number)
+{
+    char c;
+    for(unsigned idx=0; idx<number; idx++)
+    {
+        if(stream->get(c))
+        {
+            buffer[idx] = c;
+        }
+        else
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+bool BinaryStream::getNextWord(std::basic_istream<char>* stream, char* buffer)
+{
+    return getNextNBytes(stream, buffer, sizeof(ByteUtils::Word));
+}
+
+bool BinaryStream::getNextDWord(std::basic_istream<char>* stream, char* buffer)
+{
+    return getNextNBytes(stream, buffer, sizeof(ByteUtils::DWord));
+}
+
+bool BinaryStream::getNextQWord(std::basic_istream<char>* stream, char* buffer)
+{
+    return getNextNBytes(stream, buffer, sizeof(ByteUtils::QWord));
+}
+
+std::optional<ByteUtils::Word> BinaryStream::getNextWord(std::basic_istream<char>* stream, bool reverse)
+{
+    char buffer[sizeof(ByteUtils::Word)];
+    if(!BinaryStream::getNextWord(stream, buffer))
+    {
+        return false;
+    }
+    return ByteUtils::ToWord(buffer, reverse);
+}
+
+std::optional<ByteUtils::DWord> BinaryStream::getNextDWord(std::basic_istream<char>* stream)
+{
+    char buffer[sizeof(ByteUtils::DWord)];
+    if(!BinaryStream::getNextDWord(stream, buffer))
+    {
+        return false;
+    }
+    return ByteUtils::ToDWord(buffer);;
+}
+
+std::optional<ByteUtils::QWord> BinaryStream::getNextQWord(std::basic_istream<char>* stream)
+{
+    char buffer[sizeof(ByteUtils::QWord)];
+    if(!BinaryStream::getNextQWord(stream, buffer))
+    {
+        return false;
+    }
+    return ByteUtils::ToQWord(buffer);;
+}
+
+bool BinaryStream::checkNextDWord(std::basic_istream<char>* stream, const char* target)
+{
+    char buffer[sizeof(ByteUtils::DWord)];
+    if(!BinaryStream::getNextDWord(stream, buffer))
+    {
+        return false;
+    }
+
+    if(!ByteUtils::CompareDWords(buffer, target))
+    {
+        return false;
+    }
+    return true;
+}
+
+bool BinaryStream::getNextString(std::basic_istream<char>* stream, std::string& target, unsigned numBytes)
+{
+    char c;
+    for(unsigned idx=0; idx<numBytes; idx++)
+    {
+        if(!stream->get(c))
+        {
+            return false;
+        }
+        target += c;
+    }
+    return true;
+}

src/base/core/streams/BinaryStream.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <istream>
+#include <ostream>
+#include <string>
+#include <optional>
+
+#include "ByteUtils.h"
+
+class BinaryStream
+{
+public:
+
+    template<typename T>
+    static bool write(std::basic_ostream<char>* stream, T data)
+    {
+        stream->write(reinterpret_cast<char*>(&data), sizeof(data));
+        return true;
+    }
+
+    static std::optional<int> getNextByteAsInt(std::basic_istream<char>* stream);
+
+    static bool getNextNBytes(std::basic_istream<char>* stream, char* buffer, unsigned numBytes);
+
+    static bool getNextWord(std::basic_istream<char>* stream, char* buffer);
+
+    static bool getNextDWord(std::basic_istream<char>* stream, char* buffer);
+
+    static bool getNextQWord(std::basic_istream<char>* stream, char* buffer);
+
+    static std::optional<ByteUtils::Word> getNextWord(std::basic_istream<char>* stream, bool reverse = true);
+
+    static std::optional<ByteUtils::DWord> getNextDWord(std::basic_istream<char>* stream);
+
+    static std::optional<ByteUtils::QWord> getNextQWord(std::basic_istream<char>* stream);
+
+    static bool checkNextDWord(std::basic_istream<char>* stream, const char* target);
+
+    static bool getNextString(std::basic_istream<char>* stream, std::string& target, unsigned numBytes);
+};

src/base/core/streams/BitStream.cpp

@@ -0,0 +1,147 @@
+#include "BitStream.h"
+
+#include "ByteUtils.h"
+
+#include <sstream>
+#include <iostream>
+
+BitStream::~BitStream()
+{
+
+}
+
+unsigned char BitStream::getCurrentByte()
+{
+    if (mByteOffset < 0)
+    {
+        readNextByte();
+    }
+    return mCurrentByte;
+}
+
+void BitStream::write(uint32_t data)
+{
+    unsigned num_bytes = sizeof(uint32_t);
+    for(unsigned idx=0; idx<num_bytes;idx++)
+    {
+        writeByte(ByteUtils::getByteN(data, idx));
+    }
+}
+
+void BitStream::writeWord(uint16_t data)
+{
+    const auto byte0 = static_cast<unsigned char>(data >> 8);
+    const auto byte1 = static_cast<unsigned char>((data << 8) >> 8);
+    writeByte(byte0);
+    writeByte(byte1);
+}
+
+int BitStream::getCurrentByteOffset() const
+{
+    return mByteOffset;
+}
+
+unsigned BitStream::getCurrentBitOffset() const
+{
+    return mBitOffset;
+}
+
+std::string BitStream::logLocation()
+{
+    std::stringstream sstr;
+    sstr << "Byte offset " << mByteOffset<< " | Bit offset " << mBitOffset;
+    sstr << " | Working byte " << ByteUtils::toString(getCurrentByte()) << '\n';
+    return sstr.str();
+}
+
+std::string BitStream::logNextNBytes(unsigned n) const
+{
+    std::stringstream sstr;
+    unsigned count{0};
+    for(auto byte : peekNextNBytes(n))
+    {
+        sstr << mByteOffset + count << " | " << ByteUtils::toString(byte) + '\n';
+        count++;
+    }
+    return sstr.str();
+}
+
+void BitStream::writeNBits(uint32_t data, unsigned length)
+{
+    const auto num_left = 8 - mBitOffset;
+    const int overshoot = length - num_left;
+
+    if (overshoot > 0)
+    {
+        unsigned char lower_bits = ByteUtils::getLowerNBits(data, num_left);
+        mCurrentByte |= lower_bits << mBitOffset;
+
+        writeByte(mCurrentByte, false);
+
+        unsigned num_bytes = overshoot / 8;
+        for (unsigned idx=0; idx< num_bytes; idx++)
+        {
+            mCurrentByte = ByteUtils::getMBitsAtN(static_cast<unsigned char>(data), overshoot, idx*8 + num_left);
+            writeByte(mCurrentByte, false);
+        }
+
+        if (const auto remainder = overshoot % 8; remainder > 0)
+        {
+            mCurrentByte = ByteUtils::getMBitsAtN(static_cast<unsigned char>(data), remainder, num_bytes*8 + num_left);
+            mBitOffset = remainder;
+        }
+        else
+        {
+            mCurrentByte = 0;
+            mBitOffset = 0;
+        }
+    }
+    else
+    {
+        mCurrentByte |= (static_cast<unsigned char>(data) << mBitOffset);
+        mBitOffset += length;
+        if (mBitOffset == 8)
+        {
+            writeByte(mCurrentByte, false);
+            mCurrentByte = 0;
+            mBitOffset = 0;
+        }
+    }
+
+}
+
+bool BitStream::readNextNBits(unsigned n, unsigned char& buffer)
+{
+    if (mByteOffset < 0)
+    {
+        if (!readNextByte())
+        {
+            return false;
+        }
+    }
+
+    int overshoot = n + mBitOffset - 8;
+    if (overshoot > 0)
+    {
+        unsigned char last_byte = mCurrentByte;
+        if (!readNextByte())
+        {
+            return false;
+        }
+
+        auto num_lower = 8 - mBitOffset;
+        char lower_bits = ByteUtils::getHigherNBits(last_byte, num_lower);
+        char higher_bits = ByteUtils::getLowerNBits(mCurrentByte, overshoot);
+
+        buffer = (higher_bits << num_lower) | lower_bits;
+
+        mBitOffset = overshoot;
+        return true;
+    }
+    else
+    {
+        buffer = ByteUtils::getMBitsAtN(mCurrentByte, n, mBitOffset);
+        mBitOffset += n;
+        return true;
+    }
+}

src/base/core/streams/BitStream.h

@@ -0,0 +1,92 @@
+#pragma once
+
+#include "AbstractChecksumCalculator.h"
+
+#include <vector>
+#include <string>
+#include <optional>
+
+class BitStream
+{
+public:
+    virtual ~BitStream();
+
+    unsigned char getCurrentByte();
+
+    int getCurrentByteOffset() const;
+
+    unsigned getCurrentBitOffset() const;
+
+    virtual bool isFinished() const = 0;
+
+    std::string logNextNBytes(unsigned n) const;
+
+    std::string logLocation();
+
+    virtual std::vector<unsigned char> peekNextNBytes(unsigned n) const = 0;
+
+    virtual bool readNextNBits(unsigned n, unsigned char& buffer);
+
+    virtual std::optional<unsigned char> readNextByte() = 0;
+
+    virtual void writeNBits(uint32_t data, unsigned length);
+
+    virtual void writeByte(unsigned char data, bool checkOverflow = true) = 0;
+
+    void write(uint32_t data);
+
+    void writeWord(uint16_t data);
+
+    virtual void writeBytes(const std::vector<unsigned char> data) = 0;
+
+    void resetOffsets()
+    {
+        mEndByteOffset = mByteOffset;
+        mEndBitOffset = mBitOffset;
+        mEndByte = mCurrentByte;
+
+        mCurrentByte = 0;
+        mByteOffset = -1;
+        mBitOffset = 0;
+    }
+
+    virtual void reset()
+    {
+        resetOffsets();
+        mCurrentByte = 0;
+    }
+
+    void flushRemainingBits()
+    {
+        if (mBitOffset > 0)
+        {
+            writeByte(mCurrentByte, false);
+            mBitOffset = 0;
+        }
+    }
+
+    std::pair<unsigned char, unsigned> getRemainingBits() const
+    {
+        return {mEndByte, mEndBitOffset};
+    }
+
+    void setChecksumCalculator(AbstractChecksumCalculator* calc)
+    {
+        mChecksumCalculator = calc;
+    }
+
+    void clearChecksumCalculator()
+    {
+        mChecksumCalculator = nullptr;
+    }
+
+protected:
+    int mByteOffset{-1};
+    unsigned mBitOffset{0};
+    unsigned char mCurrentByte{0};
+
+    int mEndByteOffset{-1};
+    unsigned mEndBitOffset{0};
+    unsigned char mEndByte{0};
+    AbstractChecksumCalculator* mChecksumCalculator{nullptr};
+};

src/base/core/streams/BufferBitStream.cpp

@@ -0,0 +1,78 @@
+#include "BufferBitStream.h"
+
+#include "ByteUtils.h"
+
+#include <iostream>
+
+bool BufferBitStream::isFinished() const
+{
+    return mByteOffset == mBuffer.size() - 1;
+}
+
+std::vector<unsigned char> BufferBitStream::peekNextNBytes(unsigned n) const
+{
+    std::vector<unsigned char> ret (n, 0);
+    unsigned count = 0;
+
+    int start = mByteOffset;
+    if (start<0)
+    {
+        start = 0;
+    }
+
+    for(unsigned idx=start; idx<start + n; idx++)
+    {
+        if (idx == mBuffer.size())
+        {
+            break;
+        }
+
+        ret[count] = mBuffer[idx];
+        count ++;
+    }
+    return ret;
+}
+
+std::optional<unsigned char> BufferBitStream::readNextByte()
+{
+    if (mByteOffset + 1 == mBuffer.size())
+    {
+        return std::nullopt;
+    }
+    else
+    {
+        mByteOffset++;
+        mCurrentByte = mBuffer[mByteOffset];
+        return mCurrentByte;
+    }
+}
+
+void BufferBitStream::setBuffer(const std::vector<unsigned char>& data)
+{
+    mBuffer = data;
+}
+
+void BufferBitStream::writeByte(unsigned char data, bool checkOverflow)
+{
+    unsigned char out_byte{0};
+    if (checkOverflow && mBitOffset > 0)
+    {
+        out_byte = ByteUtils::getLowerNBits(mCurrentByte, mBitOffset);
+        out_byte |= data << mBitOffset;
+
+        mCurrentByte = ByteUtils::getHigherNBits(data, mBitOffset);
+    }
+    else
+    {
+        out_byte = data;
+    }
+
+    if (mChecksumCalculator)
+    {
+        mChecksumCalculator->addValue(out_byte);
+    }
+    //std::cout << "Writing byte " << ByteUtils::toString(out_byte) << " had bitoffset of " << mBitOffset << std::endl;
+    mBuffer.push_back(out_byte);
+}
+
+

src/base/core/streams/BufferBitStream.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "BitStream.h"
+
+#include <vector>
+#include <iterator>
+
+class BufferBitStream : public BitStream
+{
+public:
+    bool isFinished() const override;
+
+    std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
+
+    std::optional<unsigned char> readNextByte() override;
+
+    void setBuffer(const std::vector<unsigned char>& data);
+
+    void writeByte(unsigned char data, bool checkOverflow = true) override;
+
+    void writeBytes(const std::vector<unsigned char> data) override
+    {
+        std::copy(data.begin(), data.end(), std::back_inserter(mBuffer));
+    }
+
+    const std::vector<unsigned char>& getBuffer() const
+    {
+        return mBuffer;
+    }
+
+    void reset() override
+    {
+        BitStream::reset();
+        mBuffer.clear();
+    }
+
+private:
+    unsigned mBufferSize{0};
+    std::vector<unsigned char> mBuffer;
+};

src/base/core/streams/InputBitStream.cpp

@@ -0,0 +1,35 @@
+#include "InputBitStream.h"
+
+InputBitStream::InputBitStream(std::basic_istream<unsigned char>* stream)
+    : BitStream(),
+      mStream(stream)
+{
+
+}
+
+bool InputBitStream::isFinished() const
+{
+    return mStream->good();
+}
+
+std::vector<unsigned char> InputBitStream::peekNextNBytes(unsigned) const
+{
+    return {};
+}
+
+std::optional<unsigned char> InputBitStream::readNextByte()
+{
+    if (mStream->good())
+    {
+        return static_cast<unsigned char>(mStream->get());
+    }
+    else
+    {
+        return std::nullopt;
+    }
+}
+
+void InputBitStream::writeByte(unsigned char, bool)
+{
+
+}

src/base/core/streams/InputBitStream.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "BitStream.h"
+
+#include <istream>
+
+class InputBitStream : public BitStream
+{
+    InputBitStream(std::basic_istream<unsigned char>* stream);
+
+    bool isFinished() const override;
+
+    std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
+
+    std::optional<unsigned char> readNextByte() override;
+
+    void writeByte(unsigned char data, bool checkOverflow = true) override;
+
+    void writeBytes(const std::vector<unsigned char> data) override
+    {
+
+    }
+
+private:
+    std::basic_istream<unsigned char>* mStream{nullptr};
+};

src/base/core/streams/OutputBitStream.cpp

@@ -0,0 +1,36 @@
+#include "OutputBitStream.h"
+
+OutputBitStream::OutputBitStream(std::basic_ostream<char>* stream)
+    : BitStream(),
+      mStream(stream)
+{
+
+}
+
+bool OutputBitStream::isFinished() const
+{
+    return true;
+}
+
+std::vector<unsigned char> OutputBitStream::peekNextNBytes(unsigned) const
+{
+    return {};
+}
+
+std::optional<unsigned char> OutputBitStream::readNextByte()
+{
+    return std::nullopt;
+}
+
+void OutputBitStream::writeByte(unsigned char data, bool)
+{
+    (*mStream) << data;
+}
+
+void OutputBitStream::writeBytes(const std::vector<unsigned char> data)
+{
+    for(auto byte : data)
+    {
+        writeByte(byte);
+    }
+}

src/base/core/streams/OutputBitStream.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "BitStream.h"
+
+#include <ostream>
+
+class OutputBitStream : public BitStream
+{
+public:
+    OutputBitStream(std::basic_ostream<char>* stream);
+
+    bool isFinished() const override;
+
+    std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
+
+    std::optional<unsigned char> readNextByte() override;
+
+    void writeByte(unsigned char data, bool checkOverflow = true) override;
+
+    void writeBytes(const std::vector<unsigned char> data) override;
+
+private:
+    std::basic_ostream<char>* mStream{nullptr};
+};

src/base/database/CMakeLists.txt

@@ -0,0 +1,28 @@
+set(MODULE_NAME database)
+
+set(SQLite3_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/src/third_party/sqlite3")
+set(SQLite3_SOURCE_FILE "${CMAKE_SOURCE_DIR}/src/third_party/sqlite3/sqlite3.c")
+
+list(APPEND HEADERS 
+    Database.h
+    DatabaseManager.h
+    database_interfaces/SqliteInterface.h
+	${SQLite3_SOURCE_FILE})
+
+list(APPEND SOURCES 
+    Database.cpp
+    DatabaseManager.cpp
+    database_interfaces/SqliteInterface.cpp
+	${SQLite3_SOURCE_FILE})
+
+add_library(${MODULE_NAME} SHARED ${SOURCES} ${HEADERS})
+
+target_include_directories(${MODULE_NAME} PUBLIC
+                          ${CMAKE_CURRENT_SOURCE_DIR}
+                          ${CMAKE_CURRENT_SOURCE_DIR}/database_interfaces
+                          ${SQLite3_INCLUDE_DIR}
+                          )
+                          
+target_link_libraries(${MODULE_NAME} core)
+set_property(TARGET ${MODULE_NAME} PROPERTY FOLDER src/base)
+set_target_properties( ${MODULE_NAME} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON )

src/base/database/Database.cpp

@@ -0,0 +1,27 @@
+#include "Database.h"
+
+Database::Database()
+    : mPath()
+{
+
+}
+
+Database::~Database()
+{
+
+}
+
+std::unique_ptr<Database> Database::Create()
+{
+    return std::make_unique<Database>();
+}
+
+void Database::setPath(const Path& path)
+{
+    mPath = path;
+}
+
+const Path& Database::getPath() const
+{
+    return mPath;
+}