Further compression and png work.

src/compression/AbstractEncoder.h

@@ -0,0 +1,23 @@
+#pragma once
+
+class BitStream;
+
+class AbstractEncoder
+{
+public:
+    AbstractEncoder(BitStream* inputStream, BitStream* outputStream)
+        : mInputStream(inputStream),
+          mOutputStream(outputStream)
+    {
+
+    }
+
+    virtual ~AbstractEncoder() = default;
+
+    virtual bool encode() = 0;
+    virtual bool decode() = 0;
+
+protected:
+    BitStream* mInputStream{nullptr};
+    BitStream* mOutputStream{nullptr};
+};

src/compression/CMakeLists.txt

@@ -4,6 +4,10 @@ list(APPEND compression_LIB_INCLUDES
     HuffmanEncoder.cpp
     RunLengthEncoder.cpp
     ZlibData.cpp
+    ZlibEncoder.cpp
+    DeflateEncoder.cpp
+    DeflateBlock.cpp
+    Lz77Encoder.cpp
     )
 
 add_library(compression SHARED ${compression_LIB_INCLUDES})

src/compression/DeflateBlock.cpp

@@ -0,0 +1,262 @@
+#include "DeflateBlock.h"
+
+#include "ByteUtils.h"
+
+#include <algorithm>
+#include <iostream>
+
+DeflateBlock::DeflateBlock(BitStream* inputStream, BitStream* outputStream)
+    : mInputStream(inputStream),
+      mOutputStream(outputStream)
+{
+
+}
+
+bool DeflateBlock::readNextCodeLengthSymbol(unsigned char& final_symbol)
+{
+    unsigned working_index{0};
+    auto count = mCodeLengthMapping[working_index].first;
+    auto delta = count;
+
+    bool found{false};
+    unsigned char buffer{0};
+    unsigned char working_bits{0};
+    unsigned working_symbol{0};
+
+    while(!found)
+    {
+        auto valid = mInputStream->readNextNBits(delta, buffer);
+        working_bits = (working_bits << delta) | buffer;
+
+        for(const auto& entry : mCodeLengthMapping[working_index].second)
+        {
+            if (entry.first == working_bits)
+            {
+                found = true;
+                working_symbol = entry.second;
+                break;
+            }
+        }
+
+        if (!found)
+        {
+            working_index++;
+            if (working_index >= mCodeLengthMapping.size())
+            {
+                break;
+            }
+
+            auto new_count = mCodeLengthMapping[working_index].first;
+            delta = new_count - count;
+            count = new_count;
+        }
+    }
+
+    if (found)
+    {
+        final_symbol = working_symbol;
+        std::cout << "Found symbol " << working_symbol << " with bits " << ByteUtils::toString(working_bits) << std::endl;
+        std::cout << "At Byte offset " << mInputStream->getCurrentByteOffset() << " and bit offset " << mInputStream->getCurrentBitOffset() << std::endl;
+        return true;
+    }
+    else
+    {
+        std::cout << "SYMBOL NOT FOUND " << " with bits " << ByteUtils::toString(working_bits) << std::endl;
+        return false;
+    }
+}
+
+void DeflateBlock::setCodeLengthAlphabetLengths(const std::vector<unsigned char>& lengths)
+{
+    mCodeLengthAlphabetLengths = lengths;
+}
+
+void DeflateBlock::setCodeLengthLength(unsigned length)
+{
+    mHclen = length;
+}
+
+void DeflateBlock::setLiteralsTableLength(unsigned length)
+{
+    mHlit = length;
+}
+
+void DeflateBlock::setDistanceTableLength(unsigned length)
+{
+    mHdist = length;
+}
+
+void DeflateBlock::setIsFinalBlock(bool isFinal)
+{
+
+}
+
+void DeflateBlock::flushToStream()
+{
+
+}
+
+void DeflateBlock::readLiteralCodeLengths()
+{
+    std::vector<unsigned> lengths;
+    unsigned char symbol{0};
+
+    while(lengths.size() < mHlit)
+    {
+        bool valid = readNextCodeLengthSymbol(symbol);
+
+        if (!valid)
+        {
+            std::cout << "Hit unknown symbol - bailing out" << std::endl;
+            break;
+        }
+
+        if (symbol < 16)
+        {
+            lengths.push_back(symbol);
+        }
+        else if(symbol == 16)
+        {
+            unsigned char num_reps{0};
+            mInputStream->readNextNBits(2, num_reps);
+
+            auto last_val = lengths[lengths.size()-1];
+            std::cout << "Got val 16 doing " << 3 + num_reps << std::endl;
+            for(unsigned idx=0; idx< 3 + num_reps; idx++)
+            {
+                lengths.push_back(last_val);
+            }
+        }
+        else if(symbol == 17)
+        {
+            unsigned char num_reps{0};
+            mInputStream->readNextNBits(3, num_reps);
+
+            std::cout << "Got val 17 doing " << 3 + num_reps << std::endl;
+            for(unsigned idx=0; idx< 3 + num_reps; idx++)
+            {
+                lengths.push_back(0);
+            }
+        }
+        else if(symbol == 18)
+        {
+            unsigned char num_reps{0};
+            mInputStream->readNextNBits(7, num_reps);
+
+            std::cout << "Got val 18 doing " << 11 + num_reps << std::endl;
+            for(unsigned idx=0; idx< 11 + num_reps; idx++)
+            {
+                lengths.push_back(0);
+            }
+        }
+    }
+}
+
+void DeflateBlock::buildCodeLengthMapping()
+{
+    for(unsigned idx=1; idx<8; idx++)
+    {
+        std::vector<unsigned> entries;
+        for(unsigned jdx=0; jdx<mCodeLengthAlphabetLengths.size(); jdx++)
+        {
+            if (mCodeLengthAlphabetLengths[jdx] == idx)
+            {
+                entries.push_back(jdx);
+            }
+        }
+
+        if (entries.empty())
+        {
+            continue;
+        }
+
+        CodeLengthCountEntry count_entry{idx, {}};
+        std::sort(entries.begin(), entries.end());
+
+        unsigned char offset = 0x01 << idx - 1;
+        unsigned char count{0};
+        for (auto entry : entries)
+        {
+            count_entry.second.push_back(CodeLengthEntry{offset + count, entry});
+            count++;
+        }
+        mCodeLengthMapping.push_back(count_entry);
+    }
+
+    for (const auto& map_data : mCodeLengthMapping)
+    {
+        std::cout << "Map entry " << map_data.first << " has vals: "  << std::endl;
+        for (const auto& entry : map_data.second)
+        {
+            std::cout << "Key " << ByteUtils::toString(entry.first) << " val: " << entry.second << std::endl;
+        }
+    }
+}
+
+void DeflateBlock::readDynamicHuffmanTable()
+{
+    unsigned char h_lit{0};
+    mInputStream->readNextNBits(5, h_lit);
+    mHlit = h_lit + 257;
+    std::cout << "Got HLIT " << mHlit << std::endl;
+
+    unsigned char h_dist{0};
+    mInputStream->readNextNBits(5, h_dist);
+    mHdist = h_dist + 1;
+    std::cout << "Got HDIST " << mHdist << std::endl;
+
+    unsigned char h_clen{0};
+    mInputStream->readNextNBits(4, h_clen);
+    mHclen = h_clen + 4;
+    std::cout << "Got HCLEN " << mHclen << std::endl;
+
+    mCodeLengthAlphabetLengths = std::vector<unsigned char>(19, 0);
+    unsigned char buffer{0};
+    for(unsigned idx = 0; idx< mHclen; idx++)
+    {
+        mInputStream->readNextNBits(3, buffer);
+        mCodeLengthAlphabetLengths[CODE_LENGTH_ALPHABET_PERMUTATION[idx]] = buffer;
+        std::cout << "Got code length for " << CODE_LENGTH_ALPHABET_PERMUTATION[idx] << " of " << static_cast<unsigned>(buffer) << std::endl;
+    }
+
+    buildCodeLengthMapping();
+
+    readLiteralCodeLengths();
+}
+
+void DeflateBlock::readHeader()
+{
+    auto working_byte = mInputStream->getCurrentByte();
+    std::cout << "Into process data "<< std::endl;
+    std::cout << mInputStream->logNextNBytes(9);
+
+    unsigned char final_block{0};
+    mInputStream->readNextNBits(1, final_block);
+    mInFinalBlock = bool(final_block);
+
+    if (mInFinalBlock)
+    {
+        std::cout << "Got final block" << std::endl;
+    }
+
+    mInputStream->readNextNBits(2, mCompressionType);
+    std::cout << "Compress type byte is: " << static_cast<unsigned>(mCompressionType) << std::endl;
+    if (mCompressionType == NO_COMPRESSION)
+    {
+        std::cout << "Got NO_COMPRESSION" << std::endl;
+    }
+    else if (mCompressionType == FIXED_HUFFMAN)
+    {
+        std::cout << "Got FIXED_HUFFMAN" << std::endl;
+    }
+    else if (mCompressionType == DYNAMIC_HUFFMAN)
+    {
+        std::cout << "Got DYNAMIC_HUFFMAN" << std::endl;
+        readDynamicHuffmanTable();
+    }
+    else if (mCompressionType == ERROR)
+    {
+        std::cout << "Got ERROR" << std::endl;
+    }
+
+}

src/compression/DeflateBlock.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include "BitStream.h"
+
+class DeflateBlock
+{
+public:
+    DeflateBlock(BitStream* inputStream, BitStream* outputStream);
+
+    void readHeader();
+
+    void readDynamicHuffmanTable();
+
+    void buildCodeLengthMapping();
+
+    void readLiteralCodeLengths();
+
+    bool readNextCodeLengthSymbol(unsigned char& buffer);
+
+    bool isFinalBlock() const
+    {
+        return mInFinalBlock;
+    }
+
+    void setCodeLengthAlphabetLengths(const std::vector<unsigned char>& lengths);
+
+    void setCodeLengthLength(unsigned length);
+
+    void setLiteralsTableLength(unsigned length);
+
+    void setDistanceTableLength(unsigned length);
+
+    void setIsFinalBlock(bool isFinal);
+
+    void flushToStream();
+
+private:
+    BitStream* mInputStream;
+    BitStream* mOutputStream;
+
+    unsigned mHlit{0};
+    unsigned mHdist{0};
+    unsigned mHclen{0};
+
+    using CodeLengthEntry = std::pair<unsigned char, unsigned>;
+    using CodeLengthCountEntry = std::pair<unsigned, std::vector<CodeLengthEntry> >;
+    std::vector<CodeLengthCountEntry> mCodeLengthMapping;
+
+    std::vector<unsigned char> mCodeLengthAlphabetLengths;
+    static constexpr unsigned CODE_LENGTH_ALPHABET_PERMUTATION[19]{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
+
+    unsigned char mCompressionType{0};
+
+    bool mInFinalBlock{false};
+
+    static constexpr unsigned char NO_COMPRESSION = 0x00;
+    static constexpr unsigned char FIXED_HUFFMAN = 0x01;
+    static constexpr unsigned char DYNAMIC_HUFFMAN = 0x02;
+    static constexpr unsigned char ERROR = 0x03;
+};

src/compression/DeflateEncoder.cpp

@@ -0,0 +1,38 @@
+#include "DeflateEncoder.h"
+
+#include "BitStream.h"
+#include "DeflateBlock.h"
+
+#include <iostream>
+
+DeflateEncoder::DeflateEncoder(BitStream* inputStream, BitStream* outputStream)
+    : AbstractEncoder(inputStream, outputStream)
+{
+
+}
+
+DeflateEncoder::~DeflateEncoder()
+{
+
+}
+
+bool DeflateEncoder::encode()
+{
+    return false;
+}
+
+bool DeflateEncoder::decode()
+{
+    auto working_block = std::make_unique<DeflateBlock>(mInputStream, mOutputStream);
+    working_block->readHeader();
+
+    DeflateBlock* raw_block = working_block.get();
+
+    while(!raw_block->isFinalBlock())
+    {
+        break;
+    }
+
+    return false;
+}
+

src/compression/DeflateEncoder.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "AbstractEncoder.h"
+
+#include <vector>
+#include <memory>
+
+class DeflateBlock;
+
+class DeflateEncoder : public AbstractEncoder
+{
+public:
+    DeflateEncoder(BitStream* inputStream, BitStream* outputStream);
+
+    ~DeflateEncoder();
+
+    bool encode() override;
+
+    bool decode() override;
+
+private:
+    std::vector<std::unique_ptr<DeflateBlock > > mBlocks;
+};
+
+

src/compression/Lz77Encoder.cpp

@@ -0,0 +1,170 @@
+#include "Lz77Encoder.h"
+
+#include "StringUtils.h"
+#include "BitStream.h"
+
+Lz77Encoder::Lz77Encoder(BitStream* inputStream, BitStream* outputStream)
+    : AbstractEncoder(inputStream, outputStream)
+{
+
+}
+
+
+unsigned Lz77Encoder::lookAheadForMatchingChars(std::vector<char>& matchBuffer, unsigned searchIndex, unsigned hitOffset, const std::string& stream, unsigned streamLoc)
+{
+    auto remaining_size = stream.size() - streamLoc;
+
+    unsigned num_hits{1};
+    for (unsigned jdx=1; jdx< remaining_size; jdx++)
+    {
+        char buffer_char{0};
+        if (searchIndex + jdx < mSearchBuffer.size())
+        {
+            buffer_char = mSearchBuffer[searchIndex + jdx];
+        }
+        else
+        {
+            buffer_char = stream[jdx - hitOffset];
+        }
+
+        auto lookahead_char = stream[streamLoc + jdx];
+        if (lookahead_char == buffer_char)
+        {
+            matchBuffer.push_back(buffer_char);
+            num_hits++;
+        }
+        else
+        {
+            break;
+        }
+    }
+    return num_hits;
+}
+
+void Lz77Encoder::lookThroughSearchBuffer(char searchChar, unsigned& hitLength, unsigned& hitOffset, const std::string& stream, unsigned streamLoc)
+{
+    for(unsigned idx=0; idx<mSearchBuffer.size(); idx++)
+    {
+        auto search_index = mSearchBuffer.size() - idx - 1;
+
+        if (auto buffer_char = mSearchBuffer[search_index]; buffer_char == searchChar)
+        {
+            std::vector<char> match_buffer{buffer_char};
+            auto num_hits = lookAheadForMatchingChars(match_buffer, search_index, idx, stream, streamLoc);
+
+            if (num_hits >= hitLength)
+            {
+                hitLength = num_hits;
+                hitOffset = idx + 1;
+            }
+        }
+    }
+}
+
+bool Lz77Encoder::encode()
+{
+    /*
+    unsigned loc{0};
+    std::string ret;
+
+    while(!mInputStream->isFinished())
+    {
+        auto search_char = stream[loc];
+
+        unsigned hit_length{0};
+        unsigned hit_offset{0};
+        lookThroughSearchBuffer(search_char, hit_length, hit_offset, stream, loc);
+
+        if (hit_length > 0)
+        {
+            ret += "@" + std::to_string(hit_offset) + "L" + std::to_string(hit_length);
+            loc+=hit_length;
+
+            auto hit_loc = mSearchBuffer.size() - hit_offset;
+            for(unsigned idx=hit_loc; idx<hit_loc + hit_length; idx++)
+            {
+                mSearchBuffer.push_back(mSearchBuffer[idx]);
+            }
+        }
+        else
+        {
+            ret += search_char;
+            mSearchBuffer.push_back(search_char);
+            loc++;
+        }
+    }
+
+    return ret;
+    */
+    return false;
+}
+
+bool Lz77Encoder::decode()
+{
+    /*
+    std::string ret;
+
+    unsigned loc{0};
+    while(loc < stream.size())
+    {
+        auto working_char = stream[loc];
+        if (working_char == '@')
+        {
+            unsigned loc_working = loc;
+
+            auto remainder = stream.size() - loc;
+            std::string offset;
+
+            unsigned length_loc{0};
+            for(unsigned jdx=0; jdx< remainder; jdx++)
+            {
+                loc++;
+
+                auto offset_char = stream[loc];
+                if (offset_char == 'L')
+                {
+                    loc++;
+                    break;
+                }
+                else
+                {
+                    offset += offset_char;
+                }
+            }
+            unsigned offset_amount = std::stoul(offset);
+
+            std::string length;
+            remainder = stream.size() - loc;
+
+            for(unsigned jdx=0; jdx< remainder; jdx++)
+            {
+                auto length_char = stream[loc];
+                if (StringUtils::IsAlphabetical(length_char) || length_char == '@')
+                {
+                    break;
+                }
+                else
+                {
+                    loc++;
+                    length += length_char;
+                }
+            }
+
+            unsigned length_amount = std::stoul(length);
+
+            auto buffer_index = ret.size() - offset_amount;
+            for(unsigned jdx=buffer_index;jdx<buffer_index+length_amount; jdx++)
+            {
+                ret += ret[jdx];
+            }
+        }
+        else
+        {
+            loc++;
+            ret += working_char;
+        }
+    }
+    return ret;
+    */
+    return false;
+}

src/compression/Lz77Encoder.h

@@ -1,168 +1,33 @@
 #pragma once
 
-#include "StringUtils.h"
+#include "AbstractEncoder.h"
 
 #include <string>
 #include <vector>
 
-class Lz77Encoder
+class Lz77Encoder : public AbstractEncoder
 {
 public:
-    using DataStream = std::vector<char>;
+    using Buffer = std::vector<unsigned char>;
 
-    unsigned lookAheadForMatchingChars(std::vector<char>& matchBuffer, unsigned searchIndex, unsigned hitOffset, const std::string& stream, unsigned streamLoc)
+    Lz77Encoder(BitStream* inputStream, BitStream* outputStream);
-    {
-        auto remaining_size = stream.size() - streamLoc;
 
-        unsigned num_hits{1};
+    unsigned lookAheadForMatchingChars(std::vector<char>& matchBuffer, unsigned searchIndex, unsigned hitOffset, const std::string& stream, unsigned streamLoc);
-        for (unsigned jdx=1; jdx< remaining_size; jdx++)
-        {
-            char buffer_char{0};
-            if (searchIndex + jdx < mSearchBuffer.size())
-            {
-                buffer_char = mSearchBuffer[searchIndex + jdx];
-            }
-            else
-            {
-                buffer_char = stream[jdx - hitOffset];
-            }
 
-            auto lookahead_char = stream[streamLoc + jdx];
+    void lookThroughSearchBuffer(char searchChar, unsigned& hitLength, unsigned& hitOffset, const std::string& stream, unsigned streamLoc);
-            if (lookahead_char == buffer_char)
-            {
-                matchBuffer.push_back(buffer_char);
-                num_hits++;
-            }
-            else
-            {
-                break;
-            }
-        }
-        return num_hits;
-    }
 
-    void lookThroughSearchBuffer(char searchChar, unsigned& hitLength, unsigned& hitOffset, const std::string& stream, unsigned streamLoc)
+    bool encode() override;
-    {
-        for(unsigned idx=0; idx<mSearchBuffer.size(); idx++)
-        {
-            auto search_index = mSearchBuffer.size() - idx - 1;
 
-            if (auto buffer_char = mSearchBuffer[search_index]; buffer_char == searchChar)
+    bool decode() override;
-            {
-                std::vector<char> match_buffer{buffer_char};
-                auto num_hits = lookAheadForMatchingChars(match_buffer, search_index, idx, stream, streamLoc);
 
-                if (num_hits >= hitLength)
+    void setSearchBufferSize(unsigned size);
-                {
-                    hitLength = num_hits;
-                    hitOffset = idx + 1;
-                }
-            }
-        }
-    }
 
-    std::string encode(const std::string& stream)
+    void setLookAheadBufferSize(unsigned size);
-    {
-        unsigned loc{0};
-        std::string ret;
 
-        while(loc < stream.size())
+private:
-        {
+    unsigned mSearchBufferSize{32000};
-            auto search_char = stream[loc];
+    Buffer mSearchBuffer;
 
-            unsigned hit_length{0};
+    unsigned mLookAheadBufferSize{256};
-            unsigned hit_offset{0};
+    Buffer mLookaheadBuffer;
-            lookThroughSearchBuffer(search_char, hit_length, hit_offset, stream, loc);
-
-            if (hit_length > 0)
-            {
-                ret += "@" + std::to_string(hit_offset) + "L" + std::to_string(hit_length);
-                loc+=hit_length;
-
-                auto hit_loc = mSearchBuffer.size() - hit_offset;
-                for(unsigned idx=hit_loc; idx<hit_loc + hit_length; idx++)
-                {
-                    mSearchBuffer.push_back(mSearchBuffer[idx]);
-                }
-            }
-            else
-            {
-                ret += search_char;
-                mSearchBuffer.push_back(search_char);
-                loc++;
-            }
-        }
-
-        return ret;
-    }
-
-    std::string decode(const std::string& stream)
-    {
-        std::string ret;
-
-        unsigned loc{0};
-        while(loc < stream.size())
-        {
-            auto working_char = stream[loc];
-            if (working_char == '@')
-            {
-                unsigned loc_working = loc;
-
-                auto remainder = stream.size() - loc;
-                std::string offset;
-
-                unsigned length_loc{0};
-                for(unsigned jdx=0; jdx< remainder; jdx++)
-                {
-                    loc++;
-
-                    auto offset_char = stream[loc];
-                    if (offset_char == 'L')
-                    {
-                        loc++;
-                        break;
-                    }
-                    else
-                    {
-                        offset += offset_char;
-                    }
-                }
-                unsigned offset_amount = std::stoul(offset);
-
-                std::string length;
-                remainder = stream.size() - loc;
-
-                for(unsigned jdx=0; jdx< remainder; jdx++)
-                {
-                    auto length_char = stream[loc];
-                    if (StringUtils::IsAlphabetical(length_char) || length_char == '@')
-                    {
-                        break;
-                    }
-                    else
-                    {
-                        loc++;
-                        length += length_char;
-                    }
-                }
-
-                unsigned length_amount = std::stoul(length);
-
-                auto buffer_index = ret.size() - offset_amount;
-                for(unsigned jdx=buffer_index;jdx<buffer_index+length_amount; jdx++)
-                {
-                    ret += ret[jdx];
-                }
-            }
-            else
-            {
-                loc++;
-                ret += working_char;
-            }
-        }
-        return ret;
-    }
-
-    DataStream mSearchBuffer;
-    DataStream mLookaheadBuffer;
 };

src/compression/ZlibEncoder.cpp

@@ -0,0 +1,74 @@
+#include "ZlibEncoder.h"
+
+#include "ByteUtils.h"
+#include "DeflateEncoder.h"
+#include "FileLogger.h"
+
+#include <iostream>
+
+ZlibEncoder::ZlibEncoder(BitStream* inputStream, BitStream* outputStream)
+: AbstractEncoder(inputStream, outputStream)
+{
+
+}
+
+ZlibEncoder::~ZlibEncoder()
+{
+
+}
+
+void ZlibEncoder::setCompressionMethod(unsigned char method)
+{
+    std::cout << "Got compression input " << static_cast<int>(method) << std::endl;
+    mCmf = method;
+    mCompressionMethod = ByteUtils::getLowerNBits(method, 4);
+    mCompressionInfo = ByteUtils::getHigherNBits(method, 4);
+
+    std::cout << "Got compression method " << static_cast<int>(mCompressionMethod) << " and info " << static_cast<int>(mCompressionInfo) << std::endl;
+}
+
+void ZlibEncoder::setExtraFlags(unsigned char extraFlags)
+{
+    std::cout << "Got flags  " << static_cast<int>(extraFlags) << std::endl;
+
+    mFlg = extraFlags;
+    mFlagCheck = ByteUtils::getLowerNBits(extraFlags, 5);
+    mFlagDict = ByteUtils::getBitN(extraFlags, 5);
+    mFlagLevel = ByteUtils::getHigherNBits(extraFlags, 2);
+
+    std::cout << "Got flag check " << static_cast<int>(mFlagCheck) << " and dict " << static_cast<int>(mFlagDict) << " and level " << static_cast<int>(mFlagLevel) << std::endl;
+}
+
+bool ZlibEncoder::encode()
+{
+    if (!mWorkingEncoder)
+    {
+        if (mCompressionMethod == 8)
+        {
+            mWorkingEncoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
+        }
+        else
+        {
+            MLOG_ERROR("Zib requested decoder not recognized: " << mCompressionMethod << " aborting encode");
+            return false;
+        }
+    }
+    return mWorkingEncoder->encode();
+}
+
+bool ZlibEncoder::decode()
+{
+    if (!mWorkingEncoder)
+    {
+        if (mCompressionMethod == 8)
+        {
+            mWorkingEncoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
+        }
+        else
+        {
+            MLOG_ERROR("Zib requested decoder not recognized: " << mCompressionMethod << " aborting decode");
+            return false;
+        }
+    }
+    return mWorkingEncoder->decode();
+}

src/compression/ZlibEncoder.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "AbstractEncoder.h"
+
+#include <memory>
+#include <vector>
+
+class ZlibEncoder : public AbstractEncoder
+{
+
+public:
+    ZlibEncoder(BitStream* inputStream, BitStream* outputStream);
+    ~ZlibEncoder();
+
+    void setCompressionMethod(unsigned char method);
+
+    void setExtraFlags(unsigned char extraFlags);
+
+    bool encode() override;
+
+    bool decode() override;
+
+private:
+    unsigned char mCmf{0};
+    unsigned char mFlg{0};
+    unsigned char mCompressionMethod{8};
+    unsigned char mCompressionInfo{0};
+    unsigned char mFlagCheck{0};
+    unsigned char mFlagDict{0};
+    unsigned char mFlagLevel{0};
+    unsigned char mCheckValue{0};
+
+    std::unique_ptr<AbstractEncoder> mWorkingEncoder;
+};

src/core/CMakeLists.txt

@@ -30,6 +30,9 @@ list(APPEND core_LIB_INCLUDES
     StringUtils.cpp
     streams/BinaryStream.cpp
     streams/BitStream.cpp
+    streams/InputBitStream.cpp
+    streams/OutputBitStream.cpp
+    streams/BufferBitStream.cpp
     http/HttpResponse.cpp
     http/HttpHeader.cpp
     http/HttpRequest.cpp

src/core/StringUtils.cpp

@@ -25,6 +25,16 @@ bool StringUtils::IsAlphabetical(char c)
     return std::isalpha(c);
 }
 
+std::vector<unsigned char> StringUtils::toBytes(const std::string& input)
+{
+    return {input.begin(), input.end()};
+}
+
+std::string StringUtils::toString(const std::vector<unsigned char>& bytes)
+{
+    return {bytes.begin(), bytes.end()};
+}
+
 std::vector<std::string> StringUtils::toLines(const std::string& input)
 {
     auto result = std::vector<std::string>{};

src/core/StringUtils.h

@@ -30,4 +30,8 @@ public:
     static std::vector<std::string> toLines(const std::string& input);
 
     static std::string stripQuotes(const std::string& input);
+
+    static std::vector<unsigned char> toBytes(const std::string& input);
+    static std::string toString(const std::vector<unsigned char>& bytes);
+
 };

src/core/streams/BitStream.cpp

@@ -2,37 +2,70 @@
 
 #include "ByteUtils.h"
 
-bool BitStream::loadNextByte()
+#include <sstream>
+#include <iostream>
+
+BitStream::~BitStream()
 {
-    if (mByteOffset + 1 == mBuffer.size())
+
+}
+
+unsigned char BitStream::getCurrentByte()
+{
+    if (mByteOffset < 0)
+    {
+        readNextByte();
+    }
+    return mCurrentByte;
+}
+
+
+int BitStream::getCurrentByteOffset() const
+{
+    return mByteOffset;
+}
+
+unsigned BitStream::getCurrentBitOffset() const
+{
+    return mBitOffset;
+}
+
+std::string BitStream::logNextNBytes(unsigned n) const
+{
+    std::stringstream sstr;
+    unsigned count{0};
+    for(auto byte : peekNextNBytes(n))
+    {
+        sstr << count << " | " << ByteUtils::toString(byte) + '\n';
+        count++;
+    }
+    return sstr.str();
+}
+
+bool BitStream::readNextNBits(unsigned n, unsigned char& buffer)
+{
+    if (mByteOffset < 0)
+    {
+        if (!readNextByte())
         {
             return false;
         }
-    else
-    {
-        mByteOffset++;
-        mCurrentByte = mBuffer[mByteOffset];
-        return true;
-    }
     }
 
-bool BitStream::getNextNBits(unsigned n, unsigned char& buffer)
+    int overshoot = n + mBitOffset - 8;
-{
-    int overshoot = n + mBitOffset - 7;
-
     if (overshoot > 0)
     {
         unsigned char last_byte = mCurrentByte;
-        if (!loadNextByte())
+        if (!readNextByte())
         {
             return false;
         }
 
-        auto num_lower = 7 - mBitOffset;
+        auto num_lower = 8 - mBitOffset;
         char lower_bits = ByteUtils::getHigherNBits(last_byte, num_lower);
         char higher_bits = ByteUtils::getLowerNBits(mCurrentByte, overshoot);
 
-        buffer = (higher_bits << (8 - num_lower)) | (lower_bits >> mBitOffset);
+        buffer = (higher_bits << num_lower) | lower_bits;
 
         mBitOffset = overshoot;
         return true;
@@ -44,13 +77,3 @@ bool BitStream::getNextNBits(unsigned n, unsigned char& buffer)
         return true;
     }
 }
-
-void BitStream::setByte(unsigned idx, unsigned char data)
-{
-    mBuffer[idx] = data;
-}
-
-void BitStream::setBufferSize(std::size_t size)
-{
-    mBuffer = std::vector<unsigned char>(size);
-}

src/core/streams/BitStream.h

@@ -1,27 +1,35 @@
 #pragma once
 
 #include <vector>
+#include <string>
+#include <optional>
 
 class BitStream
 {
 public:
-    bool getNextNBits(unsigned n, unsigned char& buffer);
+    virtual ~BitStream();
 
-    bool loadNextByte();
+    unsigned char getCurrentByte();
 
-    void setByte(unsigned idx, unsigned char data);
+    int getCurrentByteOffset() const;
 
-    void setBufferSize(std::size_t size);
+    unsigned getCurrentBitOffset() const;
 
-    unsigned char getCurrentByte() const
+    virtual bool isFinished() const = 0;
-    {
-        return mCurrentByte;
-    }
 
-private:
+    std::string logNextNBytes(unsigned n) const;
-    unsigned mByteOffset{0};
+
+    virtual std::vector<unsigned char> peekNextNBytes(unsigned n) const = 0;
+
+    virtual bool readNextNBits(unsigned n, unsigned char& buffer);
+
+    virtual std::optional<unsigned char> readNextByte() = 0;
+
+    virtual void writeByte(unsigned char data) = 0;
+
+protected:
+    int mByteOffset{0};
     unsigned mBitOffset{0};
 
-    char mCurrentByte{0};
+    unsigned char mCurrentByte{0};
-    std::vector<unsigned char> mBuffer;
 };

src/core/streams/BufferBitStream.cpp

@@ -0,0 +1,49 @@
+#include "BufferBitStream.h"
+
+bool BufferBitStream::isFinished() const
+{
+    return mByteOffset == mBuffer.size();
+}
+
+std::vector<unsigned char> BufferBitStream::peekNextNBytes(unsigned n) const
+{
+    std::vector<unsigned char> ret (n, 0);
+    unsigned count = 0;
+    for(unsigned idx=mByteOffset; idx<mByteOffset + n; idx++)
+    {
+        if (idx == mBuffer.size())
+        {
+            break;
+        }
+
+        ret[count] = mBuffer[idx];
+        count ++;
+    }
+    return ret;
+}
+
+std::optional<unsigned char> BufferBitStream::readNextByte()
+{
+    if (mByteOffset + 1 == mBuffer.size())
+    {
+        return std::nullopt;
+    }
+    else
+    {
+        mByteOffset++;
+        mCurrentByte = mBuffer[mByteOffset];
+        return mCurrentByte;
+    }
+}
+
+void BufferBitStream::setBuffer(const std::vector<unsigned char>& data)
+{
+    mBuffer = data;
+}
+
+void BufferBitStream::writeByte(unsigned char data)
+{
+    mBuffer.push_back(data);
+}
+
+

src/core/streams/BufferBitStream.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "BitStream.h"
+
+#include <vector>
+
+class BufferBitStream : public BitStream
+{
+public:
+    bool isFinished() const override;
+
+    std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
+
+    std::optional<unsigned char> readNextByte() override;
+
+    void setBuffer(const std::vector<unsigned char>& data);
+
+    void writeByte(unsigned char data) override;
+
+    const std::vector<unsigned char>& getBuffer() const
+    {
+        return mBuffer;
+    }
+
+private:
+    std::vector<unsigned char> mBuffer;
+};

src/core/streams/InputBitStream.cpp

@@ -0,0 +1,35 @@
+#include "InputBitStream.h"
+
+InputBitStream::InputBitStream(std::basic_istream<unsigned char>* stream)
+    : BitStream(),
+      mStream(stream)
+{
+
+}
+
+bool InputBitStream::isFinished() const
+{
+    return mStream->good();
+}
+
+std::vector<unsigned char> InputBitStream::peekNextNBytes(unsigned n) const
+{
+    return {};
+}
+
+std::optional<unsigned char> InputBitStream::readNextByte()
+{
+    if (mStream->good())
+    {
+        return mStream->get();
+    }
+    else
+    {
+        return std::nullopt;
+    }
+}
+
+void InputBitStream::writeByte(unsigned char data)
+{
+
+}

src/core/streams/InputBitStream.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "BitStream.h"
+
+#include <istream>
+
+class InputBitStream : public BitStream
+{
+    InputBitStream(std::basic_istream<unsigned char>* stream);
+
+    bool isFinished() const override;
+
+    std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
+
+    std::optional<unsigned char> readNextByte() override;
+
+    void writeByte(unsigned char data) override;
+
+private:
+    std::basic_istream<unsigned char>* mStream{nullptr};
+};

src/core/streams/OutputBitStream.cpp

@@ -0,0 +1,28 @@
+#include "OutputBitStream.h"
+
+OutputBitStream::OutputBitStream(std::basic_ostream<char>* stream)
+    : BitStream(),
+      mStream(stream)
+{
+
+}
+
+bool OutputBitStream::isFinished() const
+{
+    return true;
+}
+
+std::vector<unsigned char> OutputBitStream::peekNextNBytes(unsigned n) const
+{
+    return {};
+}
+
+std::optional<unsigned char> OutputBitStream::readNextByte()
+{
+    return std::nullopt;
+}
+
+void OutputBitStream::writeByte(unsigned char data)
+{
+    (*mStream) << data;
+}

src/core/streams/OutputBitStream.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "BitStream.h"
+
+#include <ostream>
+
+class OutputBitStream : public BitStream
+{
+public:
+    OutputBitStream(std::basic_ostream<char>* stream);
+
+    bool isFinished() const override;
+
+    std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
+
+    std::optional<unsigned char> readNextByte() override;
+
+    void writeByte(unsigned char data) override;
+
+private:
+    std::basic_ostream<char>* mStream{nullptr};
+};

src/image/CMakeLists.txt

@@ -7,6 +7,7 @@ list(APPEND image_LIB_INCLUDES
     Image.cpp
     PngWriter.cpp
     PngReader.cpp
+    ImageBitStream.cpp
     )
 
 list(APPEND image_LIBS core compression)

src/image/ImageBitStream.cpp

@@ -0,0 +1,29 @@
+#include "ImageBitStream.h"
+
+ImageBitStream::ImageBitStream(Image<unsigned char>* image)
+    :   BitStream(),
+        mImage(image)
+{
+
+}
+
+bool ImageBitStream::isFinished() const
+{
+    return true;
+}
+
+std::vector<unsigned char> ImageBitStream::peekNextNBytes(unsigned n) const
+{
+    return {};
+}
+
+std::optional<unsigned char> ImageBitStream::readNextByte()
+{
+    return std::nullopt;
+}
+
+void ImageBitStream::writeByte(unsigned char data)
+{
+
+}
+

src/image/ImageBitStream.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "BitStream.h"
+
+#include "Image.h"
+
+class ImageBitStream : public BitStream
+{
+public:
+    ImageBitStream(Image<unsigned char>* image);
+
+    bool isFinished() const override;
+
+    std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
+
+    std::optional<unsigned char> readNextByte() override;
+
+    void writeByte(unsigned char data) override;
+
+private:
+    Image<unsigned char>* mImage{nullptr};
+};

src/image/PngElements.h

@@ -3,9 +3,18 @@
 #include <string>
 #include <sstream>
 
-
 namespace Png
 {
+    inline unsigned char getHighBitCheck()
+    {
+        return 0x89;
+    }
+
+    inline std::vector<unsigned char> getSignature()
+    {
+        return {13, 10, 26, 10};
+    }
+
     struct IHDRChunk
     {
         unsigned width{0};

src/image/PngReader.cpp

@@ -1,6 +1,10 @@
 #include "PngReader.h"
 
 #include "BinaryStream.h"
+#include "BitStream.h"
+#include "BufferBitStream.h"
+
+#include "ZlibEncoder.h"
 
 #include <iostream>
 #include <sstream>
@@ -84,12 +88,11 @@ void PngReader::readIDATChunk(unsigned length)
 {
     if (mAwaitingDataBlock)
     {
-        mImageData.setCompressionMethod(mFile->readNextByte());
+        mEncoder->setCompressionMethod(mFile->readNextByte());
-        mImageData.setExtraFlags(mFile->readNextByte());
+        mEncoder->setExtraFlags(mFile->readNextByte());
-        mImageData.setDataSize(length-2);
         for(unsigned idx=0; idx<length-2; idx++)
         {
-            mImageData.setByte(idx, mFile->readNextByte());
+            mInputStream->writeByte(mFile->readNextByte());
         }
         mAwaitingDataBlock = false;
     }
@@ -97,7 +100,7 @@ void PngReader::readIDATChunk(unsigned length)
     {
         for(unsigned idx=0; idx<length; idx++)
         {
-            mImageData.setByte(idx, mFile->readNextByte());
+            mInputStream->writeByte(mFile->readNextByte());
         }
     }
 }
@@ -135,10 +138,14 @@ std::unique_ptr<Image<unsigned char> > PngReader::read()
         return image;
     }
 
+    mInputStream = std::make_unique<BufferBitStream>();
+    mOutputStream = std::make_unique<BufferBitStream>();
+    mEncoder = std::make_unique<ZlibEncoder>(mInputStream.get(), mOutputStream.get());
+
     while(readChunk())
     {
 
     }
-    mImageData.processData();
+    mEncoder->decode();
     return std::move(image);
 }

src/image/PngReader.h

@@ -3,12 +3,15 @@
 #include "File.h"
 #include "Image.h"
 #include "PngElements.h"
-#include "ZlibData.h"
+#include "ZlibEncoder.h"
 
 #include <string>
 #include <memory>
 #include <filesystem>
 
+class BitStream;
+class ZlibEncoder;
+
 using Path = std::filesystem::path;
 
 class PngReader
@@ -37,6 +40,8 @@ private:
     std::unique_ptr<File> mFile;
     Path mPath;
 
-    ZlibData mImageData;
+    std::unique_ptr<ZlibEncoder> mEncoder;
+    std::unique_ptr<BitStream> mInputStream;
+    std::unique_ptr<BitStream> mOutputStream;
     bool mAwaitingDataBlock{true};
 };

src/image/PngWriter.cpp

@@ -1,6 +1,13 @@
 #include "PngWriter.h"
 
+#include "PngElements.h"
 #include "Image.h"
+#include "File.h"
+#include "BufferBitStream.h"
+#include "OutputBitStream.h"
+#include "ImageBitStream.h"
+
+#include "Lz77Encoder.h"
 
 #include <stdio.h>
 
@@ -9,6 +16,11 @@ PngWriter::PngWriter()
 
 }
 
+PngWriter::~PngWriter()
+{
+
+}
+
 std::unique_ptr<PngWriter> PngWriter::Create()
 {
     return std::make_unique<PngWriter>();
@@ -19,8 +31,40 @@ void PngWriter::setPath(const Path& path)
     mPath = path;
 }
 
-void PngWriter::write(const std::unique_ptr<Image<unsigned char> >& image) const
+void PngWriter::writeSignature()
 {
+    mOutStream->writeByte(Png::getHighBitCheck());
+    for (auto byte : Png::getSignature())
+    {
+        mOutStream->writeByte(byte);
+    }
+}
+
+void PngWriter::writeHeader()
+{
+    writeSignature();
+}
+
+void PngWriter::write(const std::unique_ptr<Image<unsigned char> >& image)
+{
+    if (!mPath.empty())
+    {
+        mWorkingFile = std::make_unique<File>(mPath);
+        mWorkingFile->Open(true);
+        mOutStream = std::make_unique<OutputBitStream>(mWorkingFile->GetOutHandle());
+    }
+    else
+    {
+        mOutStream = std::make_unique<BufferBitStream>();
+    }
+
+    mWorkingImage = image.get();
+    mInStream = std::make_unique<ImageBitStream>(image.get());
+
+    writeHeader();
+
+
+
     //mImpl->write(image);
     //auto fp = fopen(mPath.c_str(), "wb");
 
@@ -76,4 +120,9 @@ void PngWriter::write(const std::unique_ptr<Image<unsigned char> >& image) const
 
     //fclose(fp);
     //return;
+
+    if (mWorkingFile)
+    {
+        mWorkingFile->Close();
+    }
 }

src/image/PngWriter.h

@@ -8,19 +8,31 @@
 
 using Path = std::filesystem::path;
 
+class BitStream;
+class File;
+
 class PngWriter
 {
 public:
     PngWriter();
+    ~PngWriter();
 
     static std::unique_ptr<PngWriter> Create();
 
     void setPath(const Path& path);
 
-    void write(const std::unique_ptr<Image<unsigned char> >& image) const;
+    void write(const std::unique_ptr<Image<unsigned char> >& image);
 
 private:
+    void writeSignature();
+    void writeHeader();
+
+
     Path mPath;
+    Image<unsigned char>* mWorkingImage{nullptr};
+    std::unique_ptr<BitStream> mInStream;
+    std::unique_ptr<BitStream> mOutStream;
+    std::unique_ptr<File> mWorkingFile;
 };
 
 using PngWriterPtr = std::unique_ptr<PngWriter>;

test/CMakeLists.txt

@@ -12,6 +12,7 @@ list(APPEND TestFiles
             audio/TestMidiReader.cpp
             core/TestByteUtils.cpp
             core/TestBinaryStream.cpp
+            core/TestBitStream.cpp
             core/TestTomlReader.cpp
             compiler/TestLexer.cpp
             compiler/TestTemplatingEngine.cpp

test/compression/TestStreamCompressor.cpp

@@ -1,5 +1,6 @@
 #include <iostream>
 
+#include "BufferBitStream.h"
 #include "HuffmanEncoder.h"
 #include "RunLengthEncoder.h"
 #include "Lz77Encoder.h"
@@ -42,10 +43,14 @@ void test_lz77_encoder()
     std::string test_data = "sir sid eastman easily teases sea sick seals";
     //std::string test_data = "sir sid eastman";
 
-    Lz77Encoder encoder;
+    BufferBitStream input_stream;
-    auto encoded = encoder.encode(test_data);
+    input_stream.setBuffer(StringUtils::toBytes(test_data));
 
-    std::cout << "Encoded: " << encoded << std::endl;
+    BufferBitStream output_stream;
+    Lz77Encoder encoder(&input_stream, &output_stream);
+    encoder.encode();
+
+    std::cout << "Encoded: " << StringUtils::toString(output_stream.getBuffer()) << std::endl;
 
     //auto decoded = encoder.decode(encoded);
 

test/core/TestBitStream.cpp

@@ -0,0 +1,30 @@
+#include "ByteUtils.h"
+#include "BufferBitStream.h"
+
+#include <iostream>
+
+int main()
+{
+    std::vector<std::string> bytes{"11100101", "00110101", "00010001"};
+
+    BufferBitStream stream;
+    for(const auto& byte : bytes)
+    {
+        stream.writeByte(ByteUtils::getFromString(byte));
+    }
+
+    unsigned char buffer{0} ;
+    auto valid = stream.readNextNBits(3, buffer);
+    std::cout << "Slice0 is " << ByteUtils::toString(buffer) << std::endl;
+
+    valid = stream.readNextNBits(3, buffer);
+    std::cout << "Slice1 is " << ByteUtils::toString(buffer) << std::endl;
+
+    valid = stream.readNextNBits(5, buffer);
+    std::cout << "Slice2 is " << ByteUtils::toString(buffer) << std::endl;
+
+    valid = stream.readNextNBits(7, buffer);
+    std::cout << "Slice3 is " << ByteUtils::toString(buffer) << std::endl;
+
+    return 0;
+}

test/image/TestPngReader.cpp

@@ -1,11 +1,15 @@
 #include "PngReader.h"
 
+#include "BitStream.h"
+
 #include "Image.h"
 #include <iostream>
 
 int main()
 {
-    const auto path = "/home/jmsgrogan/Downloads/test.png";
+    //const auto path = "/home/jmsgrogan/Downloads/test.png";
+
+    const auto path = "/home/jmsgrogan/Downloads/index.png";
 
     PngReader reader;
     reader.setPath(path);

test/image/TestPngWriter.cpp

@@ -1,6 +1,8 @@
 #include "Image.h"
 #include "PngWriter.h"
 
+
+#include "BitStream.h"
 #include "ImagePrimitives.h"
 
 #include <iostream>