Further compression and png work.

This commit is contained in:
James Grogan 2022-11-23 15:41:33 +00:00
parent 318b481ccc
commit 9c8faa534b
34 changed files with 1164 additions and 203 deletions

View file

@ -0,0 +1,23 @@
#pragma once
class BitStream;
class AbstractEncoder
{
public:
AbstractEncoder(BitStream* inputStream, BitStream* outputStream)
: mInputStream(inputStream),
mOutputStream(outputStream)
{
}
virtual ~AbstractEncoder() = default;
virtual bool encode() = 0;
virtual bool decode() = 0;
protected:
BitStream* mInputStream{nullptr};
BitStream* mOutputStream{nullptr};
};

View file

@ -4,6 +4,10 @@ list(APPEND compression_LIB_INCLUDES
HuffmanEncoder.cpp HuffmanEncoder.cpp
RunLengthEncoder.cpp RunLengthEncoder.cpp
ZlibData.cpp ZlibData.cpp
ZlibEncoder.cpp
DeflateEncoder.cpp
DeflateBlock.cpp
Lz77Encoder.cpp
) )
add_library(compression SHARED ${compression_LIB_INCLUDES}) add_library(compression SHARED ${compression_LIB_INCLUDES})

View file

@ -0,0 +1,262 @@
#include "DeflateBlock.h"
#include "ByteUtils.h"
#include <algorithm>
#include <iostream>
DeflateBlock::DeflateBlock(BitStream* inputStream, BitStream* outputStream)
: mInputStream(inputStream),
mOutputStream(outputStream)
{
}
bool DeflateBlock::readNextCodeLengthSymbol(unsigned char& final_symbol)
{
unsigned working_index{0};
auto count = mCodeLengthMapping[working_index].first;
auto delta = count;
bool found{false};
unsigned char buffer{0};
unsigned char working_bits{0};
unsigned working_symbol{0};
while(!found)
{
auto valid = mInputStream->readNextNBits(delta, buffer);
working_bits = (working_bits << delta) | buffer;
for(const auto& entry : mCodeLengthMapping[working_index].second)
{
if (entry.first == working_bits)
{
found = true;
working_symbol = entry.second;
break;
}
}
if (!found)
{
working_index++;
if (working_index >= mCodeLengthMapping.size())
{
break;
}
auto new_count = mCodeLengthMapping[working_index].first;
delta = new_count - count;
count = new_count;
}
}
if (found)
{
final_symbol = working_symbol;
std::cout << "Found symbol " << working_symbol << " with bits " << ByteUtils::toString(working_bits) << std::endl;
std::cout << "At Byte offset " << mInputStream->getCurrentByteOffset() << " and bit offset " << mInputStream->getCurrentBitOffset() << std::endl;
return true;
}
else
{
std::cout << "SYMBOL NOT FOUND " << " with bits " << ByteUtils::toString(working_bits) << std::endl;
return false;
}
}
void DeflateBlock::setCodeLengthAlphabetLengths(const std::vector<unsigned char>& lengths)
{
mCodeLengthAlphabetLengths = lengths;
}
void DeflateBlock::setCodeLengthLength(unsigned length)
{
mHclen = length;
}
void DeflateBlock::setLiteralsTableLength(unsigned length)
{
mHlit = length;
}
void DeflateBlock::setDistanceTableLength(unsigned length)
{
mHdist = length;
}
void DeflateBlock::setIsFinalBlock(bool isFinal)
{
}
void DeflateBlock::flushToStream()
{
}
void DeflateBlock::readLiteralCodeLengths()
{
std::vector<unsigned> lengths;
unsigned char symbol{0};
while(lengths.size() < mHlit)
{
bool valid = readNextCodeLengthSymbol(symbol);
if (!valid)
{
std::cout << "Hit unknown symbol - bailing out" << std::endl;
break;
}
if (symbol < 16)
{
lengths.push_back(symbol);
}
else if(symbol == 16)
{
unsigned char num_reps{0};
mInputStream->readNextNBits(2, num_reps);
auto last_val = lengths[lengths.size()-1];
std::cout << "Got val 16 doing " << 3 + num_reps << std::endl;
for(unsigned idx=0; idx< 3 + num_reps; idx++)
{
lengths.push_back(last_val);
}
}
else if(symbol == 17)
{
unsigned char num_reps{0};
mInputStream->readNextNBits(3, num_reps);
std::cout << "Got val 17 doing " << 3 + num_reps << std::endl;
for(unsigned idx=0; idx< 3 + num_reps; idx++)
{
lengths.push_back(0);
}
}
else if(symbol == 18)
{
unsigned char num_reps{0};
mInputStream->readNextNBits(7, num_reps);
std::cout << "Got val 18 doing " << 11 + num_reps << std::endl;
for(unsigned idx=0; idx< 11 + num_reps; idx++)
{
lengths.push_back(0);
}
}
}
}
void DeflateBlock::buildCodeLengthMapping()
{
for(unsigned idx=1; idx<8; idx++)
{
std::vector<unsigned> entries;
for(unsigned jdx=0; jdx<mCodeLengthAlphabetLengths.size(); jdx++)
{
if (mCodeLengthAlphabetLengths[jdx] == idx)
{
entries.push_back(jdx);
}
}
if (entries.empty())
{
continue;
}
CodeLengthCountEntry count_entry{idx, {}};
std::sort(entries.begin(), entries.end());
unsigned char offset = 0x01 << idx - 1;
unsigned char count{0};
for (auto entry : entries)
{
count_entry.second.push_back(CodeLengthEntry{offset + count, entry});
count++;
}
mCodeLengthMapping.push_back(count_entry);
}
for (const auto& map_data : mCodeLengthMapping)
{
std::cout << "Map entry " << map_data.first << " has vals: " << std::endl;
for (const auto& entry : map_data.second)
{
std::cout << "Key " << ByteUtils::toString(entry.first) << " val: " << entry.second << std::endl;
}
}
}
void DeflateBlock::readDynamicHuffmanTable()
{
unsigned char h_lit{0};
mInputStream->readNextNBits(5, h_lit);
mHlit = h_lit + 257;
std::cout << "Got HLIT " << mHlit << std::endl;
unsigned char h_dist{0};
mInputStream->readNextNBits(5, h_dist);
mHdist = h_dist + 1;
std::cout << "Got HDIST " << mHdist << std::endl;
unsigned char h_clen{0};
mInputStream->readNextNBits(4, h_clen);
mHclen = h_clen + 4;
std::cout << "Got HCLEN " << mHclen << std::endl;
mCodeLengthAlphabetLengths = std::vector<unsigned char>(19, 0);
unsigned char buffer{0};
for(unsigned idx = 0; idx< mHclen; idx++)
{
mInputStream->readNextNBits(3, buffer);
mCodeLengthAlphabetLengths[CODE_LENGTH_ALPHABET_PERMUTATION[idx]] = buffer;
std::cout << "Got code length for " << CODE_LENGTH_ALPHABET_PERMUTATION[idx] << " of " << static_cast<unsigned>(buffer) << std::endl;
}
buildCodeLengthMapping();
readLiteralCodeLengths();
}
void DeflateBlock::readHeader()
{
auto working_byte = mInputStream->getCurrentByte();
std::cout << "Into process data "<< std::endl;
std::cout << mInputStream->logNextNBytes(9);
unsigned char final_block{0};
mInputStream->readNextNBits(1, final_block);
mInFinalBlock = bool(final_block);
if (mInFinalBlock)
{
std::cout << "Got final block" << std::endl;
}
mInputStream->readNextNBits(2, mCompressionType);
std::cout << "Compress type byte is: " << static_cast<unsigned>(mCompressionType) << std::endl;
if (mCompressionType == NO_COMPRESSION)
{
std::cout << "Got NO_COMPRESSION" << std::endl;
}
else if (mCompressionType == FIXED_HUFFMAN)
{
std::cout << "Got FIXED_HUFFMAN" << std::endl;
}
else if (mCompressionType == DYNAMIC_HUFFMAN)
{
std::cout << "Got DYNAMIC_HUFFMAN" << std::endl;
readDynamicHuffmanTable();
}
else if (mCompressionType == ERROR)
{
std::cout << "Got ERROR" << std::endl;
}
}

View file

@ -0,0 +1,60 @@
#pragma once
#include "BitStream.h"
class DeflateBlock
{
public:
DeflateBlock(BitStream* inputStream, BitStream* outputStream);
void readHeader();
void readDynamicHuffmanTable();
void buildCodeLengthMapping();
void readLiteralCodeLengths();
bool readNextCodeLengthSymbol(unsigned char& buffer);
bool isFinalBlock() const
{
return mInFinalBlock;
}
void setCodeLengthAlphabetLengths(const std::vector<unsigned char>& lengths);
void setCodeLengthLength(unsigned length);
void setLiteralsTableLength(unsigned length);
void setDistanceTableLength(unsigned length);
void setIsFinalBlock(bool isFinal);
void flushToStream();
private:
BitStream* mInputStream;
BitStream* mOutputStream;
unsigned mHlit{0};
unsigned mHdist{0};
unsigned mHclen{0};
using CodeLengthEntry = std::pair<unsigned char, unsigned>;
using CodeLengthCountEntry = std::pair<unsigned, std::vector<CodeLengthEntry> >;
std::vector<CodeLengthCountEntry> mCodeLengthMapping;
std::vector<unsigned char> mCodeLengthAlphabetLengths;
static constexpr unsigned CODE_LENGTH_ALPHABET_PERMUTATION[19]{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
unsigned char mCompressionType{0};
bool mInFinalBlock{false};
static constexpr unsigned char NO_COMPRESSION = 0x00;
static constexpr unsigned char FIXED_HUFFMAN = 0x01;
static constexpr unsigned char DYNAMIC_HUFFMAN = 0x02;
static constexpr unsigned char ERROR = 0x03;
};

View file

@ -0,0 +1,38 @@
#include "DeflateEncoder.h"
#include "BitStream.h"
#include "DeflateBlock.h"
#include <iostream>
DeflateEncoder::DeflateEncoder(BitStream* inputStream, BitStream* outputStream)
: AbstractEncoder(inputStream, outputStream)
{
}
DeflateEncoder::~DeflateEncoder()
{
}
bool DeflateEncoder::encode()
{
return false;
}
bool DeflateEncoder::decode()
{
auto working_block = std::make_unique<DeflateBlock>(mInputStream, mOutputStream);
working_block->readHeader();
DeflateBlock* raw_block = working_block.get();
while(!raw_block->isFinalBlock())
{
break;
}
return false;
}

View file

@ -0,0 +1,25 @@
#pragma once
#include "AbstractEncoder.h"
#include <vector>
#include <memory>
class DeflateBlock;
class DeflateEncoder : public AbstractEncoder
{
public:
DeflateEncoder(BitStream* inputStream, BitStream* outputStream);
~DeflateEncoder();
bool encode() override;
bool decode() override;
private:
std::vector<std::unique_ptr<DeflateBlock > > mBlocks;
};

View file

@ -0,0 +1,170 @@
#include "Lz77Encoder.h"
#include "StringUtils.h"
#include "BitStream.h"
Lz77Encoder::Lz77Encoder(BitStream* inputStream, BitStream* outputStream)
: AbstractEncoder(inputStream, outputStream)
{
}
unsigned Lz77Encoder::lookAheadForMatchingChars(std::vector<char>& matchBuffer, unsigned searchIndex, unsigned hitOffset, const std::string& stream, unsigned streamLoc)
{
auto remaining_size = stream.size() - streamLoc;
unsigned num_hits{1};
for (unsigned jdx=1; jdx< remaining_size; jdx++)
{
char buffer_char{0};
if (searchIndex + jdx < mSearchBuffer.size())
{
buffer_char = mSearchBuffer[searchIndex + jdx];
}
else
{
buffer_char = stream[jdx - hitOffset];
}
auto lookahead_char = stream[streamLoc + jdx];
if (lookahead_char == buffer_char)
{
matchBuffer.push_back(buffer_char);
num_hits++;
}
else
{
break;
}
}
return num_hits;
}
void Lz77Encoder::lookThroughSearchBuffer(char searchChar, unsigned& hitLength, unsigned& hitOffset, const std::string& stream, unsigned streamLoc)
{
for(unsigned idx=0; idx<mSearchBuffer.size(); idx++)
{
auto search_index = mSearchBuffer.size() - idx - 1;
if (auto buffer_char = mSearchBuffer[search_index]; buffer_char == searchChar)
{
std::vector<char> match_buffer{buffer_char};
auto num_hits = lookAheadForMatchingChars(match_buffer, search_index, idx, stream, streamLoc);
if (num_hits >= hitLength)
{
hitLength = num_hits;
hitOffset = idx + 1;
}
}
}
}
bool Lz77Encoder::encode()
{
/*
unsigned loc{0};
std::string ret;
while(!mInputStream->isFinished())
{
auto search_char = stream[loc];
unsigned hit_length{0};
unsigned hit_offset{0};
lookThroughSearchBuffer(search_char, hit_length, hit_offset, stream, loc);
if (hit_length > 0)
{
ret += "@" + std::to_string(hit_offset) + "L" + std::to_string(hit_length);
loc+=hit_length;
auto hit_loc = mSearchBuffer.size() - hit_offset;
for(unsigned idx=hit_loc; idx<hit_loc + hit_length; idx++)
{
mSearchBuffer.push_back(mSearchBuffer[idx]);
}
}
else
{
ret += search_char;
mSearchBuffer.push_back(search_char);
loc++;
}
}
return ret;
*/
return false;
}
bool Lz77Encoder::decode()
{
/*
std::string ret;
unsigned loc{0};
while(loc < stream.size())
{
auto working_char = stream[loc];
if (working_char == '@')
{
unsigned loc_working = loc;
auto remainder = stream.size() - loc;
std::string offset;
unsigned length_loc{0};
for(unsigned jdx=0; jdx< remainder; jdx++)
{
loc++;
auto offset_char = stream[loc];
if (offset_char == 'L')
{
loc++;
break;
}
else
{
offset += offset_char;
}
}
unsigned offset_amount = std::stoul(offset);
std::string length;
remainder = stream.size() - loc;
for(unsigned jdx=0; jdx< remainder; jdx++)
{
auto length_char = stream[loc];
if (StringUtils::IsAlphabetical(length_char) || length_char == '@')
{
break;
}
else
{
loc++;
length += length_char;
}
}
unsigned length_amount = std::stoul(length);
auto buffer_index = ret.size() - offset_amount;
for(unsigned jdx=buffer_index;jdx<buffer_index+length_amount; jdx++)
{
ret += ret[jdx];
}
}
else
{
loc++;
ret += working_char;
}
}
return ret;
*/
return false;
}

View file

@ -1,168 +1,33 @@
#pragma once #pragma once
#include "StringUtils.h" #include "AbstractEncoder.h"
#include <string> #include <string>
#include <vector> #include <vector>
class Lz77Encoder class Lz77Encoder : public AbstractEncoder
{ {
public: public:
using DataStream = std::vector<char>; using Buffer = std::vector<unsigned char>;
unsigned lookAheadForMatchingChars(std::vector<char>& matchBuffer, unsigned searchIndex, unsigned hitOffset, const std::string& stream, unsigned streamLoc) Lz77Encoder(BitStream* inputStream, BitStream* outputStream);
{
auto remaining_size = stream.size() - streamLoc;
unsigned num_hits{1}; unsigned lookAheadForMatchingChars(std::vector<char>& matchBuffer, unsigned searchIndex, unsigned hitOffset, const std::string& stream, unsigned streamLoc);
for (unsigned jdx=1; jdx< remaining_size; jdx++)
{
char buffer_char{0};
if (searchIndex + jdx < mSearchBuffer.size())
{
buffer_char = mSearchBuffer[searchIndex + jdx];
}
else
{
buffer_char = stream[jdx - hitOffset];
}
auto lookahead_char = stream[streamLoc + jdx]; void lookThroughSearchBuffer(char searchChar, unsigned& hitLength, unsigned& hitOffset, const std::string& stream, unsigned streamLoc);
if (lookahead_char == buffer_char)
{
matchBuffer.push_back(buffer_char);
num_hits++;
}
else
{
break;
}
}
return num_hits;
}
void lookThroughSearchBuffer(char searchChar, unsigned& hitLength, unsigned& hitOffset, const std::string& stream, unsigned streamLoc) bool encode() override;
{
for(unsigned idx=0; idx<mSearchBuffer.size(); idx++)
{
auto search_index = mSearchBuffer.size() - idx - 1;
if (auto buffer_char = mSearchBuffer[search_index]; buffer_char == searchChar) bool decode() override;
{
std::vector<char> match_buffer{buffer_char};
auto num_hits = lookAheadForMatchingChars(match_buffer, search_index, idx, stream, streamLoc);
if (num_hits >= hitLength) void setSearchBufferSize(unsigned size);
{
hitLength = num_hits;
hitOffset = idx + 1;
}
}
}
}
std::string encode(const std::string& stream) void setLookAheadBufferSize(unsigned size);
{
unsigned loc{0};
std::string ret;
while(loc < stream.size()) private:
{ unsigned mSearchBufferSize{32000};
auto search_char = stream[loc]; Buffer mSearchBuffer;
unsigned hit_length{0}; unsigned mLookAheadBufferSize{256};
unsigned hit_offset{0}; Buffer mLookaheadBuffer;
lookThroughSearchBuffer(search_char, hit_length, hit_offset, stream, loc);
if (hit_length > 0)
{
ret += "@" + std::to_string(hit_offset) + "L" + std::to_string(hit_length);
loc+=hit_length;
auto hit_loc = mSearchBuffer.size() - hit_offset;
for(unsigned idx=hit_loc; idx<hit_loc + hit_length; idx++)
{
mSearchBuffer.push_back(mSearchBuffer[idx]);
}
}
else
{
ret += search_char;
mSearchBuffer.push_back(search_char);
loc++;
}
}
return ret;
}
std::string decode(const std::string& stream)
{
std::string ret;
unsigned loc{0};
while(loc < stream.size())
{
auto working_char = stream[loc];
if (working_char == '@')
{
unsigned loc_working = loc;
auto remainder = stream.size() - loc;
std::string offset;
unsigned length_loc{0};
for(unsigned jdx=0; jdx< remainder; jdx++)
{
loc++;
auto offset_char = stream[loc];
if (offset_char == 'L')
{
loc++;
break;
}
else
{
offset += offset_char;
}
}
unsigned offset_amount = std::stoul(offset);
std::string length;
remainder = stream.size() - loc;
for(unsigned jdx=0; jdx< remainder; jdx++)
{
auto length_char = stream[loc];
if (StringUtils::IsAlphabetical(length_char) || length_char == '@')
{
break;
}
else
{
loc++;
length += length_char;
}
}
unsigned length_amount = std::stoul(length);
auto buffer_index = ret.size() - offset_amount;
for(unsigned jdx=buffer_index;jdx<buffer_index+length_amount; jdx++)
{
ret += ret[jdx];
}
}
else
{
loc++;
ret += working_char;
}
}
return ret;
}
DataStream mSearchBuffer;
DataStream mLookaheadBuffer;
}; };

View file

@ -0,0 +1,74 @@
#include "ZlibEncoder.h"
#include "ByteUtils.h"
#include "DeflateEncoder.h"
#include "FileLogger.h"
#include <iostream>
ZlibEncoder::ZlibEncoder(BitStream* inputStream, BitStream* outputStream)
: AbstractEncoder(inputStream, outputStream)
{
}
ZlibEncoder::~ZlibEncoder()
{
}
void ZlibEncoder::setCompressionMethod(unsigned char method)
{
std::cout << "Got compression input " << static_cast<int>(method) << std::endl;
mCmf = method;
mCompressionMethod = ByteUtils::getLowerNBits(method, 4);
mCompressionInfo = ByteUtils::getHigherNBits(method, 4);
std::cout << "Got compression method " << static_cast<int>(mCompressionMethod) << " and info " << static_cast<int>(mCompressionInfo) << std::endl;
}
void ZlibEncoder::setExtraFlags(unsigned char extraFlags)
{
std::cout << "Got flags " << static_cast<int>(extraFlags) << std::endl;
mFlg = extraFlags;
mFlagCheck = ByteUtils::getLowerNBits(extraFlags, 5);
mFlagDict = ByteUtils::getBitN(extraFlags, 5);
mFlagLevel = ByteUtils::getHigherNBits(extraFlags, 2);
std::cout << "Got flag check " << static_cast<int>(mFlagCheck) << " and dict " << static_cast<int>(mFlagDict) << " and level " << static_cast<int>(mFlagLevel) << std::endl;
}
bool ZlibEncoder::encode()
{
if (!mWorkingEncoder)
{
if (mCompressionMethod == 8)
{
mWorkingEncoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
}
else
{
MLOG_ERROR("Zib requested decoder not recognized: " << mCompressionMethod << " aborting encode");
return false;
}
}
return mWorkingEncoder->encode();
}
bool ZlibEncoder::decode()
{
if (!mWorkingEncoder)
{
if (mCompressionMethod == 8)
{
mWorkingEncoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
}
else
{
MLOG_ERROR("Zib requested decoder not recognized: " << mCompressionMethod << " aborting decode");
return false;
}
}
return mWorkingEncoder->decode();
}

View file

@ -0,0 +1,34 @@
#pragma once
#include "AbstractEncoder.h"
#include <memory>
#include <vector>
class ZlibEncoder : public AbstractEncoder
{
public:
ZlibEncoder(BitStream* inputStream, BitStream* outputStream);
~ZlibEncoder();
void setCompressionMethod(unsigned char method);
void setExtraFlags(unsigned char extraFlags);
bool encode() override;
bool decode() override;
private:
unsigned char mCmf{0};
unsigned char mFlg{0};
unsigned char mCompressionMethod{8};
unsigned char mCompressionInfo{0};
unsigned char mFlagCheck{0};
unsigned char mFlagDict{0};
unsigned char mFlagLevel{0};
unsigned char mCheckValue{0};
std::unique_ptr<AbstractEncoder> mWorkingEncoder;
};

View file

@ -30,6 +30,9 @@ list(APPEND core_LIB_INCLUDES
StringUtils.cpp StringUtils.cpp
streams/BinaryStream.cpp streams/BinaryStream.cpp
streams/BitStream.cpp streams/BitStream.cpp
streams/InputBitStream.cpp
streams/OutputBitStream.cpp
streams/BufferBitStream.cpp
http/HttpResponse.cpp http/HttpResponse.cpp
http/HttpHeader.cpp http/HttpHeader.cpp
http/HttpRequest.cpp http/HttpRequest.cpp

View file

@ -25,6 +25,16 @@ bool StringUtils::IsAlphabetical(char c)
return std::isalpha(c); return std::isalpha(c);
} }
std::vector<unsigned char> StringUtils::toBytes(const std::string& input)
{
return {input.begin(), input.end()};
}
std::string StringUtils::toString(const std::vector<unsigned char>& bytes)
{
return {bytes.begin(), bytes.end()};
}
std::vector<std::string> StringUtils::toLines(const std::string& input) std::vector<std::string> StringUtils::toLines(const std::string& input)
{ {
auto result = std::vector<std::string>{}; auto result = std::vector<std::string>{};

View file

@ -30,4 +30,8 @@ public:
static std::vector<std::string> toLines(const std::string& input); static std::vector<std::string> toLines(const std::string& input);
static std::string stripQuotes(const std::string& input); static std::string stripQuotes(const std::string& input);
static std::vector<unsigned char> toBytes(const std::string& input);
static std::string toString(const std::vector<unsigned char>& bytes);
}; };

View file

@ -2,37 +2,70 @@
#include "ByteUtils.h" #include "ByteUtils.h"
bool BitStream::loadNextByte() #include <sstream>
#include <iostream>
BitStream::~BitStream()
{ {
if (mByteOffset + 1 == mBuffer.size())
}
unsigned char BitStream::getCurrentByte()
{
if (mByteOffset < 0)
{
readNextByte();
}
return mCurrentByte;
}
int BitStream::getCurrentByteOffset() const
{
return mByteOffset;
}
unsigned BitStream::getCurrentBitOffset() const
{
return mBitOffset;
}
std::string BitStream::logNextNBytes(unsigned n) const
{
std::stringstream sstr;
unsigned count{0};
for(auto byte : peekNextNBytes(n))
{
sstr << count << " | " << ByteUtils::toString(byte) + '\n';
count++;
}
return sstr.str();
}
bool BitStream::readNextNBits(unsigned n, unsigned char& buffer)
{
if (mByteOffset < 0)
{
if (!readNextByte())
{ {
return false; return false;
} }
else
{
mByteOffset++;
mCurrentByte = mBuffer[mByteOffset];
return true;
}
} }
bool BitStream::getNextNBits(unsigned n, unsigned char& buffer) int overshoot = n + mBitOffset - 8;
{
int overshoot = n + mBitOffset - 7;
if (overshoot > 0) if (overshoot > 0)
{ {
unsigned char last_byte = mCurrentByte; unsigned char last_byte = mCurrentByte;
if (!loadNextByte()) if (!readNextByte())
{ {
return false; return false;
} }
auto num_lower = 7 - mBitOffset; auto num_lower = 8 - mBitOffset;
char lower_bits = ByteUtils::getHigherNBits(last_byte, num_lower); char lower_bits = ByteUtils::getHigherNBits(last_byte, num_lower);
char higher_bits = ByteUtils::getLowerNBits(mCurrentByte, overshoot); char higher_bits = ByteUtils::getLowerNBits(mCurrentByte, overshoot);
buffer = (higher_bits << (8 - num_lower)) | (lower_bits >> mBitOffset); buffer = (higher_bits << num_lower) | lower_bits;
mBitOffset = overshoot; mBitOffset = overshoot;
return true; return true;
@ -44,13 +77,3 @@ bool BitStream::getNextNBits(unsigned n, unsigned char& buffer)
return true; return true;
} }
} }
void BitStream::setByte(unsigned idx, unsigned char data)
{
mBuffer[idx] = data;
}
void BitStream::setBufferSize(std::size_t size)
{
mBuffer = std::vector<unsigned char>(size);
}

View file

@ -1,27 +1,35 @@
#pragma once #pragma once
#include <vector> #include <vector>
#include <string>
#include <optional>
class BitStream class BitStream
{ {
public: public:
bool getNextNBits(unsigned n, unsigned char& buffer); virtual ~BitStream();
bool loadNextByte(); unsigned char getCurrentByte();
void setByte(unsigned idx, unsigned char data); int getCurrentByteOffset() const;
void setBufferSize(std::size_t size); unsigned getCurrentBitOffset() const;
unsigned char getCurrentByte() const virtual bool isFinished() const = 0;
{
return mCurrentByte;
}
private: std::string logNextNBytes(unsigned n) const;
unsigned mByteOffset{0};
virtual std::vector<unsigned char> peekNextNBytes(unsigned n) const = 0;
virtual bool readNextNBits(unsigned n, unsigned char& buffer);
virtual std::optional<unsigned char> readNextByte() = 0;
virtual void writeByte(unsigned char data) = 0;
protected:
int mByteOffset{0};
unsigned mBitOffset{0}; unsigned mBitOffset{0};
char mCurrentByte{0}; unsigned char mCurrentByte{0};
std::vector<unsigned char> mBuffer;
}; };

View file

@ -0,0 +1,49 @@
#include "BufferBitStream.h"
bool BufferBitStream::isFinished() const
{
return mByteOffset == mBuffer.size();
}
std::vector<unsigned char> BufferBitStream::peekNextNBytes(unsigned n) const
{
std::vector<unsigned char> ret (n, 0);
unsigned count = 0;
for(unsigned idx=mByteOffset; idx<mByteOffset + n; idx++)
{
if (idx == mBuffer.size())
{
break;
}
ret[count] = mBuffer[idx];
count ++;
}
return ret;
}
std::optional<unsigned char> BufferBitStream::readNextByte()
{
if (mByteOffset + 1 == mBuffer.size())
{
return std::nullopt;
}
else
{
mByteOffset++;
mCurrentByte = mBuffer[mByteOffset];
return mCurrentByte;
}
}
void BufferBitStream::setBuffer(const std::vector<unsigned char>& data)
{
mBuffer = data;
}
void BufferBitStream::writeByte(unsigned char data)
{
mBuffer.push_back(data);
}

View file

@ -0,0 +1,27 @@
#pragma once
#include "BitStream.h"
#include <vector>
class BufferBitStream : public BitStream
{
public:
bool isFinished() const override;
std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
std::optional<unsigned char> readNextByte() override;
void setBuffer(const std::vector<unsigned char>& data);
void writeByte(unsigned char data) override;
const std::vector<unsigned char>& getBuffer() const
{
return mBuffer;
}
private:
std::vector<unsigned char> mBuffer;
};

View file

@ -0,0 +1,35 @@
#include "InputBitStream.h"
InputBitStream::InputBitStream(std::basic_istream<unsigned char>* stream)
: BitStream(),
mStream(stream)
{
}
bool InputBitStream::isFinished() const
{
return mStream->good();
}
std::vector<unsigned char> InputBitStream::peekNextNBytes(unsigned n) const
{
return {};
}
std::optional<unsigned char> InputBitStream::readNextByte()
{
if (mStream->good())
{
return mStream->get();
}
else
{
return std::nullopt;
}
}
void InputBitStream::writeByte(unsigned char data)
{
}

View file

@ -0,0 +1,21 @@
#pragma once
#include "BitStream.h"
#include <istream>
class InputBitStream : public BitStream
{
InputBitStream(std::basic_istream<unsigned char>* stream);
bool isFinished() const override;
std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
std::optional<unsigned char> readNextByte() override;
void writeByte(unsigned char data) override;
private:
std::basic_istream<unsigned char>* mStream{nullptr};
};

View file

@ -0,0 +1,28 @@
#include "OutputBitStream.h"
OutputBitStream::OutputBitStream(std::basic_ostream<char>* stream)
: BitStream(),
mStream(stream)
{
}
bool OutputBitStream::isFinished() const
{
return true;
}
std::vector<unsigned char> OutputBitStream::peekNextNBytes(unsigned n) const
{
return {};
}
std::optional<unsigned char> OutputBitStream::readNextByte()
{
return std::nullopt;
}
void OutputBitStream::writeByte(unsigned char data)
{
(*mStream) << data;
}

View file

@ -0,0 +1,22 @@
#pragma once
#include "BitStream.h"
#include <ostream>
class OutputBitStream : public BitStream
{
public:
OutputBitStream(std::basic_ostream<char>* stream);
bool isFinished() const override;
std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
std::optional<unsigned char> readNextByte() override;
void writeByte(unsigned char data) override;
private:
std::basic_ostream<char>* mStream{nullptr};
};

View file

@ -7,6 +7,7 @@ list(APPEND image_LIB_INCLUDES
Image.cpp Image.cpp
PngWriter.cpp PngWriter.cpp
PngReader.cpp PngReader.cpp
ImageBitStream.cpp
) )
list(APPEND image_LIBS core compression) list(APPEND image_LIBS core compression)

View file

@ -0,0 +1,29 @@
#include "ImageBitStream.h"
ImageBitStream::ImageBitStream(Image<unsigned char>* image)
: BitStream(),
mImage(image)
{
}
bool ImageBitStream::isFinished() const
{
return true;
}
std::vector<unsigned char> ImageBitStream::peekNextNBytes(unsigned n) const
{
return {};
}
std::optional<unsigned char> ImageBitStream::readNextByte()
{
return std::nullopt;
}
void ImageBitStream::writeByte(unsigned char data)
{
}

View file

@ -0,0 +1,22 @@
#pragma once
#include "BitStream.h"
#include "Image.h"
class ImageBitStream : public BitStream
{
public:
ImageBitStream(Image<unsigned char>* image);
bool isFinished() const override;
std::vector<unsigned char> peekNextNBytes(unsigned n) const override;
std::optional<unsigned char> readNextByte() override;
void writeByte(unsigned char data) override;
private:
Image<unsigned char>* mImage{nullptr};
};

View file

@ -3,9 +3,18 @@
#include <string> #include <string>
#include <sstream> #include <sstream>
namespace Png namespace Png
{ {
inline unsigned char getHighBitCheck()
{
return 0x89;
}
inline std::vector<unsigned char> getSignature()
{
return {13, 10, 26, 10};
}
struct IHDRChunk struct IHDRChunk
{ {
unsigned width{0}; unsigned width{0};

View file

@ -1,6 +1,10 @@
#include "PngReader.h" #include "PngReader.h"
#include "BinaryStream.h" #include "BinaryStream.h"
#include "BitStream.h"
#include "BufferBitStream.h"
#include "ZlibEncoder.h"
#include <iostream> #include <iostream>
#include <sstream> #include <sstream>
@ -84,12 +88,11 @@ void PngReader::readIDATChunk(unsigned length)
{ {
if (mAwaitingDataBlock) if (mAwaitingDataBlock)
{ {
mImageData.setCompressionMethod(mFile->readNextByte()); mEncoder->setCompressionMethod(mFile->readNextByte());
mImageData.setExtraFlags(mFile->readNextByte()); mEncoder->setExtraFlags(mFile->readNextByte());
mImageData.setDataSize(length-2);
for(unsigned idx=0; idx<length-2; idx++) for(unsigned idx=0; idx<length-2; idx++)
{ {
mImageData.setByte(idx, mFile->readNextByte()); mInputStream->writeByte(mFile->readNextByte());
} }
mAwaitingDataBlock = false; mAwaitingDataBlock = false;
} }
@ -97,7 +100,7 @@ void PngReader::readIDATChunk(unsigned length)
{ {
for(unsigned idx=0; idx<length; idx++) for(unsigned idx=0; idx<length; idx++)
{ {
mImageData.setByte(idx, mFile->readNextByte()); mInputStream->writeByte(mFile->readNextByte());
} }
} }
} }
@ -135,10 +138,14 @@ std::unique_ptr<Image<unsigned char> > PngReader::read()
return image; return image;
} }
mInputStream = std::make_unique<BufferBitStream>();
mOutputStream = std::make_unique<BufferBitStream>();
mEncoder = std::make_unique<ZlibEncoder>(mInputStream.get(), mOutputStream.get());
while(readChunk()) while(readChunk())
{ {
} }
mImageData.processData(); mEncoder->decode();
return std::move(image); return std::move(image);
} }

View file

@ -3,12 +3,15 @@
#include "File.h" #include "File.h"
#include "Image.h" #include "Image.h"
#include "PngElements.h" #include "PngElements.h"
#include "ZlibData.h" #include "ZlibEncoder.h"
#include <string> #include <string>
#include <memory> #include <memory>
#include <filesystem> #include <filesystem>
class BitStream;
class ZlibEncoder;
using Path = std::filesystem::path; using Path = std::filesystem::path;
class PngReader class PngReader
@ -37,6 +40,8 @@ private:
std::unique_ptr<File> mFile; std::unique_ptr<File> mFile;
Path mPath; Path mPath;
ZlibData mImageData; std::unique_ptr<ZlibEncoder> mEncoder;
std::unique_ptr<BitStream> mInputStream;
std::unique_ptr<BitStream> mOutputStream;
bool mAwaitingDataBlock{true}; bool mAwaitingDataBlock{true};
}; };

View file

@ -1,6 +1,13 @@
#include "PngWriter.h" #include "PngWriter.h"
#include "PngElements.h"
#include "Image.h" #include "Image.h"
#include "File.h"
#include "BufferBitStream.h"
#include "OutputBitStream.h"
#include "ImageBitStream.h"
#include "Lz77Encoder.h"
#include <stdio.h> #include <stdio.h>
@ -9,6 +16,11 @@ PngWriter::PngWriter()
} }
PngWriter::~PngWriter()
{
}
std::unique_ptr<PngWriter> PngWriter::Create() std::unique_ptr<PngWriter> PngWriter::Create()
{ {
return std::make_unique<PngWriter>(); return std::make_unique<PngWriter>();
@ -19,8 +31,40 @@ void PngWriter::setPath(const Path& path)
mPath = path; mPath = path;
} }
void PngWriter::write(const std::unique_ptr<Image<unsigned char> >& image) const void PngWriter::writeSignature()
{ {
mOutStream->writeByte(Png::getHighBitCheck());
for (auto byte : Png::getSignature())
{
mOutStream->writeByte(byte);
}
}
void PngWriter::writeHeader()
{
writeSignature();
}
void PngWriter::write(const std::unique_ptr<Image<unsigned char> >& image)
{
if (!mPath.empty())
{
mWorkingFile = std::make_unique<File>(mPath);
mWorkingFile->Open(true);
mOutStream = std::make_unique<OutputBitStream>(mWorkingFile->GetOutHandle());
}
else
{
mOutStream = std::make_unique<BufferBitStream>();
}
mWorkingImage = image.get();
mInStream = std::make_unique<ImageBitStream>(image.get());
writeHeader();
//mImpl->write(image); //mImpl->write(image);
//auto fp = fopen(mPath.c_str(), "wb"); //auto fp = fopen(mPath.c_str(), "wb");
@ -76,4 +120,9 @@ void PngWriter::write(const std::unique_ptr<Image<unsigned char> >& image) const
//fclose(fp); //fclose(fp);
//return; //return;
if (mWorkingFile)
{
mWorkingFile->Close();
}
} }

View file

@ -8,19 +8,31 @@
using Path = std::filesystem::path; using Path = std::filesystem::path;
class BitStream;
class File;
class PngWriter class PngWriter
{ {
public: public:
PngWriter(); PngWriter();
~PngWriter();
static std::unique_ptr<PngWriter> Create(); static std::unique_ptr<PngWriter> Create();
void setPath(const Path& path); void setPath(const Path& path);
void write(const std::unique_ptr<Image<unsigned char> >& image) const; void write(const std::unique_ptr<Image<unsigned char> >& image);
private: private:
void writeSignature();
void writeHeader();
Path mPath; Path mPath;
Image<unsigned char>* mWorkingImage{nullptr};
std::unique_ptr<BitStream> mInStream;
std::unique_ptr<BitStream> mOutStream;
std::unique_ptr<File> mWorkingFile;
}; };
using PngWriterPtr = std::unique_ptr<PngWriter>; using PngWriterPtr = std::unique_ptr<PngWriter>;

View file

@ -12,6 +12,7 @@ list(APPEND TestFiles
audio/TestMidiReader.cpp audio/TestMidiReader.cpp
core/TestByteUtils.cpp core/TestByteUtils.cpp
core/TestBinaryStream.cpp core/TestBinaryStream.cpp
core/TestBitStream.cpp
core/TestTomlReader.cpp core/TestTomlReader.cpp
compiler/TestLexer.cpp compiler/TestLexer.cpp
compiler/TestTemplatingEngine.cpp compiler/TestTemplatingEngine.cpp

View file

@ -1,5 +1,6 @@
#include <iostream> #include <iostream>
#include "BufferBitStream.h"
#include "HuffmanEncoder.h" #include "HuffmanEncoder.h"
#include "RunLengthEncoder.h" #include "RunLengthEncoder.h"
#include "Lz77Encoder.h" #include "Lz77Encoder.h"
@ -42,10 +43,14 @@ void test_lz77_encoder()
std::string test_data = "sir sid eastman easily teases sea sick seals"; std::string test_data = "sir sid eastman easily teases sea sick seals";
//std::string test_data = "sir sid eastman"; //std::string test_data = "sir sid eastman";
Lz77Encoder encoder; BufferBitStream input_stream;
auto encoded = encoder.encode(test_data); input_stream.setBuffer(StringUtils::toBytes(test_data));
std::cout << "Encoded: " << encoded << std::endl; BufferBitStream output_stream;
Lz77Encoder encoder(&input_stream, &output_stream);
encoder.encode();
std::cout << "Encoded: " << StringUtils::toString(output_stream.getBuffer()) << std::endl;
//auto decoded = encoder.decode(encoded); //auto decoded = encoder.decode(encoded);

View file

@ -0,0 +1,30 @@
#include "ByteUtils.h"
#include "BufferBitStream.h"
#include <iostream>
int main()
{
std::vector<std::string> bytes{"11100101", "00110101", "00010001"};
BufferBitStream stream;
for(const auto& byte : bytes)
{
stream.writeByte(ByteUtils::getFromString(byte));
}
unsigned char buffer{0} ;
auto valid = stream.readNextNBits(3, buffer);
std::cout << "Slice0 is " << ByteUtils::toString(buffer) << std::endl;
valid = stream.readNextNBits(3, buffer);
std::cout << "Slice1 is " << ByteUtils::toString(buffer) << std::endl;
valid = stream.readNextNBits(5, buffer);
std::cout << "Slice2 is " << ByteUtils::toString(buffer) << std::endl;
valid = stream.readNextNBits(7, buffer);
std::cout << "Slice3 is " << ByteUtils::toString(buffer) << std::endl;
return 0;
}

View file

@ -1,11 +1,15 @@
#include "PngReader.h" #include "PngReader.h"
#include "BitStream.h"
#include "Image.h" #include "Image.h"
#include <iostream> #include <iostream>
int main() int main()
{ {
const auto path = "/home/jmsgrogan/Downloads/test.png"; //const auto path = "/home/jmsgrogan/Downloads/test.png";
const auto path = "/home/jmsgrogan/Downloads/index.png";
PngReader reader; PngReader reader;
reader.setPath(path); reader.setPath(path);

View file

@ -1,6 +1,8 @@
#include "Image.h" #include "Image.h"
#include "PngWriter.h" #include "PngWriter.h"
#include "BitStream.h"
#include "ImagePrimitives.h" #include "ImagePrimitives.h"
#include <iostream> #include <iostream>