jgrogan/stuff-from-scratch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Continue png writing.

Browse source
This commit is contained in:
James Grogan 2022-11-24 16:15:41 +00:00
parent 5400a232dd
commit 8f97e9b7a1
29 changed files with 714 additions and 302 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/compression/AbstractEncoder.h
View file

@ -1,5 +1,9 @@
#pragma once
#include "AbstractChecksumCalculator.h"
#include <vector>
class BitStream;
class AbstractEncoder
@ -14,10 +18,17 @@ public:
virtual ~AbstractEncoder() = default;
void addChecksumCalculator(AbstractChecksumCalculator* calculator)
{
mChecksumCalculators.push_back(calculator);
}
virtual bool encode() = 0;
virtual bool decode() = 0;
protected:
std::vector<AbstractChecksumCalculator*> mChecksumCalculators;
BitStream* mInputStream{nullptr};
BitStream* mOutputStream{nullptr};
};

29
src/compression/Adler32Checksum.h Normal file
View file

@ -0,0 +1,29 @@
#pragma once
#include "AbstractChecksumCalculator.h"
class Adler32Checksum : public AbstractChecksumCalculator
{
public:
void addValue(unsigned char val) override
{
mSum1 = (mSum1 + val) % MOD_ADLER32;
mSum2 = (mSum2 + mSum1) % MOD_ADLER32;
}
uint32_t getChecksum() const override
{
return (mSum2 << 16) | mSum1;
}
void reset() override
{
mSum1 = 1;
mSum2 = 0;
}
private:
static constexpr unsigned MOD_ADLER32{65536};
uint32_t mSum1{1};
uint32_t mSum2{0};
};

10
src/compression/CMakeLists.txt
View file

@ -3,10 +3,9 @@ list(APPEND compression_LIB_INCLUDES
StreamCompressor.cpp
HuffmanEncoder.cpp
RunLengthEncoder.cpp
ZlibData.cpp
ZlibEncoder.cpp
DeflateEncoder.cpp
DeflateBlock.cpp
deflate/DeflateEncoder.cpp
deflate/DeflateBlock.cpp
Lz77Encoder.cpp
CyclicRedundancyChecker.cpp
)
@ -14,9 +13,10 @@ list(APPEND compression_LIB_INCLUDES
add_library(compression SHARED ${compression_LIB_INCLUDES})
target_include_directories(compression PUBLIC
"${CMAKE_CURRENT_SOURCE_DIR}"
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/deflate
)
target_link_libraries(compression PUBLIC core)
set_property(TARGET compression PROPERTY FOLDER src)
set_target_properties( compression PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON )
set_target_properties( compression PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON )

38
src/compression/DeflateEncoder.cpp
View file

@ -1,38 +0,0 @@
#include "DeflateEncoder.h"
#include "BitStream.h"
#include "DeflateBlock.h"
#include <iostream>
DeflateEncoder::DeflateEncoder(BitStream* inputStream, BitStream* outputStream)
: AbstractEncoder(inputStream, outputStream)
{
}
DeflateEncoder::~DeflateEncoder()
{
}
bool DeflateEncoder::encode()
{
return false;
}
bool DeflateEncoder::decode()
{
auto working_block = std::make_unique<DeflateBlock>(mInputStream, mOutputStream);
working_block->readHeader();
DeflateBlock* raw_block = working_block.get();
while(!raw_block->isFinalBlock())
{
break;
}
return false;
}

0
src/compression/ZlibData.cpp
View file

107
src/compression/ZlibData.h
View file

@ -1,107 +0,0 @@
#pragma once
#include "ByteUtils.h"
#include "BitStream.h"
#include <vector>
#include <iostream>
class ZlibData
{
public:
void setByte(unsigned idx, unsigned char data)
{
mBitStream.setByte(idx, data);
}
void setDataSize(std::size_t size)
{
mBitStream.setBufferSize(size);
}
void setCompressionMethod(unsigned char method)
{
std::cout << "Got compression input " << static_cast<int>(method) << std::endl;
mCmf = method;
mCompressionMethod = ByteUtils::getLowerNBits(method, 4);
mCompressionInfo = ByteUtils::getHigherNBits(method, 4);
std::cout << "Got compression method " << static_cast<int>(mCompressionMethod) << " and info " << static_cast<int>(mCompressionInfo) << std::endl;
}
void setExtraFlags(unsigned char extraFlags)
{
std::cout << "Got flags " << static_cast<int>(extraFlags) << std::endl;
mFlg = extraFlags;
mFlagCheck = ByteUtils::getLowerNBits(extraFlags, 5);
mFlagDict = ByteUtils::getBitN(extraFlags, 5);
mFlagLevel = ByteUtils::getHigherNBits(extraFlags, 2);
std::cout << "Got flag check " << static_cast<int>(mFlagCheck) << " and dict " << static_cast<int>(mFlagDict) << " and level " << static_cast<int>(mFlagLevel) << std::endl;
}
void processData()
{
unsigned char NO_COMPRESSION = 0x00;
unsigned char FIXED_HUFFMAN = 0x01;
unsigned char DYNAMIC_HUFFMAN = 0x02;
unsigned char ERROR = 0x03;
bool in_final_block = false;
while(mBitStream.loadNextByte())
{
auto working_byte = mBitStream.getCurrentByte();
std::cout << "Into process data, byte is: " << static_cast<unsigned>(working_byte) << std::endl;
unsigned char final_block{0};
mBitStream.getNextNBits(1, final_block);
if (final_block)
{
std::cout << "Got final block" << std::endl;
in_final_block = true;
}
unsigned char compress_type{0};
mBitStream.getNextNBits(2, compress_type);
std::cout << "Compress type byte is: " << static_cast<unsigned>(compress_type) << std::endl;
if (compress_type == NO_COMPRESSION)
{
std::cout << "Got NO_COMPRESSION" << std::endl;
}
else if (compress_type == FIXED_HUFFMAN)
{
std::cout << "Got FIXED_HUFFMAN" << std::endl;
}
else if (compress_type == DYNAMIC_HUFFMAN)
{
std::cout << "Got DYNAMIC_HUFFMAN" << std::endl;
unsigned char h_list{0};
mBitStream.getNextNBits(5, h_list);
mHlist = h_list + 257;
std::cout << "Got HLIST " << mHlist << std::endl;
}
else if (compress_type == ERROR)
{
std::cout << "Got ERROR" << std::endl;
}
break;
}
}
private:
BitStream mBitStream;
unsigned mHlist{0};
unsigned char mCmf{0};
unsigned char mFlg{0};
unsigned char mCompressionMethod{0};
unsigned char mCompressionInfo{0};
unsigned char mFlagCheck{0};
unsigned char mFlagDict{0};
unsigned char mFlagLevel{0};
unsigned char mCheckValue{0};
};

117
src/compression/ZlibEncoder.cpp
View file

@ -3,13 +3,18 @@
#include "ByteUtils.h"
#include "DeflateEncoder.h"
#include "FileLogger.h"
#include "BitStream.h"
#include "Adler32Checksum.h"
#include <math.h>
#include <iostream>
#include <sstream>
ZlibEncoder::ZlibEncoder(BitStream* inputStream, BitStream* outputStream)
: AbstractEncoder(inputStream, outputStream)
{
mChecksumCalculator = std::make_unique<Adler32Checksum>();
}
ZlibEncoder::~ZlibEncoder()
@ -17,58 +22,132 @@ ZlibEncoder::~ZlibEncoder()
}
void ZlibEncoder::setCompressionMethod(unsigned char method)
void ZlibEncoder::setWindowSize(unsigned size)
{
std::cout << "Got compression input " << static_cast<int>(method) << std::endl;
mCmf = method;
mCompressionMethod = ByteUtils::getLowerNBits(method, 4);
mCompressionInfo = ByteUtils::getHigherNBits(method, 4);
std::cout << "Got compression method " << static_cast<int>(mCompressionMethod) << " and info " << static_cast<int>(mCompressionInfo) << std::endl;
mWindowSize = size;
}
void ZlibEncoder::setExtraFlags(unsigned char extraFlags)
std::string ZlibEncoder::toString(CompressionLevel level) const
{
switch(level)
{
case CompressionLevel::FASTEST:
return "FASTEST";
case CompressionLevel::FAST:
return "FAST";
case CompressionLevel::DEFAULT:
return "DEFAULT";
case CompressionLevel::MAX_COMPRESSION:
return "MAX_COMPRESSION";
default:
return "UNKNOWN";
}
}
std::string ZlibEncoder::toString(CompressionMethod method) const
{
return method == CompressionMethod::DEFLATE ? "DEFLATE" : "UNKNOWN";
}
void ZlibEncoder::parseCompressionMethod(unsigned char method)
{
std::cout << "Got compression input " << static_cast<int>(method) << std::endl;
mCompressionMethod = static_cast<CompressionMethod>(ByteUtils::getLowerNBits(method, 4));
auto compression_info = ByteUtils::getHigherNBits(method, 4);
if (mCompressionMethod == CompressionMethod::DEFLATE)
{
mWindowSize = pow(2, compression_info + 8);
}
}
void ZlibEncoder::parseExtraFlags(unsigned char extraFlags)
{
std::cout << "Got flags " << static_cast<int>(extraFlags) << std::endl;
mFlg = extraFlags;
mFlagCheck = ByteUtils::getLowerNBits(extraFlags, 5);
mFlagDict = ByteUtils::getBitN(extraFlags, 5);
mFlagLevel = ByteUtils::getHigherNBits(extraFlags, 2);
mUseDictionary = bool(ByteUtils::getBitN(extraFlags, 5));
mFlagLevel = static_cast<CompressionLevel>(ByteUtils::getHigherNBits(extraFlags, 2));
}
std::string ZlibEncoder::getData() const
{
std::stringstream sstream;
sstream << "ZlibEncoder data \n";
sstream << "Compression method: " << toString(mCompressionMethod) << '\n';
sstream << "Window size: " << mWindowSize << '\n';
sstream << "Flag check: " << mFlagCheck << '\n';
sstream << "Use dictionary: " << mUseDictionary << '\n';
sstream << "Flag level: " << toString(mFlagLevel) << '\n';
return sstream.str();
std::cout << "Got flag check " << static_cast<int>(mFlagCheck) << " and dict " << static_cast<int>(mFlagDict) << " and level " << static_cast<int>(mFlagLevel) << std::endl;
}
bool ZlibEncoder::encode()
{
DeflateEncoder* deflate_encoder{nullptr};
if (!mWorkingEncoder)
{
if (mCompressionMethod == 8)
if (mCompressionMethod == CompressionMethod::DEFLATE)
{
mWorkingEncoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
auto uq_deflate_encoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
deflate_encoder = uq_deflate_encoder.get();
mWorkingEncoder = std::move(uq_deflate_encoder);
mWorkingEncoder->addChecksumCalculator(mChecksumCalculator.get());
}
else
{
MLOG_ERROR("Zib requested decoder not recognized: " << mCompressionMethod << " aborting encode");
MLOG_ERROR("Zib requested decoder not recognized: " << static_cast<int>(mCompressionMethod) << " aborting encode");
return false;
}
}
return mWorkingEncoder->encode();
deflate_encoder->setCompressionMethod(mDeflateCompressionMethod);
auto compression_info = static_cast<unsigned char>(log2(mWindowSize) - 8);
const unsigned char compression_byte = (compression_info << 4) | static_cast<unsigned char>(mCompressionMethod);
std::cout << "ZlibEncoder Writing compression byte " << static_cast<int>(compression_byte) << " with info " << static_cast<int>(compression_info) << std::endl;
mOutputStream->writeByte(compression_byte);
unsigned char flag_byte{0};
flag_byte |= (static_cast<unsigned char>(mUseDictionary) << 5);
flag_byte |= (static_cast<unsigned char>(mFlagLevel) << 6);
const auto mod = (unsigned(compression_byte)*256 + flag_byte) % 31;
flag_byte += (31 - mod);
std::cout << "ZlibEncoder Writing Flag byte " << static_cast<int>(flag_byte) << std::endl;
mOutputStream->writeByte(flag_byte);
if(!mWorkingEncoder->encode())
{
MLOG_ERROR("Sub-Encoder failed - aborting zlib encode");
//return false;
}
const auto checksum = mChecksumCalculator->getChecksum();
std::cout << "ZlibEncoder Writing Checksum " << checksum << std::endl;
mOutputStream->write(checksum);
return true;
}
bool ZlibEncoder::decode()
{
parseCompressionMethod(*mInputStream->readNextByte());
parseExtraFlags(*mInputStream->readNextByte());
if (!mWorkingEncoder)
{
if (mCompressionMethod == 8)
if (mCompressionMethod == CompressionMethod::DEFLATE)
{
mWorkingEncoder = std::make_unique<DeflateEncoder>(mInputStream, mOutputStream);
}
else
{
MLOG_ERROR("Zib requested decoder not recognized: " << mCompressionMethod << " aborting decode");
MLOG_ERROR("Zib requested decoder not recognized: " << static_cast<int>(mCompressionMethod) << " aborting decode");
return false;
}
}
return mWorkingEncoder->decode();
}

49
src/compression/ZlibEncoder.h
View file

@ -2,33 +2,54 @@
#include "AbstractEncoder.h"
#include "DeflateElements.h"
#include <memory>
#include <vector>
class AbstractChecksumCalculator;
class ZlibEncoder : public AbstractEncoder
{
public:
enum class CompressionMethod : unsigned char
{
DEFLATE = 8,
};
enum class CompressionLevel : unsigned char
{
FASTEST,
FAST,
DEFAULT,
MAX_COMPRESSION
};
ZlibEncoder(BitStream* inputStream, BitStream* outputStream);
~ZlibEncoder();
void setCompressionMethod(unsigned char method);
void setExtraFlags(unsigned char extraFlags);
void setWindowSize(unsigned size);
bool encode() override;
bool decode() override;
private:
unsigned char mCmf{0};
unsigned char mFlg{0};
unsigned char mCompressionMethod{8};
unsigned char mCompressionInfo{0};
unsigned char mFlagCheck{0};
unsigned char mFlagDict{0};
unsigned char mFlagLevel{0};
unsigned char mCheckValue{0};
std::string getData() const;
std::string toString(CompressionLevel level) const;
std::string toString(CompressionMethod method) const;
private:
void parseCompressionMethod(unsigned char method);
void parseExtraFlags(unsigned char extraFlags);
CompressionMethod mCompressionMethod{CompressionMethod::DEFLATE};
Deflate::CompressionMethod mDeflateCompressionMethod{Deflate::CompressionMethod::NONE};
unsigned mWindowSize{32768}; // Window size, n in 2^(n+8) bytes
unsigned char mFlagCheck{0};
bool mUseDictionary{false};
CompressionLevel mFlagLevel{CompressionLevel::DEFAULT};
std::unique_ptr<AbstractChecksumCalculator> mChecksumCalculator;
std::unique_ptr<AbstractEncoder> mWorkingEncoder;
};

74
src/compression/DeflateBlock.cpp → src/compression/deflate/DeflateBlock.cpp
View file

@ -1,9 +1,11 @@
#include "DeflateBlock.h"
#include "ByteUtils.h"
#include "AbstractChecksumCalculator.h"
#include <algorithm>
#include <iostream>
#include <sstream>
DeflateBlock::DeflateBlock(BitStream* inputStream, BitStream* outputStream)
: mInputStream(inputStream),
@ -88,7 +90,7 @@ void DeflateBlock::setDistanceTableLength(unsigned length)
void DeflateBlock::setIsFinalBlock(bool isFinal)
{
mInFinalBlock = isFinal;
}
void DeflateBlock::flushToStream()
@ -224,6 +226,22 @@ void DeflateBlock::readDynamicHuffmanTable()
readLiteralCodeLengths();
}
std::string DeflateBlock::getMetaData() const
{
std::stringstream sstr;
sstr << "DeflateBlock Metadata \n";
sstr << "Final block: " << mInFinalBlock << '\n';
sstr << "Compression method: " << Deflate::toString(mCompressionMethod) << '\n';
return sstr.str();
}
bool DeflateBlock::isFinalBlock() const
{
return mInFinalBlock;
}
void DeflateBlock::readHeader()
{
auto working_byte = mInputStream->getCurrentByte();
@ -234,29 +252,33 @@ void DeflateBlock::readHeader()
mInputStream->readNextNBits(1, final_block);
mInFinalBlock = bool(final_block);
if (mInFinalBlock)
{
std::cout << "Got final block" << std::endl;
}
mInputStream->readNextNBits(2, mCompressionType);
std::cout << "Compress type byte is: " << static_cast<unsigned>(mCompressionType) << std::endl;
if (mCompressionType == NO_COMPRESSION)
{
std::cout << "Got NO_COMPRESSION" << std::endl;
}
else if (mCompressionType == FIXED_HUFFMAN)
{
std::cout << "Got FIXED_HUFFMAN" << std::endl;
}
else if (mCompressionType == DYNAMIC_HUFFMAN)
{
std::cout << "Got DYNAMIC_HUFFMAN" << std::endl;
readDynamicHuffmanTable();
}
else if (mCompressionType == ERROR)
{
std::cout << "Got ERROR" << std::endl;
}
unsigned char compression_type{0};
mInputStream->readNextNBits(2, compression_type);
mCompressionMethod = static_cast<Deflate::CompressionMethod>(compression_type);
}
void DeflateBlock::write(uint16_t datalength)
{
unsigned char working_block{0};
working_block |= static_cast<unsigned char>(mInFinalBlock);
working_block |= static_cast<unsigned char>(mCompressionMethod) << 1;
if (mCompressionMethod == Deflate::CompressionMethod::NONE)
{
std::cout << "Writing compression block header " << static_cast<int>(working_block) << std::endl;
mOutputStream->writeByte(working_block);
std::cout << "Writing data length " << datalength << " " << ByteUtils::toString(datalength) << std::endl;
mOutputStream->writeWord(datalength);
std::cout << "Writing iverse data length " << ~datalength << " " << ByteUtils::toString(~datalength) << std::endl;
mOutputStream->writeWord(static_cast<uint16_t>(~datalength));
for(unsigned idx=0; idx<datalength;idx++)
{
auto byte = *mInputStream->readNextByte();
std::cout << "Writing next byte " << static_cast<int>(byte) << std::endl;
mOutputStream->writeByte(byte);
}
}
}

29
src/compression/DeflateBlock.h → src/compression/deflate/DeflateBlock.h
View file

@ -1,27 +1,32 @@
#pragma once
#include "DeflateElements.h"
#include "BitStream.h"
class AbstractChecksumCalculator;
class DeflateBlock
{
public:
DeflateBlock(BitStream* inputStream, BitStream* outputStream);
void buildCodeLengthMapping();
std::string getMetaData() const;
void flushToStream();
bool isFinalBlock() const;
void readHeader();
void readDynamicHuffmanTable();
void buildCodeLengthMapping();
void readLiteralCodeLengths();
bool readNextCodeLengthSymbol(unsigned char& buffer);
bool isFinalBlock() const
{
return mInFinalBlock;
}
void setCodeLengthAlphabetLengths(const std::vector<unsigned char>& lengths);
void setCodeLengthLength(unsigned length);
@ -32,7 +37,7 @@ public:
void setIsFinalBlock(bool isFinal);
void flushToStream();
void write(uint16_t datalength);
private:
BitStream* mInputStream;
@ -49,12 +54,6 @@ private:
std::vector<unsigned char> mCodeLengthAlphabetLengths;
static constexpr unsigned CODE_LENGTH_ALPHABET_PERMUTATION[19]{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
unsigned char mCompressionType{0};
bool mInFinalBlock{false};
static constexpr unsigned char NO_COMPRESSION = 0x00;
static constexpr unsigned char FIXED_HUFFMAN = 0x01;
static constexpr unsigned char DYNAMIC_HUFFMAN = 0x02;
static constexpr unsigned char ERROR = 0x03;
Deflate::CompressionMethod mCompressionMethod{Deflate::CompressionMethod::NONE};
};

33
src/compression/deflate/DeflateElements.h Normal file
View file

@ -0,0 +1,33 @@
#pragma once
#include <string>
namespace Deflate
{
enum class CompressionMethod
{
NONE,
FIXED_HUFFMAN,
DYNAMIC_HUFFMAN,
ERROR
};
inline std::string toString(CompressionMethod method)
{
switch (method)
{
case CompressionMethod::NONE:
return "NONE";
case CompressionMethod::FIXED_HUFFMAN:
return "FIXED_HUFFMAN";
case CompressionMethod::DYNAMIC_HUFFMAN:
return "DYNAMIC_HUFFMAN";
case CompressionMethod::ERROR:
return "ERROR";
default:
return "UNKNOWN";
}
}
}

78
src/compression/deflate/DeflateEncoder.cpp Normal file
View file

@ -0,0 +1,78 @@
#include "DeflateEncoder.h"
#include "BitStream.h"
#include "DeflateBlock.h"
#include "BufferBitStream.h"
#include <iostream>
DeflateEncoder::DeflateEncoder(BitStream* inputStream, BitStream* outputStream)
: AbstractEncoder(inputStream, outputStream)
{
}
DeflateEncoder::~DeflateEncoder()
{
}
bool DeflateEncoder::encode()
{
uint16_t count = 0;
BufferBitStream stream;
std::unique_ptr<DeflateBlock> working_block = std::make_unique<DeflateBlock>(&stream, mOutputStream);
AbstractChecksumCalculator* checksum_calc;
if (mChecksumCalculators.size() > 0)
{
std::cout << "Setting checksum calculator " << std::endl;
mOutputStream->setChecksumCalculator(mChecksumCalculators[0]);
}
while(true)
{
if (count == mMaxBlockSize)
{
std::cout << working_block->getMetaData();
working_block->write(count);
working_block = std::make_unique<DeflateBlock>(&stream, mOutputStream);
stream.reset();
}
if (auto byte = mInputStream->readNextByte())
{
stream.writeByte(*byte);
}
else
{
stream.resetOffsets();
working_block->setIsFinalBlock(true);
std::cout << working_block->getMetaData();
working_block->write(count);
break;
}
count++;
}
mOutputStream->clearChecksumCalculator();
return true;
}
bool DeflateEncoder::decode()
{
auto working_block = std::make_unique<DeflateBlock>(mInputStream, mOutputStream);
working_block->readHeader();
DeflateBlock* raw_block = working_block.get();
while(!raw_block->isFinalBlock())
{
break;
}
return false;
}

10
src/compression/DeflateEncoder.h → src/compression/deflate/DeflateEncoder.h
View file

@ -1,6 +1,7 @@
#pragma once
#include "AbstractEncoder.h"
#include "DeflateElements.h"
#include <vector>
#include <memory>
@ -18,8 +19,15 @@ public:
bool decode() override;
void setCompressionMethod(Deflate::CompressionMethod method)
{
mCompressionMethod = method;
}
private:
std::vector<std::unique_ptr<DeflateBlock > > mBlocks;
uint16_t mMaxBlockSize{65535};
Deflate::CompressionMethod mCompressionMethod{Deflate::CompressionMethod::NONE};
std::unique_ptr<DeflateBlock > mLastBlock;
};

16
src/core/streams/AbstractChecksumCalculator.h Normal file
View file

@ -0,0 +1,16 @@
#pragma once
#include "stdint.h"
class AbstractChecksumCalculator
{
public:
virtual ~AbstractChecksumCalculator() = default;
virtual void addValue(unsigned char val) = 0;
virtual uint32_t getChecksum() const = 0;
virtual void reset()
{
}
};

9
src/core/streams/BitStream.cpp
View file

@ -28,6 +28,15 @@ void BitStream::write(uint32_t data)
}
}
void BitStream::writeWord(uint16_t data)
{
unsigned num_bytes = sizeof(uint16_t);
for(unsigned idx=0; idx<num_bytes;idx++)
{
writeByte(ByteUtils::getByteN(data, idx));
}
}
int BitStream::getCurrentByteOffset() const
{
return mByteOffset;

32
src/core/streams/BitStream.h
View file

@ -1,5 +1,7 @@
#pragma once
#include "AbstractChecksumCalculator.h"
#include <vector>
#include <string>
#include <optional>
@ -29,11 +31,39 @@ public:
void write(uint32_t data);
void writeWord(uint16_t data);
virtual void writeBytes(const std::vector<unsigned char> data) = 0;
void resetOffsets()
{
mByteOffset = -1;
mBitOffset = 0;
}
virtual void reset()
{
resetOffsets();
mCurrentByte = 0;
}
void setChecksumCalculator(AbstractChecksumCalculator* calc)
{
mChecksumCalculator = calc;
}
void clearChecksumCalculator()
{
mChecksumCalculator = nullptr;
}
//unsigned getSize() = 0;
protected:
int mByteOffset{0};
int mByteOffset{-1};
unsigned mBitOffset{0};
unsigned char mCurrentByte{0};
AbstractChecksumCalculator* mChecksumCalculator{nullptr};
};

8
src/core/streams/BufferBitStream.cpp
View file

@ -1,8 +1,10 @@
#include "BufferBitStream.h"
#include <iostream>
bool BufferBitStream::isFinished() const
{
return mByteOffset == mBuffer.size();
return mByteOffset == mBuffer.size() - 1;
}
std::vector<unsigned char> BufferBitStream::peekNextNBytes(unsigned n) const
@ -43,6 +45,10 @@ void BufferBitStream::setBuffer(const std::vector<unsigned char>& data)
void BufferBitStream::writeByte(unsigned char data)
{
if (mChecksumCalculator)
{
mChecksumCalculator->addValue(data);
}
mBuffer.push_back(data);
}

7
src/core/streams/BufferBitStream.h
View file

@ -27,12 +27,13 @@ public:
return mBuffer;
}
void resetOffsets()
void reset() override
{
mByteOffset = 0;
mBitOffset = 0;
BitStream::reset();
mBuffer.clear();
}
private:
unsigned mBufferSize{0};
std::vector<unsigned char> mBuffer;
};

1
src/image/CMakeLists.txt
View file

@ -8,6 +8,7 @@ list(APPEND image_LIB_INCLUDES
png/PngWriter.cpp
png/PngReader.cpp
png/PngHeader.cpp
png/PngInfo.cpp
ImageBitStream.cpp
)

11
src/image/ImageBitStream.cpp
View file

@ -9,7 +9,7 @@ ImageBitStream::ImageBitStream(Image<unsigned char>* image)
bool ImageBitStream::isFinished() const
{
return true;
return mByteOffset == mImage->getDataRef().size();
}
std::vector<unsigned char> ImageBitStream::peekNextNBytes(unsigned n) const
@ -19,7 +19,14 @@ std::vector<unsigned char> ImageBitStream::peekNextNBytes(unsigned n) const
std::optional<unsigned char> ImageBitStream::readNextByte()
{
return std::nullopt;
mByteOffset++;
if (isFinished() )
{
return std::nullopt;
}
const auto val = mImage->getDataRef()[mByteOffset];
return val;
}
void ImageBitStream::writeByte(unsigned char data)

5
src/image/ImageBitStream.h
View file

@ -22,6 +22,11 @@ public:
}
unsigned getBytesPerScanline() const
{
return mImage->getBytesPerRow();
}
private:
Image<unsigned char>* mImage{nullptr};
};

73
src/image/png/PngFilter.h Normal file
View file

@ -0,0 +1,73 @@
#pragma once
#include "ImageBitStream.h"
#include "BitStream.h"
#include "FileLogger.h"
#include <iostream>
class PngFilter
{
public:
enum class FilterType : unsigned char
{
NONE,
SUB,
UP,
AVERAGE,
PAETH
};
PngFilter(BitStream* inputStream, BitStream* outputStream)
: mInputStream(inputStream),
mOutputStream(outputStream)
{
}
void encode()
{
auto image_stream = dynamic_cast<ImageBitStream*>(mInputStream);
if (!image_stream)
{
MLOG_ERROR("Expected ImageStream in PngFilter encode - aborting.");
return;
}
const auto bytes_per_scanline = image_stream->getBytesPerScanline();
unsigned count{0};
if (mFilterType == FilterType::NONE)
{
while(true)
{
if (const auto byte = image_stream->readNextByte())
{
if (count % bytes_per_scanline == 0)
{
mOutputStream->writeByte(0);
}
mOutputStream->writeByte(*byte);
}
else
{
break;
}
count++;
}
}
}
void decode()
{
}
private:
FilterType mFilterType{FilterType::NONE};
BitStream* mInputStream{nullptr};
BitStream* mOutputStream{nullptr};
};

8
src/image/png/PngHeader.cpp
View file

@ -37,7 +37,7 @@ std::string PngHeader::getFileName() const
const std::string& PngHeader::getChunkName() const
{
return "IHDR";
return mName;
}
const std::vector<unsigned char>& PngHeader::getData() const
@ -61,9 +61,9 @@ void PngHeader::updateData()
}
mData.push_back(mBitDepth);
mData.push_back(static_cast<unsigned char>(mPngInfo.mColorType));
mData.push_back(mPngInfo.mCompressionMethod);
mData.push_back(mPngInfo.mFilterMethod);
mData.push_back(mPngInfo.mInterlaceMethod);
mData.push_back(static_cast<unsigned char>(mPngInfo.mCompressionMethod));
mData.push_back(static_cast<unsigned char>(mPngInfo.mFilterMethod));
mData.push_back(static_cast<unsigned char>(mPngInfo.mInterlaceMethod));
}
uint32_t PngHeader::getCrc() const

95
src/image/png/PngInfo.cpp Normal file
View file

@ -0,0 +1,95 @@
#include "PngInfo.h"
std::string PngInfo::toString(ColorType colorType) const
{
switch(colorType)
{
case ColorType::GREYSCALE:
return "GREYSCALE";
case ColorType::RGB:
return "RGB";
case ColorType::PALETTE:
return "PALETTE";
case ColorType::GREYSCALE_ALPHA:
return "GREYSCALE_ALPHA";
case ColorType::RGB_ALPHA:
return "RGB_ALPHA";
default:
return "UNKNOWN";
}
}
std::string PngInfo::toString(CompressionMethod method) const
{
switch(method)
{
case CompressionMethod::DEFLATE:
return "DEFLATE";
default:
return "UNKNOWN";
}
}
std::string PngInfo::toString(FilterMethod method) const
{
switch(method)
{
case FilterMethod::ADAPTIVE:
return "ADAPTIVE";
default:
return "UNKNOWN";
}
}
std::string PngInfo::toString(InterlaceMethod method) const
{
switch(method)
{
case InterlaceMethod::NONE:
return "NONE";
case InterlaceMethod::ADAM7:
return "ADAM7";
default:
return "UNKNOWN";
}
}
bool PngInfo::bitDepthIsValid(ColorType colorType, unsigned char bitDepth) const
{
switch(colorType)
{
case ColorType::GREYSCALE:
return (bitDepth == 1) || (bitDepth == 2) || (bitDepth == 4) || (bitDepth == 8) || (bitDepth == 16) ;
case ColorType::RGB:
return (bitDepth == 8) || (bitDepth == 16);
case ColorType::PALETTE:
return (bitDepth == 1) || (bitDepth == 2) || (bitDepth == 4) || (bitDepth == 8);
case ColorType::GREYSCALE_ALPHA:
return (bitDepth == 8) || (bitDepth == 16);
case ColorType::RGB_ALPHA:
return (bitDepth == 8) || (bitDepth == 16);
default:
return false;
}
}
bool PngInfo::compressionMethodIsValid(unsigned char method)
{
return method == 0;
}
bool PngInfo::filterMethodIsValid(unsigned char method)
{
return method == 0;
}
std::string PngInfo::toString() const
{
std::stringstream sstr;
sstr << "PngInfo" << "\n";
sstr << "colorType: " << toString(mColorType) << "\n";
sstr << "compressionMethod: " << toString(mCompressionMethod) << "\n";
sstr << "filterMethod: " << toString(mFilterMethod) << "\n";
sstr << "interlaceMethod: " << toString(mInterlaceMethod) << "\n";
return sstr.str();
}

62
src/image/png/PngInfo.h
View file

@ -12,43 +12,45 @@ public:
RGB = 2,
PALETTE = 3,
GREYSCALE_ALPHA = 4,
RGB_ALPHA = 6,
RGB_ALPHA = 6
};
std::string toString(ColorType colorType) const
enum class CompressionMethod : unsigned char
{
switch(colorType)
{
case ColorType::GREYSCALE:
return "GREYSCALE";
case ColorType::RGB:
return "RGB";
case ColorType::PALETTE:
return "PALETTE";
case ColorType::GREYSCALE_ALPHA:
return "GREYSCALE_ALPHA";
case ColorType::RGB_ALPHA:
return "RGB_ALPHA";
default:
return "UNKNOWN";
}
}
DEFLATE = 0
};
std::string toString() const
enum class FilterMethod : unsigned char
{
std::stringstream sstr;
sstr << "PngInfo" << "\n";
sstr << "colorType: " << toString(mColorType) << "\n";
sstr << "compressionMethod: " << (int)mCompressionMethod << "\n";
sstr << "filterMethod: " << (int)mFilterMethod << "\n";
sstr << "interlaceMethod: " << (int)mInterlaceMethod << "\n";
return sstr.str();
}
ADAPTIVE = 0
};
enum class InterlaceMethod : unsigned char
{
NONE = 0,
ADAM7 = 1
};
std::string toString(ColorType colorType) const;
std::string toString(CompressionMethod method) const;
std::string toString(FilterMethod method) const;
std::string toString(InterlaceMethod method) const;
bool bitDepthIsValid(ColorType colorType, unsigned char bitDepth) const;
bool compressionMethodIsValid(unsigned char method);
bool filterMethodIsValid(unsigned char method);
std::string toString() const;
ColorType mColorType{ColorType::RGB};
unsigned char mCompressionMethod{0};
unsigned char mFilterMethod{0};
unsigned char mInterlaceMethod{0};
CompressionMethod mCompressionMethod{CompressionMethod::DEFLATE};
FilterMethod mFilterMethod{FilterMethod::ADAPTIVE};
InterlaceMethod mInterlaceMethod{InterlaceMethod::NONE};
};

26
src/image/png/PngReader.cpp
View file

@ -86,22 +86,9 @@ bool PngReader::readChunk()
void PngReader::readIDATChunk(unsigned length)
{
if (mAwaitingDataBlock)
for(unsigned idx=0; idx<length; idx++)
{
mEncoder->setCompressionMethod(*mFile->readNextByte());
mEncoder->setExtraFlags(*mFile->readNextByte());
for(unsigned idx=0; idx<length-2; idx++)
{
mInputStream->writeByte(*mFile->readNextByte());
}
mAwaitingDataBlock = false;
}
else
{
for(unsigned idx=0; idx<length; idx++)
{
mInputStream->writeByte(*mFile->readNextByte());
}
mInputStream->writeByte(*mFile->readNextByte());
}
}
@ -114,11 +101,10 @@ void PngReader::readHeaderChunk()
mHeader.setImageData(width, height, bitDepth);
PngInfo info;
info.mColorType = static_cast<PngInfo::ColorType>(mFile->GetInHandle()->get());
info.mCompressionMethod = mFile->GetInHandle()->get();
info.mFilterMethod = mFile->GetInHandle()->get();
info.mInterlaceMethod = mFile->GetInHandle()->get();
info.mCompressionMethod = static_cast<PngInfo::CompressionMethod>(mFile->GetInHandle()->get());
info.mFilterMethod = static_cast<PngInfo::FilterMethod>(mFile->GetInHandle()->get());
info.mInterlaceMethod = static_cast<PngInfo::InterlaceMethod>(mFile->GetInHandle()->get());
mHeader.setPngInfo(info);
mCurrentOffset += 13;
@ -152,6 +138,8 @@ std::unique_ptr<Image<unsigned char> > PngReader::read()
{
}
std::cout << mEncoder->getData() << std::endl;
mEncoder->decode();
return std::move(image);
}

60
src/image/png/PngWriter.cpp
View file

@ -6,13 +6,14 @@
#include "OutputBitStream.h"
#include "ImageBitStream.h"
#include "PngFilter.h"
#include "Lz77Encoder.h"
#include "ZlibEncoder.h"
#include "CyclicRedundancyChecker.h"
#include "ByteUtils.h"
#include <stdio.h>
#include <iostream>
PngWriter::PngWriter()
{
@ -82,6 +83,8 @@ void PngWriter::writeHeader()
mPngHeader.updateData();
mOutStream->writeBytes(mPngHeader.getData());
std::cout << "Writing header " << mPngHeader.toString() << std::endl;
auto crc = mPngHeader.getCrc();
mOutStream->write(crc);
}
@ -92,17 +95,24 @@ void PngWriter::writeEndChunk()
mOutStream->write(length);
mOutStream->writeBytes(StringUtils::toBytes("IEND"));
std::vector<unsigned char> char_data = StringUtils::toBytes("IEND");
CyclicRedundancyChecker crc_check;
auto crc = crc_check.doCrc(nullptr, 0);
auto crc = crc_check.doCrc(char_data.data(), char_data.size());
mOutStream->write(crc);
std::cout << "Writing end chunk" << std::endl;
}
void PngWriter::writeDataChunks(const BufferBitStream& buffer)
{
auto num_bytes = buffer.getBuffer().size();
auto max_bytes{32000};
std::vector<unsigned char> crc_buffer(max_bytes, 0);
std::vector<unsigned char> crc_buffer(num_bytes + 4, 0);
crc_buffer[0] = 'I';
crc_buffer[1] = 'D';
crc_buffer[2] = 'A';
crc_buffer[3] = 'T';
unsigned num_dat_chunks = num_bytes/max_bytes + 1;
unsigned offset = 0;
@ -114,19 +124,22 @@ void PngWriter::writeDataChunks(const BufferBitStream& buffer)
length = num_bytes - num_dat_chunks*num_bytes;
}
mOutStream->write(length);
std::cout << "Writing idat length " << num_bytes << std::endl;
mOutStream->write(num_bytes);
mOutStream->writeBytes(StringUtils::toBytes("IDAT"));
for(unsigned jdx=0; jdx<length; jdx++)
for(unsigned jdx=0; jdx<num_bytes; jdx++)
{
auto val = buffer.getBuffer()[idx*max_bytes + jdx];
crc_buffer[jdx] = val;
crc_buffer[jdx + 4] = val;
mOutStream->writeByte(val);
}
CyclicRedundancyChecker crc_check;
auto crc = crc_check.doCrc(crc_buffer.data(), crc_buffer.size());
std::cout << "Writing idat crc" << crc << std::endl;
mOutStream->write(crc);
}
}
@ -146,18 +159,41 @@ void PngWriter::write(const std::unique_ptr<Image<unsigned char> >& image)
}
mWorkingImage = image.get();
mInStream = std::make_unique<ImageBitStream>(image.get());
auto image_bit_stream = std::make_unique<ImageBitStream>(image.get());
auto raw_image_stream = image_bit_stream.get();
mInStream = std::move(image_bit_stream);
writeHeader();
BufferBitStream lz77_out_stream;
Lz77Encoder lz77_encoder(mInStream.get(), &lz77_out_stream);
auto filter_out_stream = std::make_unique<BufferBitStream>();
PngFilter filter(raw_image_stream, filter_out_stream.get());
filter.encode();
//while(!filter_out_stream->isFinished())
//{
//std::cout << "Got pix " << static_cast<int>(*filter_out_stream->readNextByte()) << std::endl;
//}
lz77_encoder.encode();
lz77_out_stream.resetOffsets();
filter_out_stream->resetOffsets();
std::unique_ptr<BufferBitStream> lz77_out_stream;
if (mCompressionMethod == Deflate::CompressionMethod::NONE)
{
lz77_out_stream = std::move(filter_out_stream);
}
else
{
lz77_out_stream = std::make_unique<BufferBitStream>();
Lz77Encoder lz77_encoder(filter_out_stream.get(), lz77_out_stream.get());
lz77_encoder.encode();
lz77_out_stream->resetOffsets();
}
BufferBitStream zlib_out_stream;
ZlibEncoder zlib_encoder(&lz77_out_stream, &zlib_out_stream);
ZlibEncoder zlib_encoder(lz77_out_stream.get(), &zlib_out_stream);
zlib_encoder.encode();
zlib_out_stream.resetOffsets();

3
src/image/png/PngWriter.h
View file

@ -2,6 +2,7 @@
#include "PngHeader.h"
#include "Image.h"
#include "DeflateElements.h"
#include <memory>
#include <string>
@ -46,6 +47,8 @@ private:
unsigned mPngInfoUserSet{false};
PngInfo mPngInfo;
PngHeader mPngHeader;
Deflate::CompressionMethod mCompressionMethod{Deflate::CompressionMethod::NONE};
};
using PngWriterPtr = std::unique_ptr<PngWriter>;

15
test/image/TestPngWriter.cpp
View file

@ -9,21 +9,26 @@
int main()
{
unsigned width = 20;
unsigned height = 20;
unsigned numChannels = 3;
unsigned width = 10;
unsigned height = 10;
unsigned numChannels = 1;
auto image = Image<unsigned char>::Create(width, height);
image->setNumChannels(numChannels);
image->setBitDepth(8);
std::vector<unsigned char> data(image->getBytesPerRow()*height, 0);
std::vector<unsigned char> data(width*height, 0);
for (unsigned idx=0; idx<width*height; idx++)
{
data[idx] = 10;
}
ImagePrimitives::drawAlternatingStrips(data, width, height, numChannels, image->getBytesPerRow());
image->setData(data);
PngWriter writer;
writer.setPath("test.png");
writer.write(image);
return 0;
File test_file("test.png");
test_file.SetAccessMode(File::AccessMode::Read);
test_file.Open(true);