From 8f97e9b7a1d8a1dad6981ca21aec10aed9557d69 Mon Sep 17 00:00:00 2001 From: James Grogan Date: Thu, 24 Nov 2022 16:15:41 +0000 Subject: [PATCH] Continue png writing. --- src/compression/AbstractEncoder.h | 11 ++ src/compression/Adler32Checksum.h | 29 +++++ src/compression/CMakeLists.txt | 10 +- src/compression/DeflateEncoder.cpp | 38 ------ src/compression/ZlibData.cpp | 0 src/compression/ZlibData.h | 107 ---------------- src/compression/ZlibEncoder.cpp | 117 +++++++++++++++--- src/compression/ZlibEncoder.h | 49 +++++--- .../{ => deflate}/DeflateBlock.cpp | 74 +++++++---- src/compression/{ => deflate}/DeflateBlock.h | 29 +++-- src/compression/deflate/DeflateElements.h | 33 +++++ src/compression/deflate/DeflateEncoder.cpp | 78 ++++++++++++ .../{ => deflate}/DeflateEncoder.h | 10 +- src/core/streams/AbstractChecksumCalculator.h | 16 +++ src/core/streams/BitStream.cpp | 9 ++ src/core/streams/BitStream.h | 32 ++++- src/core/streams/BufferBitStream.cpp | 8 +- src/core/streams/BufferBitStream.h | 7 +- src/image/CMakeLists.txt | 1 + src/image/ImageBitStream.cpp | 11 +- src/image/ImageBitStream.h | 5 + src/image/png/PngFilter.h | 73 +++++++++++ src/image/png/PngHeader.cpp | 8 +- src/image/png/PngInfo.cpp | 95 ++++++++++++++ src/image/png/PngInfo.h | 62 +++++----- src/image/png/PngReader.cpp | 26 ++-- src/image/png/PngWriter.cpp | 60 +++++++-- src/image/png/PngWriter.h | 3 + test/image/TestPngWriter.cpp | 15 ++- 29 files changed, 714 insertions(+), 302 deletions(-) create mode 100644 src/compression/Adler32Checksum.h delete mode 100644 src/compression/DeflateEncoder.cpp delete mode 100644 src/compression/ZlibData.cpp delete mode 100644 src/compression/ZlibData.h rename src/compression/{ => deflate}/DeflateBlock.cpp (79%) rename src/compression/{ => deflate}/DeflateBlock.h (78%) create mode 100644 src/compression/deflate/DeflateElements.h create mode 100644 src/compression/deflate/DeflateEncoder.cpp rename src/compression/{ => deflate}/DeflateEncoder.h (51%) create mode 100644 src/core/streams/AbstractChecksumCalculator.h create mode 100644 src/image/png/PngFilter.h create mode 100644 src/image/png/PngInfo.cpp diff --git a/src/compression/AbstractEncoder.h b/src/compression/AbstractEncoder.h index f20f7ee..6e59c3c 100644 --- a/src/compression/AbstractEncoder.h +++ b/src/compression/AbstractEncoder.h @@ -1,5 +1,9 @@ #pragma once +#include "AbstractChecksumCalculator.h" + +#include + class BitStream; class AbstractEncoder @@ -14,10 +18,17 @@ public: virtual ~AbstractEncoder() = default; + void addChecksumCalculator(AbstractChecksumCalculator* calculator) + { + mChecksumCalculators.push_back(calculator); + } + virtual bool encode() = 0; virtual bool decode() = 0; protected: + + std::vector mChecksumCalculators; BitStream* mInputStream{nullptr}; BitStream* mOutputStream{nullptr}; }; diff --git a/src/compression/Adler32Checksum.h b/src/compression/Adler32Checksum.h new file mode 100644 index 0000000..d2de9e4 --- /dev/null +++ b/src/compression/Adler32Checksum.h @@ -0,0 +1,29 @@ +#pragma once + +#include "AbstractChecksumCalculator.h" + +class Adler32Checksum : public AbstractChecksumCalculator +{ +public: + void addValue(unsigned char val) override + { + mSum1 = (mSum1 + val) % MOD_ADLER32; + mSum2 = (mSum2 + mSum1) % MOD_ADLER32; + } + + uint32_t getChecksum() const override + { + return (mSum2 << 16) | mSum1; + } + + void reset() override + { + mSum1 = 1; + mSum2 = 0; + } + +private: + static constexpr unsigned MOD_ADLER32{65536}; + uint32_t mSum1{1}; + uint32_t mSum2{0}; +}; diff --git a/src/compression/CMakeLists.txt b/src/compression/CMakeLists.txt index 7eebe23..aa60c3e 100644 --- a/src/compression/CMakeLists.txt +++ b/src/compression/CMakeLists.txt @@ -3,10 +3,9 @@ list(APPEND compression_LIB_INCLUDES StreamCompressor.cpp HuffmanEncoder.cpp RunLengthEncoder.cpp - ZlibData.cpp ZlibEncoder.cpp - DeflateEncoder.cpp - DeflateBlock.cpp + deflate/DeflateEncoder.cpp + deflate/DeflateBlock.cpp Lz77Encoder.cpp CyclicRedundancyChecker.cpp ) @@ -14,9 +13,10 @@ list(APPEND compression_LIB_INCLUDES add_library(compression SHARED ${compression_LIB_INCLUDES}) target_include_directories(compression PUBLIC - "${CMAKE_CURRENT_SOURCE_DIR}" + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/deflate ) target_link_libraries(compression PUBLIC core) set_property(TARGET compression PROPERTY FOLDER src) -set_target_properties( compression PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON ) \ No newline at end of file +set_target_properties( compression PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON ) diff --git a/src/compression/DeflateEncoder.cpp b/src/compression/DeflateEncoder.cpp deleted file mode 100644 index 6df5680..0000000 --- a/src/compression/DeflateEncoder.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "DeflateEncoder.h" - -#include "BitStream.h" -#include "DeflateBlock.h" - -#include - -DeflateEncoder::DeflateEncoder(BitStream* inputStream, BitStream* outputStream) - : AbstractEncoder(inputStream, outputStream) -{ - -} - -DeflateEncoder::~DeflateEncoder() -{ - -} - -bool DeflateEncoder::encode() -{ - return false; -} - -bool DeflateEncoder::decode() -{ - auto working_block = std::make_unique(mInputStream, mOutputStream); - working_block->readHeader(); - - DeflateBlock* raw_block = working_block.get(); - - while(!raw_block->isFinalBlock()) - { - break; - } - - return false; -} - diff --git a/src/compression/ZlibData.cpp b/src/compression/ZlibData.cpp deleted file mode 100644 index e69de29..0000000 diff --git a/src/compression/ZlibData.h b/src/compression/ZlibData.h deleted file mode 100644 index 02a8c89..0000000 --- a/src/compression/ZlibData.h +++ /dev/null @@ -1,107 +0,0 @@ -#pragma once - -#include "ByteUtils.h" -#include "BitStream.h" - -#include -#include - -class ZlibData -{ -public: - void setByte(unsigned idx, unsigned char data) - { - mBitStream.setByte(idx, data); - } - - void setDataSize(std::size_t size) - { - mBitStream.setBufferSize(size); - } - - void setCompressionMethod(unsigned char method) - { - std::cout << "Got compression input " << static_cast(method) << std::endl; - mCmf = method; - mCompressionMethod = ByteUtils::getLowerNBits(method, 4); - mCompressionInfo = ByteUtils::getHigherNBits(method, 4); - - std::cout << "Got compression method " << static_cast(mCompressionMethod) << " and info " << static_cast(mCompressionInfo) << std::endl; - } - - void setExtraFlags(unsigned char extraFlags) - { - std::cout << "Got flags " << static_cast(extraFlags) << std::endl; - - mFlg = extraFlags; - mFlagCheck = ByteUtils::getLowerNBits(extraFlags, 5); - mFlagDict = ByteUtils::getBitN(extraFlags, 5); - mFlagLevel = ByteUtils::getHigherNBits(extraFlags, 2); - - std::cout << "Got flag check " << static_cast(mFlagCheck) << " and dict " << static_cast(mFlagDict) << " and level " << static_cast(mFlagLevel) << std::endl; - } - - void processData() - { - unsigned char NO_COMPRESSION = 0x00; - unsigned char FIXED_HUFFMAN = 0x01; - unsigned char DYNAMIC_HUFFMAN = 0x02; - unsigned char ERROR = 0x03; - - bool in_final_block = false; - while(mBitStream.loadNextByte()) - { - auto working_byte = mBitStream.getCurrentByte(); - std::cout << "Into process data, byte is: " << static_cast(working_byte) << std::endl; - - unsigned char final_block{0}; - mBitStream.getNextNBits(1, final_block); - if (final_block) - { - std::cout << "Got final block" << std::endl; - in_final_block = true; - } - - unsigned char compress_type{0}; - mBitStream.getNextNBits(2, compress_type); - std::cout << "Compress type byte is: " << static_cast(compress_type) << std::endl; - if (compress_type == NO_COMPRESSION) - { - std::cout << "Got NO_COMPRESSION" << std::endl; - } - else if (compress_type == FIXED_HUFFMAN) - { - std::cout << "Got FIXED_HUFFMAN" << std::endl; - } - else if (compress_type == DYNAMIC_HUFFMAN) - { - std::cout << "Got DYNAMIC_HUFFMAN" << std::endl; - - unsigned char h_list{0}; - mBitStream.getNextNBits(5, h_list); - mHlist = h_list + 257; - std::cout << "Got HLIST " << mHlist << std::endl; - - } - else if (compress_type == ERROR) - { - std::cout << "Got ERROR" << std::endl; - } - break; - } - } - -private: - BitStream mBitStream; - - unsigned mHlist{0}; - - unsigned char mCmf{0}; - unsigned char mFlg{0}; - unsigned char mCompressionMethod{0}; - unsigned char mCompressionInfo{0}; - unsigned char mFlagCheck{0}; - unsigned char mFlagDict{0}; - unsigned char mFlagLevel{0}; - unsigned char mCheckValue{0}; -}; diff --git a/src/compression/ZlibEncoder.cpp b/src/compression/ZlibEncoder.cpp index a8425d4..9a456e3 100644 --- a/src/compression/ZlibEncoder.cpp +++ b/src/compression/ZlibEncoder.cpp @@ -3,13 +3,18 @@ #include "ByteUtils.h" #include "DeflateEncoder.h" #include "FileLogger.h" +#include "BitStream.h" +#include "Adler32Checksum.h" + +#include #include +#include ZlibEncoder::ZlibEncoder(BitStream* inputStream, BitStream* outputStream) : AbstractEncoder(inputStream, outputStream) { - + mChecksumCalculator = std::make_unique(); } ZlibEncoder::~ZlibEncoder() @@ -17,58 +22,132 @@ ZlibEncoder::~ZlibEncoder() } -void ZlibEncoder::setCompressionMethod(unsigned char method) +void ZlibEncoder::setWindowSize(unsigned size) { - std::cout << "Got compression input " << static_cast(method) << std::endl; - mCmf = method; - mCompressionMethod = ByteUtils::getLowerNBits(method, 4); - mCompressionInfo = ByteUtils::getHigherNBits(method, 4); - - std::cout << "Got compression method " << static_cast(mCompressionMethod) << " and info " << static_cast(mCompressionInfo) << std::endl; + mWindowSize = size; } -void ZlibEncoder::setExtraFlags(unsigned char extraFlags) +std::string ZlibEncoder::toString(CompressionLevel level) const +{ + switch(level) + { + case CompressionLevel::FASTEST: + return "FASTEST"; + case CompressionLevel::FAST: + return "FAST"; + case CompressionLevel::DEFAULT: + return "DEFAULT"; + case CompressionLevel::MAX_COMPRESSION: + return "MAX_COMPRESSION"; + default: + return "UNKNOWN"; + } +} + +std::string ZlibEncoder::toString(CompressionMethod method) const +{ + return method == CompressionMethod::DEFLATE ? "DEFLATE" : "UNKNOWN"; +} + +void ZlibEncoder::parseCompressionMethod(unsigned char method) +{ + std::cout << "Got compression input " << static_cast(method) << std::endl; + mCompressionMethod = static_cast(ByteUtils::getLowerNBits(method, 4)); + auto compression_info = ByteUtils::getHigherNBits(method, 4); + + if (mCompressionMethod == CompressionMethod::DEFLATE) + { + mWindowSize = pow(2, compression_info + 8); + } +} + +void ZlibEncoder::parseExtraFlags(unsigned char extraFlags) { std::cout << "Got flags " << static_cast(extraFlags) << std::endl; - mFlg = extraFlags; mFlagCheck = ByteUtils::getLowerNBits(extraFlags, 5); - mFlagDict = ByteUtils::getBitN(extraFlags, 5); - mFlagLevel = ByteUtils::getHigherNBits(extraFlags, 2); + mUseDictionary = bool(ByteUtils::getBitN(extraFlags, 5)); + mFlagLevel = static_cast(ByteUtils::getHigherNBits(extraFlags, 2)); +} + +std::string ZlibEncoder::getData() const +{ + std::stringstream sstream; + sstream << "ZlibEncoder data \n"; + sstream << "Compression method: " << toString(mCompressionMethod) << '\n'; + sstream << "Window size: " << mWindowSize << '\n'; + sstream << "Flag check: " << mFlagCheck << '\n'; + sstream << "Use dictionary: " << mUseDictionary << '\n'; + sstream << "Flag level: " << toString(mFlagLevel) << '\n'; + return sstream.str(); - std::cout << "Got flag check " << static_cast(mFlagCheck) << " and dict " << static_cast(mFlagDict) << " and level " << static_cast(mFlagLevel) << std::endl; } bool ZlibEncoder::encode() { + DeflateEncoder* deflate_encoder{nullptr}; if (!mWorkingEncoder) { - if (mCompressionMethod == 8) + if (mCompressionMethod == CompressionMethod::DEFLATE) { - mWorkingEncoder = std::make_unique(mInputStream, mOutputStream); + auto uq_deflate_encoder = std::make_unique(mInputStream, mOutputStream); + deflate_encoder = uq_deflate_encoder.get(); + mWorkingEncoder = std::move(uq_deflate_encoder); + mWorkingEncoder->addChecksumCalculator(mChecksumCalculator.get()); } else { - MLOG_ERROR("Zib requested decoder not recognized: " << mCompressionMethod << " aborting encode"); + MLOG_ERROR("Zib requested decoder not recognized: " << static_cast(mCompressionMethod) << " aborting encode"); return false; } } - return mWorkingEncoder->encode(); + deflate_encoder->setCompressionMethod(mDeflateCompressionMethod); + + auto compression_info = static_cast(log2(mWindowSize) - 8); + const unsigned char compression_byte = (compression_info << 4) | static_cast(mCompressionMethod); + + std::cout << "ZlibEncoder Writing compression byte " << static_cast(compression_byte) << " with info " << static_cast(compression_info) << std::endl; + mOutputStream->writeByte(compression_byte); + + unsigned char flag_byte{0}; + flag_byte |= (static_cast(mUseDictionary) << 5); + flag_byte |= (static_cast(mFlagLevel) << 6); + + const auto mod = (unsigned(compression_byte)*256 + flag_byte) % 31; + flag_byte += (31 - mod); + + std::cout << "ZlibEncoder Writing Flag byte " << static_cast(flag_byte) << std::endl; + mOutputStream->writeByte(flag_byte); + + if(!mWorkingEncoder->encode()) + { + MLOG_ERROR("Sub-Encoder failed - aborting zlib encode"); + //return false; + } + + const auto checksum = mChecksumCalculator->getChecksum(); + std::cout << "ZlibEncoder Writing Checksum " << checksum << std::endl; + mOutputStream->write(checksum); + return true; } bool ZlibEncoder::decode() { + parseCompressionMethod(*mInputStream->readNextByte()); + parseExtraFlags(*mInputStream->readNextByte()); + if (!mWorkingEncoder) { - if (mCompressionMethod == 8) + if (mCompressionMethod == CompressionMethod::DEFLATE) { mWorkingEncoder = std::make_unique(mInputStream, mOutputStream); } else { - MLOG_ERROR("Zib requested decoder not recognized: " << mCompressionMethod << " aborting decode"); + MLOG_ERROR("Zib requested decoder not recognized: " << static_cast(mCompressionMethod) << " aborting decode"); return false; } } + return mWorkingEncoder->decode(); } diff --git a/src/compression/ZlibEncoder.h b/src/compression/ZlibEncoder.h index 53077e3..ec6653e 100644 --- a/src/compression/ZlibEncoder.h +++ b/src/compression/ZlibEncoder.h @@ -2,33 +2,54 @@ #include "AbstractEncoder.h" +#include "DeflateElements.h" + #include #include +class AbstractChecksumCalculator; + class ZlibEncoder : public AbstractEncoder { - public: + + enum class CompressionMethod : unsigned char + { + DEFLATE = 8, + }; + + enum class CompressionLevel : unsigned char + { + FASTEST, + FAST, + DEFAULT, + MAX_COMPRESSION + }; + ZlibEncoder(BitStream* inputStream, BitStream* outputStream); ~ZlibEncoder(); - void setCompressionMethod(unsigned char method); - - void setExtraFlags(unsigned char extraFlags); + void setWindowSize(unsigned size); bool encode() override; - bool decode() override; -private: - unsigned char mCmf{0}; - unsigned char mFlg{0}; - unsigned char mCompressionMethod{8}; - unsigned char mCompressionInfo{0}; - unsigned char mFlagCheck{0}; - unsigned char mFlagDict{0}; - unsigned char mFlagLevel{0}; - unsigned char mCheckValue{0}; + std::string getData() const; + std::string toString(CompressionLevel level) const; + std::string toString(CompressionMethod method) const; +private: + void parseCompressionMethod(unsigned char method); + void parseExtraFlags(unsigned char extraFlags); + + CompressionMethod mCompressionMethod{CompressionMethod::DEFLATE}; + Deflate::CompressionMethod mDeflateCompressionMethod{Deflate::CompressionMethod::NONE}; + unsigned mWindowSize{32768}; // Window size, n in 2^(n+8) bytes + + unsigned char mFlagCheck{0}; + bool mUseDictionary{false}; + CompressionLevel mFlagLevel{CompressionLevel::DEFAULT}; + + std::unique_ptr mChecksumCalculator; std::unique_ptr mWorkingEncoder; }; diff --git a/src/compression/DeflateBlock.cpp b/src/compression/deflate/DeflateBlock.cpp similarity index 79% rename from src/compression/DeflateBlock.cpp rename to src/compression/deflate/DeflateBlock.cpp index 8ae07dd..d178b53 100644 --- a/src/compression/DeflateBlock.cpp +++ b/src/compression/deflate/DeflateBlock.cpp @@ -1,9 +1,11 @@ #include "DeflateBlock.h" #include "ByteUtils.h" +#include "AbstractChecksumCalculator.h" #include #include +#include DeflateBlock::DeflateBlock(BitStream* inputStream, BitStream* outputStream) : mInputStream(inputStream), @@ -88,7 +90,7 @@ void DeflateBlock::setDistanceTableLength(unsigned length) void DeflateBlock::setIsFinalBlock(bool isFinal) { - + mInFinalBlock = isFinal; } void DeflateBlock::flushToStream() @@ -224,6 +226,22 @@ void DeflateBlock::readDynamicHuffmanTable() readLiteralCodeLengths(); } +std::string DeflateBlock::getMetaData() const +{ + std::stringstream sstr; + sstr << "DeflateBlock Metadata \n"; + + sstr << "Final block: " << mInFinalBlock << '\n'; + sstr << "Compression method: " << Deflate::toString(mCompressionMethod) << '\n'; + + return sstr.str(); +} + +bool DeflateBlock::isFinalBlock() const +{ + return mInFinalBlock; +} + void DeflateBlock::readHeader() { auto working_byte = mInputStream->getCurrentByte(); @@ -234,29 +252,33 @@ void DeflateBlock::readHeader() mInputStream->readNextNBits(1, final_block); mInFinalBlock = bool(final_block); - if (mInFinalBlock) - { - std::cout << "Got final block" << std::endl; - } - - mInputStream->readNextNBits(2, mCompressionType); - std::cout << "Compress type byte is: " << static_cast(mCompressionType) << std::endl; - if (mCompressionType == NO_COMPRESSION) - { - std::cout << "Got NO_COMPRESSION" << std::endl; - } - else if (mCompressionType == FIXED_HUFFMAN) - { - std::cout << "Got FIXED_HUFFMAN" << std::endl; - } - else if (mCompressionType == DYNAMIC_HUFFMAN) - { - std::cout << "Got DYNAMIC_HUFFMAN" << std::endl; - readDynamicHuffmanTable(); - } - else if (mCompressionType == ERROR) - { - std::cout << "Got ERROR" << std::endl; - } - + unsigned char compression_type{0}; + mInputStream->readNextNBits(2, compression_type); + mCompressionMethod = static_cast(compression_type); +} + +void DeflateBlock::write(uint16_t datalength) +{ + unsigned char working_block{0}; + working_block |= static_cast(mInFinalBlock); + working_block |= static_cast(mCompressionMethod) << 1; + + if (mCompressionMethod == Deflate::CompressionMethod::NONE) + { + std::cout << "Writing compression block header " << static_cast(working_block) << std::endl; + mOutputStream->writeByte(working_block); + + std::cout << "Writing data length " << datalength << " " << ByteUtils::toString(datalength) << std::endl; + mOutputStream->writeWord(datalength); + + std::cout << "Writing iverse data length " << ~datalength << " " << ByteUtils::toString(~datalength) << std::endl; + mOutputStream->writeWord(static_cast(~datalength)); + + for(unsigned idx=0; idxreadNextByte(); + std::cout << "Writing next byte " << static_cast(byte) << std::endl; + mOutputStream->writeByte(byte); + } + } } diff --git a/src/compression/DeflateBlock.h b/src/compression/deflate/DeflateBlock.h similarity index 78% rename from src/compression/DeflateBlock.h rename to src/compression/deflate/DeflateBlock.h index bffd269..7683b38 100644 --- a/src/compression/DeflateBlock.h +++ b/src/compression/deflate/DeflateBlock.h @@ -1,27 +1,32 @@ #pragma once +#include "DeflateElements.h" + #include "BitStream.h" +class AbstractChecksumCalculator; + class DeflateBlock { public: DeflateBlock(BitStream* inputStream, BitStream* outputStream); + void buildCodeLengthMapping(); + + std::string getMetaData() const; + + void flushToStream(); + + bool isFinalBlock() const; + void readHeader(); void readDynamicHuffmanTable(); - void buildCodeLengthMapping(); - void readLiteralCodeLengths(); bool readNextCodeLengthSymbol(unsigned char& buffer); - bool isFinalBlock() const - { - return mInFinalBlock; - } - void setCodeLengthAlphabetLengths(const std::vector& lengths); void setCodeLengthLength(unsigned length); @@ -32,7 +37,7 @@ public: void setIsFinalBlock(bool isFinal); - void flushToStream(); + void write(uint16_t datalength); private: BitStream* mInputStream; @@ -49,12 +54,6 @@ private: std::vector mCodeLengthAlphabetLengths; static constexpr unsigned CODE_LENGTH_ALPHABET_PERMUTATION[19]{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; - unsigned char mCompressionType{0}; - bool mInFinalBlock{false}; - - static constexpr unsigned char NO_COMPRESSION = 0x00; - static constexpr unsigned char FIXED_HUFFMAN = 0x01; - static constexpr unsigned char DYNAMIC_HUFFMAN = 0x02; - static constexpr unsigned char ERROR = 0x03; + Deflate::CompressionMethod mCompressionMethod{Deflate::CompressionMethod::NONE}; }; diff --git a/src/compression/deflate/DeflateElements.h b/src/compression/deflate/DeflateElements.h new file mode 100644 index 0000000..9cbc146 --- /dev/null +++ b/src/compression/deflate/DeflateElements.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace Deflate +{ + enum class CompressionMethod + { + NONE, + FIXED_HUFFMAN, + DYNAMIC_HUFFMAN, + ERROR + }; + + inline std::string toString(CompressionMethod method) + { + switch (method) + { + case CompressionMethod::NONE: + return "NONE"; + case CompressionMethod::FIXED_HUFFMAN: + return "FIXED_HUFFMAN"; + case CompressionMethod::DYNAMIC_HUFFMAN: + return "DYNAMIC_HUFFMAN"; + case CompressionMethod::ERROR: + return "ERROR"; + default: + return "UNKNOWN"; + } + } + + +} diff --git a/src/compression/deflate/DeflateEncoder.cpp b/src/compression/deflate/DeflateEncoder.cpp new file mode 100644 index 0000000..c887b19 --- /dev/null +++ b/src/compression/deflate/DeflateEncoder.cpp @@ -0,0 +1,78 @@ +#include "DeflateEncoder.h" + +#include "BitStream.h" +#include "DeflateBlock.h" +#include "BufferBitStream.h" + +#include + +DeflateEncoder::DeflateEncoder(BitStream* inputStream, BitStream* outputStream) + : AbstractEncoder(inputStream, outputStream) +{ + +} + +DeflateEncoder::~DeflateEncoder() +{ + +} + +bool DeflateEncoder::encode() +{ + uint16_t count = 0; + BufferBitStream stream; + std::unique_ptr working_block = std::make_unique(&stream, mOutputStream); + + AbstractChecksumCalculator* checksum_calc; + if (mChecksumCalculators.size() > 0) + { + std::cout << "Setting checksum calculator " << std::endl; + mOutputStream->setChecksumCalculator(mChecksumCalculators[0]); + } + + while(true) + { + if (count == mMaxBlockSize) + { + std::cout << working_block->getMetaData(); + working_block->write(count); + + working_block = std::make_unique(&stream, mOutputStream); + stream.reset(); + } + + if (auto byte = mInputStream->readNextByte()) + { + stream.writeByte(*byte); + } + else + { + stream.resetOffsets(); + working_block->setIsFinalBlock(true); + + std::cout << working_block->getMetaData(); + working_block->write(count); + break; + } + count++; + } + + mOutputStream->clearChecksumCalculator(); + return true; +} + +bool DeflateEncoder::decode() +{ + auto working_block = std::make_unique(mInputStream, mOutputStream); + working_block->readHeader(); + + DeflateBlock* raw_block = working_block.get(); + + while(!raw_block->isFinalBlock()) + { + break; + } + + return false; +} + diff --git a/src/compression/DeflateEncoder.h b/src/compression/deflate/DeflateEncoder.h similarity index 51% rename from src/compression/DeflateEncoder.h rename to src/compression/deflate/DeflateEncoder.h index fe0c890..9d74cc0 100644 --- a/src/compression/DeflateEncoder.h +++ b/src/compression/deflate/DeflateEncoder.h @@ -1,6 +1,7 @@ #pragma once #include "AbstractEncoder.h" +#include "DeflateElements.h" #include #include @@ -18,8 +19,15 @@ public: bool decode() override; + void setCompressionMethod(Deflate::CompressionMethod method) + { + mCompressionMethod = method; + } + private: - std::vector > mBlocks; + uint16_t mMaxBlockSize{65535}; + Deflate::CompressionMethod mCompressionMethod{Deflate::CompressionMethod::NONE}; + std::unique_ptr mLastBlock; }; diff --git a/src/core/streams/AbstractChecksumCalculator.h b/src/core/streams/AbstractChecksumCalculator.h new file mode 100644 index 0000000..f0a133a --- /dev/null +++ b/src/core/streams/AbstractChecksumCalculator.h @@ -0,0 +1,16 @@ +#pragma once + +#include "stdint.h" + +class AbstractChecksumCalculator +{ +public: + virtual ~AbstractChecksumCalculator() = default; + + virtual void addValue(unsigned char val) = 0; + virtual uint32_t getChecksum() const = 0; + + virtual void reset() + { + } +}; diff --git a/src/core/streams/BitStream.cpp b/src/core/streams/BitStream.cpp index 678b234..98116e9 100644 --- a/src/core/streams/BitStream.cpp +++ b/src/core/streams/BitStream.cpp @@ -28,6 +28,15 @@ void BitStream::write(uint32_t data) } } +void BitStream::writeWord(uint16_t data) +{ + unsigned num_bytes = sizeof(uint16_t); + for(unsigned idx=0; idx #include #include @@ -29,11 +31,39 @@ public: void write(uint32_t data); + void writeWord(uint16_t data); + virtual void writeBytes(const std::vector data) = 0; + void resetOffsets() + { + mByteOffset = -1; + mBitOffset = 0; + } + + virtual void reset() + { + resetOffsets(); + mCurrentByte = 0; + } + + void setChecksumCalculator(AbstractChecksumCalculator* calc) + { + mChecksumCalculator = calc; + } + + void clearChecksumCalculator() + { + mChecksumCalculator = nullptr; + } + + //unsigned getSize() = 0; + protected: - int mByteOffset{0}; + int mByteOffset{-1}; unsigned mBitOffset{0}; unsigned char mCurrentByte{0}; + + AbstractChecksumCalculator* mChecksumCalculator{nullptr}; }; diff --git a/src/core/streams/BufferBitStream.cpp b/src/core/streams/BufferBitStream.cpp index 4f794c9..2aa9fbc 100644 --- a/src/core/streams/BufferBitStream.cpp +++ b/src/core/streams/BufferBitStream.cpp @@ -1,8 +1,10 @@ #include "BufferBitStream.h" +#include + bool BufferBitStream::isFinished() const { - return mByteOffset == mBuffer.size(); + return mByteOffset == mBuffer.size() - 1; } std::vector BufferBitStream::peekNextNBytes(unsigned n) const @@ -43,6 +45,10 @@ void BufferBitStream::setBuffer(const std::vector& data) void BufferBitStream::writeByte(unsigned char data) { + if (mChecksumCalculator) + { + mChecksumCalculator->addValue(data); + } mBuffer.push_back(data); } diff --git a/src/core/streams/BufferBitStream.h b/src/core/streams/BufferBitStream.h index 4d570fc..2ace2da 100644 --- a/src/core/streams/BufferBitStream.h +++ b/src/core/streams/BufferBitStream.h @@ -27,12 +27,13 @@ public: return mBuffer; } - void resetOffsets() + void reset() override { - mByteOffset = 0; - mBitOffset = 0; + BitStream::reset(); + mBuffer.clear(); } private: + unsigned mBufferSize{0}; std::vector mBuffer; }; diff --git a/src/image/CMakeLists.txt b/src/image/CMakeLists.txt index f61ec8d..247e6f0 100644 --- a/src/image/CMakeLists.txt +++ b/src/image/CMakeLists.txt @@ -8,6 +8,7 @@ list(APPEND image_LIB_INCLUDES png/PngWriter.cpp png/PngReader.cpp png/PngHeader.cpp + png/PngInfo.cpp ImageBitStream.cpp ) diff --git a/src/image/ImageBitStream.cpp b/src/image/ImageBitStream.cpp index e65324c..83e7aca 100644 --- a/src/image/ImageBitStream.cpp +++ b/src/image/ImageBitStream.cpp @@ -9,7 +9,7 @@ ImageBitStream::ImageBitStream(Image* image) bool ImageBitStream::isFinished() const { - return true; + return mByteOffset == mImage->getDataRef().size(); } std::vector ImageBitStream::peekNextNBytes(unsigned n) const @@ -19,7 +19,14 @@ std::vector ImageBitStream::peekNextNBytes(unsigned n) const std::optional ImageBitStream::readNextByte() { - return std::nullopt; + mByteOffset++; + + if (isFinished() ) + { + return std::nullopt; + } + const auto val = mImage->getDataRef()[mByteOffset]; + return val; } void ImageBitStream::writeByte(unsigned char data) diff --git a/src/image/ImageBitStream.h b/src/image/ImageBitStream.h index 7a9324b..86af9ee 100644 --- a/src/image/ImageBitStream.h +++ b/src/image/ImageBitStream.h @@ -22,6 +22,11 @@ public: } + unsigned getBytesPerScanline() const + { + return mImage->getBytesPerRow(); + } + private: Image* mImage{nullptr}; }; diff --git a/src/image/png/PngFilter.h b/src/image/png/PngFilter.h new file mode 100644 index 0000000..02dd76f --- /dev/null +++ b/src/image/png/PngFilter.h @@ -0,0 +1,73 @@ +#pragma once + +#include "ImageBitStream.h" +#include "BitStream.h" + +#include "FileLogger.h" + +#include + +class PngFilter +{ +public: + enum class FilterType : unsigned char + { + NONE, + SUB, + UP, + AVERAGE, + PAETH + }; + + PngFilter(BitStream* inputStream, BitStream* outputStream) + : mInputStream(inputStream), + mOutputStream(outputStream) + + { + + } + + void encode() + { + auto image_stream = dynamic_cast(mInputStream); + if (!image_stream) + { + MLOG_ERROR("Expected ImageStream in PngFilter encode - aborting."); + return; + } + + const auto bytes_per_scanline = image_stream->getBytesPerScanline(); + unsigned count{0}; + if (mFilterType == FilterType::NONE) + { + while(true) + { + if (const auto byte = image_stream->readNextByte()) + { + if (count % bytes_per_scanline == 0) + { + mOutputStream->writeByte(0); + } + + mOutputStream->writeByte(*byte); + } + else + { + break; + } + count++; + } + } + } + + void decode() + { + + } + +private: + + FilterType mFilterType{FilterType::NONE}; + BitStream* mInputStream{nullptr}; + BitStream* mOutputStream{nullptr}; +}; diff --git a/src/image/png/PngHeader.cpp b/src/image/png/PngHeader.cpp index 536e9bb..b162e8e 100644 --- a/src/image/png/PngHeader.cpp +++ b/src/image/png/PngHeader.cpp @@ -37,7 +37,7 @@ std::string PngHeader::getFileName() const const std::string& PngHeader::getChunkName() const { - return "IHDR"; + return mName; } const std::vector& PngHeader::getData() const @@ -61,9 +61,9 @@ void PngHeader::updateData() } mData.push_back(mBitDepth); mData.push_back(static_cast(mPngInfo.mColorType)); - mData.push_back(mPngInfo.mCompressionMethod); - mData.push_back(mPngInfo.mFilterMethod); - mData.push_back(mPngInfo.mInterlaceMethod); + mData.push_back(static_cast(mPngInfo.mCompressionMethod)); + mData.push_back(static_cast(mPngInfo.mFilterMethod)); + mData.push_back(static_cast(mPngInfo.mInterlaceMethod)); } uint32_t PngHeader::getCrc() const diff --git a/src/image/png/PngInfo.cpp b/src/image/png/PngInfo.cpp new file mode 100644 index 0000000..f0f603e --- /dev/null +++ b/src/image/png/PngInfo.cpp @@ -0,0 +1,95 @@ +#include "PngInfo.h" + +std::string PngInfo::toString(ColorType colorType) const +{ + switch(colorType) + { + case ColorType::GREYSCALE: + return "GREYSCALE"; + case ColorType::RGB: + return "RGB"; + case ColorType::PALETTE: + return "PALETTE"; + case ColorType::GREYSCALE_ALPHA: + return "GREYSCALE_ALPHA"; + case ColorType::RGB_ALPHA: + return "RGB_ALPHA"; + default: + return "UNKNOWN"; + } +} + +std::string PngInfo::toString(CompressionMethod method) const +{ + switch(method) + { + case CompressionMethod::DEFLATE: + return "DEFLATE"; + default: + return "UNKNOWN"; + } +} + +std::string PngInfo::toString(FilterMethod method) const +{ + switch(method) + { + case FilterMethod::ADAPTIVE: + return "ADAPTIVE"; + default: + return "UNKNOWN"; + } +} + +std::string PngInfo::toString(InterlaceMethod method) const +{ + switch(method) + { + case InterlaceMethod::NONE: + return "NONE"; + case InterlaceMethod::ADAM7: + return "ADAM7"; + default: + return "UNKNOWN"; + } +} + +bool PngInfo::bitDepthIsValid(ColorType colorType, unsigned char bitDepth) const +{ + switch(colorType) + { + case ColorType::GREYSCALE: + return (bitDepth == 1) || (bitDepth == 2) || (bitDepth == 4) || (bitDepth == 8) || (bitDepth == 16) ; + case ColorType::RGB: + return (bitDepth == 8) || (bitDepth == 16); + case ColorType::PALETTE: + return (bitDepth == 1) || (bitDepth == 2) || (bitDepth == 4) || (bitDepth == 8); + case ColorType::GREYSCALE_ALPHA: + return (bitDepth == 8) || (bitDepth == 16); + case ColorType::RGB_ALPHA: + return (bitDepth == 8) || (bitDepth == 16); + default: + return false; + } +} + +bool PngInfo::compressionMethodIsValid(unsigned char method) +{ + return method == 0; +} + +bool PngInfo::filterMethodIsValid(unsigned char method) +{ + return method == 0; +} + +std::string PngInfo::toString() const +{ + std::stringstream sstr; + sstr << "PngInfo" << "\n"; + sstr << "colorType: " << toString(mColorType) << "\n"; + sstr << "compressionMethod: " << toString(mCompressionMethod) << "\n"; + sstr << "filterMethod: " << toString(mFilterMethod) << "\n"; + sstr << "interlaceMethod: " << toString(mInterlaceMethod) << "\n"; + return sstr.str(); +} diff --git a/src/image/png/PngInfo.h b/src/image/png/PngInfo.h index c5347ae..3c78f72 100644 --- a/src/image/png/PngInfo.h +++ b/src/image/png/PngInfo.h @@ -12,43 +12,45 @@ public: RGB = 2, PALETTE = 3, GREYSCALE_ALPHA = 4, - RGB_ALPHA = 6, + RGB_ALPHA = 6 }; - std::string toString(ColorType colorType) const + enum class CompressionMethod : unsigned char { - switch(colorType) - { - case ColorType::GREYSCALE: - return "GREYSCALE"; - case ColorType::RGB: - return "RGB"; - case ColorType::PALETTE: - return "PALETTE"; - case ColorType::GREYSCALE_ALPHA: - return "GREYSCALE_ALPHA"; - case ColorType::RGB_ALPHA: - return "RGB_ALPHA"; - default: - return "UNKNOWN"; - } - } + DEFLATE = 0 + }; - std::string toString() const + enum class FilterMethod : unsigned char { - std::stringstream sstr; - sstr << "PngInfo" << "\n"; - sstr << "colorType: " << toString(mColorType) << "\n"; - sstr << "compressionMethod: " << (int)mCompressionMethod << "\n"; - sstr << "filterMethod: " << (int)mFilterMethod << "\n"; - sstr << "interlaceMethod: " << (int)mInterlaceMethod << "\n"; - return sstr.str(); - } + ADAPTIVE = 0 + }; + + enum class InterlaceMethod : unsigned char + { + NONE = 0, + ADAM7 = 1 + }; + + std::string toString(ColorType colorType) const; + + std::string toString(CompressionMethod method) const; + + std::string toString(FilterMethod method) const; + + std::string toString(InterlaceMethod method) const; + + bool bitDepthIsValid(ColorType colorType, unsigned char bitDepth) const; + + bool compressionMethodIsValid(unsigned char method); + + bool filterMethodIsValid(unsigned char method); + + std::string toString() const; ColorType mColorType{ColorType::RGB}; - unsigned char mCompressionMethod{0}; - unsigned char mFilterMethod{0}; - unsigned char mInterlaceMethod{0}; + CompressionMethod mCompressionMethod{CompressionMethod::DEFLATE}; + FilterMethod mFilterMethod{FilterMethod::ADAPTIVE}; + InterlaceMethod mInterlaceMethod{InterlaceMethod::NONE}; }; diff --git a/src/image/png/PngReader.cpp b/src/image/png/PngReader.cpp index 038abf3..522717b 100644 --- a/src/image/png/PngReader.cpp +++ b/src/image/png/PngReader.cpp @@ -86,22 +86,9 @@ bool PngReader::readChunk() void PngReader::readIDATChunk(unsigned length) { - if (mAwaitingDataBlock) + for(unsigned idx=0; idxsetCompressionMethod(*mFile->readNextByte()); - mEncoder->setExtraFlags(*mFile->readNextByte()); - for(unsigned idx=0; idxwriteByte(*mFile->readNextByte()); - } - mAwaitingDataBlock = false; - } - else - { - for(unsigned idx=0; idxwriteByte(*mFile->readNextByte()); - } + mInputStream->writeByte(*mFile->readNextByte()); } } @@ -114,11 +101,10 @@ void PngReader::readHeaderChunk() mHeader.setImageData(width, height, bitDepth); PngInfo info; - info.mColorType = static_cast(mFile->GetInHandle()->get()); - info.mCompressionMethod = mFile->GetInHandle()->get(); - info.mFilterMethod = mFile->GetInHandle()->get(); - info.mInterlaceMethod = mFile->GetInHandle()->get(); + info.mCompressionMethod = static_cast(mFile->GetInHandle()->get()); + info.mFilterMethod = static_cast(mFile->GetInHandle()->get()); + info.mInterlaceMethod = static_cast(mFile->GetInHandle()->get()); mHeader.setPngInfo(info); mCurrentOffset += 13; @@ -152,6 +138,8 @@ std::unique_ptr > PngReader::read() { } + + std::cout << mEncoder->getData() << std::endl; mEncoder->decode(); return std::move(image); } diff --git a/src/image/png/PngWriter.cpp b/src/image/png/PngWriter.cpp index df8650a..5c17a01 100644 --- a/src/image/png/PngWriter.cpp +++ b/src/image/png/PngWriter.cpp @@ -6,13 +6,14 @@ #include "OutputBitStream.h" #include "ImageBitStream.h" +#include "PngFilter.h" #include "Lz77Encoder.h" #include "ZlibEncoder.h" #include "CyclicRedundancyChecker.h" #include "ByteUtils.h" -#include +#include PngWriter::PngWriter() { @@ -82,6 +83,8 @@ void PngWriter::writeHeader() mPngHeader.updateData(); mOutStream->writeBytes(mPngHeader.getData()); + std::cout << "Writing header " << mPngHeader.toString() << std::endl; + auto crc = mPngHeader.getCrc(); mOutStream->write(crc); } @@ -92,17 +95,24 @@ void PngWriter::writeEndChunk() mOutStream->write(length); mOutStream->writeBytes(StringUtils::toBytes("IEND")); + std::vector char_data = StringUtils::toBytes("IEND"); CyclicRedundancyChecker crc_check; - auto crc = crc_check.doCrc(nullptr, 0); + auto crc = crc_check.doCrc(char_data.data(), char_data.size()); mOutStream->write(crc); + + std::cout << "Writing end chunk" << std::endl; } void PngWriter::writeDataChunks(const BufferBitStream& buffer) { auto num_bytes = buffer.getBuffer().size(); auto max_bytes{32000}; - std::vector crc_buffer(max_bytes, 0); + std::vector crc_buffer(num_bytes + 4, 0); + crc_buffer[0] = 'I'; + crc_buffer[1] = 'D'; + crc_buffer[2] = 'A'; + crc_buffer[3] = 'T'; unsigned num_dat_chunks = num_bytes/max_bytes + 1; unsigned offset = 0; @@ -114,19 +124,22 @@ void PngWriter::writeDataChunks(const BufferBitStream& buffer) length = num_bytes - num_dat_chunks*num_bytes; } - mOutStream->write(length); + std::cout << "Writing idat length " << num_bytes << std::endl; + mOutStream->write(num_bytes); mOutStream->writeBytes(StringUtils::toBytes("IDAT")); - for(unsigned jdx=0; jdxwriteByte(val); } CyclicRedundancyChecker crc_check; auto crc = crc_check.doCrc(crc_buffer.data(), crc_buffer.size()); + + std::cout << "Writing idat crc" << crc << std::endl; mOutStream->write(crc); } } @@ -146,18 +159,41 @@ void PngWriter::write(const std::unique_ptr >& image) } mWorkingImage = image.get(); - mInStream = std::make_unique(image.get()); + + auto image_bit_stream = std::make_unique(image.get()); + auto raw_image_stream = image_bit_stream.get(); + + mInStream = std::move(image_bit_stream); writeHeader(); - BufferBitStream lz77_out_stream; - Lz77Encoder lz77_encoder(mInStream.get(), &lz77_out_stream); + auto filter_out_stream = std::make_unique(); + PngFilter filter(raw_image_stream, filter_out_stream.get()); + filter.encode(); + //while(!filter_out_stream->isFinished()) + //{ + //std::cout << "Got pix " << static_cast(*filter_out_stream->readNextByte()) << std::endl; + //} - lz77_encoder.encode(); - lz77_out_stream.resetOffsets(); + filter_out_stream->resetOffsets(); + + std::unique_ptr lz77_out_stream; + + if (mCompressionMethod == Deflate::CompressionMethod::NONE) + { + lz77_out_stream = std::move(filter_out_stream); + } + else + { + lz77_out_stream = std::make_unique(); + Lz77Encoder lz77_encoder(filter_out_stream.get(), lz77_out_stream.get()); + + lz77_encoder.encode(); + lz77_out_stream->resetOffsets(); + } BufferBitStream zlib_out_stream; - ZlibEncoder zlib_encoder(&lz77_out_stream, &zlib_out_stream); + ZlibEncoder zlib_encoder(lz77_out_stream.get(), &zlib_out_stream); zlib_encoder.encode(); zlib_out_stream.resetOffsets(); diff --git a/src/image/png/PngWriter.h b/src/image/png/PngWriter.h index 35f5856..f0bb851 100644 --- a/src/image/png/PngWriter.h +++ b/src/image/png/PngWriter.h @@ -2,6 +2,7 @@ #include "PngHeader.h" #include "Image.h" +#include "DeflateElements.h" #include #include @@ -46,6 +47,8 @@ private: unsigned mPngInfoUserSet{false}; PngInfo mPngInfo; PngHeader mPngHeader; + + Deflate::CompressionMethod mCompressionMethod{Deflate::CompressionMethod::NONE}; }; using PngWriterPtr = std::unique_ptr; diff --git a/test/image/TestPngWriter.cpp b/test/image/TestPngWriter.cpp index e3db61a..05e9097 100644 --- a/test/image/TestPngWriter.cpp +++ b/test/image/TestPngWriter.cpp @@ -9,21 +9,26 @@ int main() { - unsigned width = 20; - unsigned height = 20; - unsigned numChannels = 3; + unsigned width = 10; + unsigned height = 10; + unsigned numChannels = 1; auto image = Image::Create(width, height); image->setNumChannels(numChannels); + image->setBitDepth(8); - std::vector data(image->getBytesPerRow()*height, 0); + std::vector data(width*height, 0); + for (unsigned idx=0; idxgetBytesPerRow()); image->setData(data); PngWriter writer; writer.setPath("test.png"); writer.write(image); + return 0; File test_file("test.png"); test_file.SetAccessMode(File::AccessMode::Read); test_file.Open(true);