jgrogan/stuff-from-scratch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Work on dynamic huffman coding.

Browse source
This commit is contained in:
James Grogan 2022-11-28 18:05:39 +00:00
parent 7f5009fb5e
commit a6e31c8d39
16 changed files with 456 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/compression/deflate/DeflateBlock.cpp
View file

@ -39,7 +39,7 @@ bool DeflateBlock::read()
{
auto working_byte = *mInputStream->readNextByte();
std::cout << mInputStream->logNextNBytes(11);
std::cout << mInputStream->logNextNBytes(60);
std::cout << "DeflateBlock::read location " << mInputStream->logLocation();
unsigned char final_block{0};

2
src/compression/deflate/DeflateEncoder.cpp
View file

@ -84,6 +84,6 @@ bool DeflateEncoder::decode()
break;
}
return false;
return true;
}

88
src/compression/huffman/HuffmanCodeLengthTable.cpp
View file

@ -2,6 +2,7 @@
#include "ByteUtils.h"
#include "RunLengthEncoder.h"
#include "BitStream.h"
#include <algorithm>
#include <sstream>
@ -24,20 +25,33 @@ void HuffmanCodeLengthTable::buildCompressedLengthSequence()
const auto count = entry.second;
if (count < 3)
{
for(unsigned idx=0; idx<3; idx++)
for(unsigned idx=0; idx<count; idx++)
{
mCompressedLengthSequence.push_back({length, 0});
}
}
else if (length == 0)
{
if(count >=3 && count <=10)
unsigned num_big = count / 138;
for(unsigned idx=0; idx<num_big; idx++)
{
mCompressedLengthSequence.push_back({17, count-3});
mCompressedLengthSequence.push_back({18, 127});
}
auto remainder_big = count % 138;
if (remainder_big > 10)
{
mCompressedLengthSequence.push_back({18, remainder_big-11});
}
else if(remainder_big > 2)
{
mCompressedLengthSequence.push_back({17, remainder_big-3});
}
else
{
mCompressedLengthSequence.push_back({18, count-11});
for(unsigned idx=0; idx<remainder_big; idx++)
{
mCompressedLengthSequence.push_back({0, 0});
}
}
}
else
@ -80,6 +94,69 @@ const std::vector<unsigned> HuffmanCodeLengthTable::getCompressedLengthCounts()
return mCompressedLengthCounts;
}
std::optional<PrefixCode> HuffmanCodeLengthTable::getCodeForSymbol(unsigned symbol) const
{
return mTree.getCode(symbol);
}
bool HuffmanCodeLengthTable::readNextSymbol(unsigned& result, BitStream* stream)
{
if (getNumCodeLengths() == 0)
{
return false;
}
unsigned working_index{0};
auto length = getCodeLength(working_index);
auto delta = length;
bool found{false};
unsigned char buffer{0};
uint32_t working_bits{0};
unsigned working_symbol{0};
while(!found)
{
auto valid = stream->readNextNBits(delta, buffer);
//std::cout << "Got buffer " << ByteUtils::toString(buffer) << std::endl;;
unsigned hold = buffer;
working_bits = working_bits | (hold << (length - delta));
//std::cout << "Read " << delta << " bits with length " << length << " and value " << ByteUtils::toString(working_bits) << std::endl;
if (const auto symbol = findMatch(working_index, working_bits))
{
found = true;
working_symbol = *symbol;
}
else
{
working_index++;
if (working_index >= getNumCodeLengths())
{
break;
}
auto new_length = getCodeLength(working_index);
delta = new_length - length;
length = new_length;
}
}
if (found)
{
result = working_symbol;
// std::cout << "Found symbol " << working_symbol << " with bits " << ByteUtils::toString(working_bits) << std::endl;
// std::cout << "At Byte offset " << stream->getCurrentByteOffset() << " and bit offset " << stream->getCurrentBitOffset() << std::endl;
return true;
}
else
{
std::cout << "SYMBOL NOT FOUND " << " with bits " << ByteUtils::toString(working_bits) << " and index " << working_index << std::endl;
return false;
}
}
void HuffmanCodeLengthTable::buildPrefixCodes()
{
if(mInputLengthSequence.empty())
@ -100,6 +177,7 @@ void HuffmanCodeLengthTable::buildPrefixCodes()
for (unsigned bits = 1; bits <= max_length; bits++)
{
code = (code + counts[bits-1]) << 1;
//std::cout << "Start code for bit " << bits << " is " << ByteUtils::toString(code) << " | dec " << code << " count " << counts[bits-1] << std::endl;
next_code[bits] = code;
}
@ -115,7 +193,7 @@ void HuffmanCodeLengthTable::buildPrefixCodes()
}
}
mTree.sortTable();
//std::cout << dumpPrefixCodes();
std::cout << dumpPrefixCodes();
}
const PrefixCode& HuffmanCodeLengthTable::getCode(std::size_t index) const

6
src/compression/huffman/HuffmanCodeLengthTable.h
View file

@ -6,6 +6,8 @@
#include <string>
#include <optional>
class BitStream;
class HuffmanCodeLengthTable
{
public:
@ -21,6 +23,8 @@ public:
const PrefixCode& getCode(std::size_t index) const;
std::optional<PrefixCode> getCodeForSymbol(unsigned symbol) const;
using CompressedSequenceEntry = std::pair<unsigned, unsigned>;
const std::vector<CompressedSequenceEntry>& getCompressedLengthSequence() const;
@ -34,6 +38,8 @@ public:
void setInputLengthSequence(const std::vector<unsigned char>& sequence, bool targetDeflate = true);
bool readNextSymbol(unsigned& buffer, BitStream* stream);
private:
HuffmanTree mTree;

258
src/compression/huffman/HuffmanStream.cpp
View file

@ -8,6 +8,13 @@
#include <unordered_map>
#include <sstream>
std::vector<unsigned> DISTANCE_OFFSETS
{
5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193,
258, 385, 513, 769, 1025, 1537, 2049, 3073, 4097,
6145, 8193, 12289, 16385, 24577
};
HuffmanStream::HuffmanStream(BitStream* inputStream, BitStream* outputStream)
: mInputStream(inputStream),
mOutputStream(outputStream)
@ -22,69 +29,78 @@ void HuffmanStream::generateFixedCodeMapping()
mCodeLengthTable.buildPrefixCodes();
}
bool HuffmanStream::readNextCodeLengthSymbol(unsigned& final_symbol)
bool HuffmanStream::readNextCodeLengthSymbol(unsigned& buffer)
{
if (mCodeLengthTable.getNumCodeLengths() == 0)
return mCodeLengthTable.readNextSymbol(buffer, mInputStream);
}
bool HuffmanStream::readNextLiteralSymbol(unsigned& buffer)
{
return mLiteralTable.readNextSymbol(buffer, mInputStream);
}
bool HuffmanStream::readNextDistanceSymbol(unsigned& buffer)
{
unsigned base_symbol{0};
unsigned char extra_bits{0};
const auto valid = mDistanceTable.readNextSymbol(base_symbol, mInputStream);
if (!valid)
{
return false;
}
unsigned working_index{0};
auto length = mCodeLengthTable.getCodeLength(working_index);
auto delta = length;
//std::cout << "Got distance base symbol " << base_symbol << std::endl;
bool found{false};
unsigned char buffer{0};
uint32_t working_bits{0};
unsigned working_symbol{0};
while(!found)
if (base_symbol <= 3)
{
auto valid = mInputStream->readNextNBits(delta, buffer);
//std::cout << "Got buffer " << ByteUtils::toString(buffer) << std::endl;;
working_bits = working_bits | (buffer << (length - delta));
std::cout << "Read " << delta << " bits with length " << length << " and value " << ByteUtils::toString(working_bits) << std::endl;
if (const auto symbol = mCodeLengthTable.findMatch(working_index, working_bits))
{
found = true;
working_symbol = *symbol;
}
else
{
working_index++;
if (working_index >= mCodeLengthTable.getNumCodeLengths())
{
break;
}
auto new_length = mCodeLengthTable.getCodeLength(working_index);
delta = new_length - length;
length = new_length;
}
}
if (found)
{
final_symbol = working_symbol;
std::cout << "Found symbol " << working_symbol << " with bits " << ByteUtils::toString(working_bits) << std::endl;
std::cout << "At Byte offset " << mInputStream->getCurrentByteOffset() << " and bit offset " << mInputStream->getCurrentBitOffset() << std::endl;
return true;
buffer = 1 + base_symbol;
}
else
{
std::cout << "SYMBOL NOT FOUND " << " with bits " << ByteUtils::toString(working_bits) << std::endl;
return false;
const auto num_extra_bits = (base_symbol - 3 - 1)/2 + 1;
unsigned extra_sum{0};
if (num_extra_bits > 8)
{
auto byte_val = *mInputStream->readNextByte();
mInputStream->readNextNBits(num_extra_bits-8, extra_bits);
extra_sum = extra_bits;
extra_sum = extra_sum << (num_extra_bits - 8);
extra_sum |= byte_val;
}
else
{
mInputStream->readNextNBits(num_extra_bits, extra_bits);
extra_sum = extra_bits;
}
buffer = DISTANCE_OFFSETS[base_symbol - 4] + extra_sum;
}
return true;
}
void HuffmanStream::readLiteralCodeLengths()
void HuffmanStream::addValue(unsigned value, unsigned& count, unsigned& lastValue, std::vector<unsigned char>& literals, unsigned numLiterals, std::vector<unsigned char>& distances)
{
std::vector<unsigned> lengths;
unsigned symbol{0};
if (count < mNumLiterals)
{
literals[count] = value;
}
else
{
distances[count - mNumLiterals] = value;
}
lastValue = value;
count++;
}
while(lengths.size() < mNumLiterals)
void HuffmanStream::readCodeLengths()
{
std::vector<unsigned char> literal_lengths(288, 0);
std::vector<unsigned char> distance_lengths(32, 0);
unsigned symbol{0};
unsigned count{0};
unsigned last_value{0};
while(count < mNumLiterals + mNumDistances)
{
bool valid = readNextCodeLengthSymbol(symbol);
@ -96,18 +112,17 @@ void HuffmanStream::readLiteralCodeLengths()
if (symbol < 16)
{
lengths.push_back(symbol);
addValue(symbol, count, last_value, literal_lengths, mNumLiterals, distance_lengths);
}
else if(symbol == 16)
{
unsigned char num_reps{0};
mInputStream->readNextNBits(2, num_reps);
auto last_val = lengths[lengths.size()-1];
std::cout << "Got val 16 doing " << 3 + num_reps << std::endl;
for(unsigned idx=0; idx< 3 + num_reps; idx++)
{
lengths.push_back(last_val);
addValue(last_value, count, last_value, literal_lengths, mNumLiterals, distance_lengths);
}
}
else if(symbol == 17)
@ -118,7 +133,7 @@ void HuffmanStream::readLiteralCodeLengths()
std::cout << "Got val 17 doing " << 3 + num_reps << std::endl;
for(unsigned idx=0; idx< 3 + num_reps; idx++)
{
lengths.push_back(0);
addValue(0, count, last_value, literal_lengths, mNumLiterals, distance_lengths);
}
}
else if(symbol == 18)
@ -129,10 +144,133 @@ void HuffmanStream::readLiteralCodeLengths()
std::cout << "Got val 18 doing " << 11 + num_reps << std::endl;
for(unsigned idx=0; idx< 11 + num_reps; idx++)
{
lengths.push_back(0);
addValue(0, count, last_value, literal_lengths, mNumLiterals, distance_lengths);
}
}
}
std::cout << "Got final literal length sequence " << std::endl;
for(unsigned idx=0; idx<literal_lengths.size(); idx++)
{
std::cout << static_cast<int>(literal_lengths[idx]) << "," ;
}
std::cout << std::endl;
std::cout << "Got final distance length sequence " << std::endl;
for(unsigned idx=0; idx<distance_lengths.size(); idx++)
{
std::cout << static_cast<int>(distance_lengths[idx]) << "," ;
}
std::cout << std::endl;
mLiteralTable.setInputLengthSequence(literal_lengths, false);
mLiteralTable.buildPrefixCodes();
mDistanceTable.setInputLengthSequence(distance_lengths, false);
mDistanceTable.buildPrefixCodes();
}
void HuffmanStream::copyFromBuffer(unsigned length, unsigned distance)
{
unsigned offset = mBuffer.size() - 1 - distance;
for(unsigned idx=0; idx<length; idx++)
{
auto symbol = mBuffer[offset + idx];
mOutputStream->writeByte(symbol);
mBuffer.push_back(symbol);
}
}
void HuffmanStream::readSymbols()
{
bool hit_end_stream{false};
unsigned symbol{0};
unsigned distance{0};
while(!hit_end_stream)
{
const auto valid = readNextLiteralSymbol(symbol);
if (!valid)
{
std::cout << "Hit unknown symbol - bailing out" << std::endl;
break;
}
std::cout << "Got symbol " << symbol << std::endl;
if(symbol <= 255)
{
mOutputStream->writeByte(symbol);
mBuffer.push_back(symbol);
}
else if(symbol == 256)
{
hit_end_stream = true;
break;
}
else if (symbol <= 264)
{
auto length = 3 + symbol - 257;
const auto valid_dist = readNextDistanceSymbol(distance);
copyFromBuffer(length, distance);
}
else if (symbol <= 268)
{
unsigned char extra{0};
mInputStream->readNextNBits(1, extra);
auto length = 11 + 2*(symbol - 265) + extra;
const auto valid_dist = readNextDistanceSymbol(distance);
copyFromBuffer(length, distance);
}
else if (symbol <= 272)
{
unsigned char extra{0};
mInputStream->readNextNBits(2, extra);
auto length = 19 + 4*(symbol - 269) + extra;
const auto valid_dist = readNextDistanceSymbol(distance);
copyFromBuffer(length, distance);
}
else if (symbol <= 276)
{
unsigned char extra{0};
mInputStream->readNextNBits(3, extra);
auto length = 35 + 8*(symbol - 273) + extra;
const auto valid_dist = readNextDistanceSymbol(distance);
copyFromBuffer(length, distance);
}
else if (symbol <= 280)
{
unsigned char extra{0};
mInputStream->readNextNBits(4, extra);
auto length = 67 + 16*(symbol - 277) + extra;
const auto valid_dist = readNextDistanceSymbol(distance);
copyFromBuffer(length, distance);
}
else if (symbol <= 284)
{
unsigned char extra{0};
mInputStream->readNextNBits(5, extra);
auto length = 131 + 32*(symbol - 281) + extra;
const auto valid_dist = readNextDistanceSymbol(distance);
copyFromBuffer(length, distance);
}
else if (symbol == 285)
{
auto length = 258;
const auto valid_dist = readNextDistanceSymbol(distance);
copyFromBuffer(length, distance);
}
}
if (hit_end_stream)
{
std::cout << "Found end of stream ok" << std::endl;
}
}
bool HuffmanStream::decode()
@ -140,6 +278,14 @@ bool HuffmanStream::decode()
if (!mUsingFixedCodes)
{
readCodingsTable();
readSymbols();
std::cout << "Got final buffer size " << mBuffer.size() << std::endl;
for(unsigned idx=0; idx< 100; idx++)
{
//std::cout << idx << " | " << mBuffer[idx] << std::endl;
}
}
else
{
@ -163,8 +309,6 @@ bool HuffmanStream::decode()
}
}
return false;
}
@ -190,13 +334,13 @@ void HuffmanStream::readCodingsTable()
unsigned char buffer{0};
for(unsigned idx = 0; idx< num_code_lengths; idx++)
{
std::cout << "After codings " << mInputStream->logLocation();
mInputStream->readNextNBits(3, buffer);
std::cout << "Got coding table value " << idx << " | " << static_cast<int>(buffer) << " | " << ByteUtils::toString(buffer) << std::endl;
sequence[idx] = buffer;
}
mCodeLengthTable.setInputLengthSequence(sequence, true);
mCodeLengthTable.buildPrefixCodes();
readLiteralCodeLengths();
readCodeLengths();
}

17
src/compression/huffman/HuffmanStream.h
View file

@ -22,17 +22,30 @@ public:
private:
void readCodingsTable();
void readLiteralCodeLengths();
void readCodeLengths();
void readSymbols();
void copyFromBuffer(unsigned length, unsigned distance);
bool readNextLiteralSymbol(unsigned& buffer);
bool readNextDistanceSymbol(unsigned& buffer);
bool readNextCodeLengthSymbol(unsigned& buffer);
void addValue(unsigned value, unsigned& count, unsigned& lastValue, std::vector<unsigned char>& literals, unsigned numLiterals, std::vector<unsigned char>& distances);
BitStream* mInputStream;
BitStream* mOutputStream;
std::vector<unsigned> mBuffer;
unsigned mNumLiterals{0}; // HLIT + 257
unsigned mNumDistances{0}; // HDIST + 1
bool mUsingFixedCodes{false};
HuffmanCodeLengthTable mCodeLengthTable;
HuffmanCodeLengthTable mLiteralTable;
HuffmanCodeLengthTable mDistanceTable;
};

15
src/compression/huffman/HuffmanTree.cpp
View file

@ -81,6 +81,21 @@ std::optional<HuffmanTree::Symbol> HuffmanTree::findMatch(std::size_t treeIndex,
return std::nullopt;
}
std::optional<PrefixCode> HuffmanTree::getCode(Symbol symbol) const
{
for(const auto& entry : mTable)
{
for(const auto& data : entry.second)
{
if (data.second == symbol)
{
return data.first;
}
}
}
return std::nullopt;
}
std::size_t HuffmanTree::getNumCodeLengths() const
{
return mTable.size();

2
src/compression/huffman/HuffmanTree.h
View file

@ -46,6 +46,8 @@ public:
unsigned getCodeLength(std::size_t idx) const;
std::optional<PrefixCode> getCode(Symbol symbol) const;
void sortTable();
private:
std::vector<CodeLengthData> mTable;

38
src/core/ByteUtils.cpp
View file

@ -26,9 +26,9 @@ unsigned char ByteUtils::getByteN(uint32_t input, unsigned n)
return (input << 8*n) >> 24;
}
unsigned char ByteUtils::mirror(unsigned char byte, unsigned length)
uint32_t ByteUtils::mirror(uint32_t byte, unsigned length)
{
unsigned char ret{0};
uint32_t ret{0};
for(unsigned idx=0; idx<length; idx++)
{
if (getBitN(byte, length - 1 - idx))
@ -121,31 +121,25 @@ unsigned char ByteUtils::getFromString(const std::string& string)
std::string ByteUtils::toString(uint32_t input, unsigned length)
{
std::string ret;
std::string working;
for(unsigned idx=0; idx<length; idx++)
if (length > 8)
{
if (idx > 0 && idx % 8 == 0)
unsigned overshoot = length - 8;
for(unsigned idx=0; idx<overshoot; idx++)
{
if (ret.empty())
{
ret = working;
}
else
{
ret = working + '-' + ret;
}
working = "";
ret += getBitN(input, length - 1 - idx) ? '1' : '0';
}
ret += "-";
for(unsigned idx=0; idx<8; idx++)
{
ret += getBitN(input, 7 - idx) ? '1' : '0';
}
working += getBitN(input, 7 - idx) ? '1' : '0';
}
if (length <= 8)
else
{
ret = working;
}
else if(!working.empty())
{
ret = working + '-' + ret;
for(unsigned idx=0; idx<length; idx++)
{
ret += getBitN(input, 7 - idx) ? '1' : '0';
}
}
return ret;
}

2
src/core/ByteUtils.h
View file

@ -33,7 +33,7 @@ public:
static std::string toString(uint32_t input, unsigned length = 8);
static unsigned char mirror(unsigned char byte, unsigned length=0);
static uint32_t mirror(uint32_t input, unsigned length=0);
static void ReverseBuffer(char* buffer, char* reverse, unsigned size, unsigned targetSize);

1
src/core/streams/BitStream.cpp
View file

@ -24,7 +24,6 @@ void BitStream::write(uint32_t data)
unsigned num_bytes = sizeof(uint32_t);
for(unsigned idx=0; idx<num_bytes;idx++)
{
std::cout << "Writing byte " << idx << " for multibyte" << std::endl;
writeByte(ByteUtils::getByteN(data, idx));
}
}

2
src/core/streams/BufferBitStream.cpp
View file

@ -71,7 +71,7 @@ void BufferBitStream::writeByte(unsigned char data, bool checkOverflow)
{
mChecksumCalculator->addValue(out_byte);
}
std::cout << "Writing byte " << ByteUtils::toString(out_byte) << " had bitoffset of " << mBitOffset << std::endl;
//std::cout << "Writing byte " << ByteUtils::toString(out_byte) << " had bitoffset of " << mBitOffset << std::endl;
mBuffer.push_back(out_byte);
}

4
src/image/png/PngFilter.h
View file

@ -85,6 +85,10 @@ public:
{
image_stream->writeByte(*byte);
}
else
{
//std::cout << "Got filter type " << static_cast<int>(working_filter_type) << std::endl;
}
}
count++;
}