Source code
Revision control
Copy as Markdown
Other Tools
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "lib/jpegli/huffman.h"
#include <limits>
#include <vector>
#include "lib/jpegli/common.h"
#include "lib/jpegli/error.h"
#include "lib/jxl/base/status.h"
namespace jpegli {
// Returns the table width of the next 2nd level table, count is the histogram
// of bit lengths for the remaining symbols, len is the code length of the next
// processed symbol.
static inline int NextTableBitSize(const int* count, int len) {
int left = 1 << (len - kJpegHuffmanRootTableBits);
while (len < static_cast<int>(kJpegHuffmanMaxBitLength)) {
left -= count[len];
if (left <= 0) break;
++len;
left <<= 1;
}
return len - kJpegHuffmanRootTableBits;
}
void BuildJpegHuffmanTable(const uint32_t* count, const uint32_t* symbols,
HuffmanTableEntry* lut) {
HuffmanTableEntry code; // current table entry
HuffmanTableEntry* table; // next available space in table
int len; // current code length
int idx; // symbol index
int key; // prefix code
int reps; // number of replicate key values in current table
int low; // low bits for current root entry
int table_bits; // key length of current table
int table_size; // size of current table
// Make a local copy of the input bit length histogram.
int tmp_count[kJpegHuffmanMaxBitLength + 1] = {0};
int total_count = 0;
for (len = 1; len <= static_cast<int>(kJpegHuffmanMaxBitLength); ++len) {
tmp_count[len] = count[len];
total_count += tmp_count[len];
}
table = lut;
table_bits = kJpegHuffmanRootTableBits;
table_size = 1 << table_bits;
// Special case code with only one value.
if (total_count == 1) {
code.bits = 0;
code.value = symbols[0];
for (key = 0; key < table_size; ++key) {
table[key] = code;
}
return;
}
// Fill in root table.
key = 0;
idx = 0;
for (len = 1; len <= kJpegHuffmanRootTableBits; ++len) {
for (; tmp_count[len] > 0; --tmp_count[len]) {
code.bits = len;
code.value = symbols[idx++];
reps = 1 << (kJpegHuffmanRootTableBits - len);
while (reps--) {
table[key++] = code;
}
}
}
// Fill in 2nd level tables and add pointers to root table.
table += table_size;
table_size = 0;
low = 0;
for (len = kJpegHuffmanRootTableBits + 1;
len <= static_cast<int>(kJpegHuffmanMaxBitLength); ++len) {
for (; tmp_count[len] > 0; --tmp_count[len]) {
// Start a new sub-table if the previous one is full.
if (low >= table_size) {
table += table_size;
table_bits = NextTableBitSize(tmp_count, len);
table_size = 1 << table_bits;
low = 0;
lut[key].bits = table_bits + kJpegHuffmanRootTableBits;
lut[key].value = (table - lut) - key;
++key;
}
code.bits = len - kJpegHuffmanRootTableBits;
code.value = symbols[idx++];
reps = 1 << (table_bits - code.bits);
while (reps--) {
table[low++] = code;
}
}
}
}
// A node of a Huffman tree.
struct HuffmanTree {
HuffmanTree(uint32_t count, int16_t left, int16_t right)
: total_count(count), index_left(left), index_right_or_value(right) {}
uint32_t total_count;
int16_t index_left;
int16_t index_right_or_value;
};
void SetDepth(const HuffmanTree& p, HuffmanTree* pool, uint8_t* depth,
uint8_t level) {
if (p.index_left >= 0) {
++level;
SetDepth(pool[p.index_left], pool, depth, level);
SetDepth(pool[p.index_right_or_value], pool, depth, level);
} else {
depth[p.index_right_or_value] = level;
}
}
// Sort the root nodes, least popular first.
static JXL_INLINE bool Compare(const HuffmanTree& v0, const HuffmanTree& v1) {
return v0.total_count < v1.total_count;
}
// This function will create a Huffman tree.
//
// The catch here is that the tree cannot be arbitrarily deep.
// Brotli specifies a maximum depth of 15 bits for "code trees"
// and 7 bits for "code length code trees."
//
// count_limit is the value that is to be faked as the minimum value
// and this minimum value is raised until the tree matches the
// maximum length requirement.
//
// This algorithm is not of excellent performance for very long data blocks,
// especially when population counts are longer than 2**tree_limit, but
// we are not planning to use this with extremely long blocks.
//
void CreateHuffmanTree(const uint32_t* data, const size_t length,
const int tree_limit, uint8_t* depth) {
// For block sizes below 64 kB, we never need to do a second iteration
// of this loop. Probably all of our block sizes will be smaller than
// that, so this loop is mostly of academic interest. If we actually
// would need this, we would be better off with the Katajainen algorithm.
for (uint32_t count_limit = 1;; count_limit *= 2) {
std::vector<HuffmanTree> tree;
tree.reserve(2 * length + 1);
for (size_t i = length; i != 0;) {
--i;
if (data[i]) {
const uint32_t count = std::max(data[i], count_limit - 1);
tree.emplace_back(count, -1, static_cast<int16_t>(i));
}
}
const size_t n = tree.size();
if (n == 1) {
// Fake value; will be fixed on upper level.
depth[tree[0].index_right_or_value] = 1;
break;
}
std::stable_sort(tree.begin(), tree.end(), Compare);
// The nodes are:
// [0, n): the sorted leaf nodes that we start with.
// [n]: we add a sentinel here.
// [n + 1, 2n): new parent nodes are added here, starting from
// (n+1). These are naturally in ascending order.
// [2n]: we add a sentinel at the end as well.
// There will be (2n+1) elements at the end.
const HuffmanTree sentinel(std::numeric_limits<uint32_t>::max(), -1, -1);
tree.push_back(sentinel);
tree.push_back(sentinel);
size_t i = 0; // Points to the next leaf node.
size_t j = n + 1; // Points to the next non-leaf node.
for (size_t k = n - 1; k != 0; --k) {
size_t left;
size_t right;
if (tree[i].total_count <= tree[j].total_count) {
left = i;
++i;
} else {
left = j;
++j;
}
if (tree[i].total_count <= tree[j].total_count) {
right = i;
++i;
} else {
right = j;
++j;
}
// The sentinel node becomes the parent node.
size_t j_end = tree.size() - 1;
tree[j_end].total_count =
tree[left].total_count + tree[right].total_count;
tree[j_end].index_left = static_cast<int16_t>(left);
tree[j_end].index_right_or_value = static_cast<int16_t>(right);
// Add back the last sentinel node.
tree.push_back(sentinel);
}
JXL_DASSERT(tree.size() == 2 * n + 1);
SetDepth(tree[2 * n - 1], tree.data(), depth, 0);
// We need to pack the Huffman tree in tree_limit bits.
// If this was not successful, add fake entities to the lowest values
// and retry.
if (*std::max_element(&depth[0], &depth[length]) <= tree_limit) {
break;
}
}
}
void ValidateHuffmanTable(j_common_ptr cinfo, const JHUFF_TBL* table,
bool is_dc) {
size_t total_symbols = 0;
size_t total_p = 0;
size_t max_depth = 0;
for (size_t d = 1; d <= kJpegHuffmanMaxBitLength; ++d) {
uint8_t count = table->bits[d];
if (count) {
total_symbols += count;
total_p += (1u << (kJpegHuffmanMaxBitLength - d)) * count;
max_depth = d;
}
}
total_p += 1u << (kJpegHuffmanMaxBitLength - max_depth); // sentinel symbol
if (total_symbols == 0) {
JPEGLI_ERROR("Empty Huffman table");
}
if (total_symbols > kJpegHuffmanAlphabetSize) {
JPEGLI_ERROR("Too many symbols in Huffman table");
}
if (total_p != (1u << kJpegHuffmanMaxBitLength)) {
JPEGLI_ERROR("Invalid bit length distribution");
}
uint8_t symbol_seen[kJpegHuffmanAlphabetSize] = {};
for (size_t i = 0; i < total_symbols; ++i) {
uint8_t symbol = table->huffval[i];
if (symbol_seen[symbol]) {
JPEGLI_ERROR("Duplicate symbol %d in Huffman table", symbol);
}
symbol_seen[symbol] = 1;
}
}
void AddStandardHuffmanTables(j_common_ptr cinfo, bool is_dc) {
// Huffman tables from the JPEG standard.
static constexpr JHUFF_TBL kStandardDCTables[2] = {
// DC luma
{{0, 0, 1, 5, 1, 1, 1, 1, 1, 1},
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
FALSE},
// DC chroma
{{0, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1},
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
FALSE}};
static constexpr JHUFF_TBL kStandardACTables[2] = {
// AC luma
{{0, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 125},
{0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06,
0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08,
0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72,
0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28,
0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45,
0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59,
0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75,
0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89,
0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3,
0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6,
0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9,
0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2,
0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4,
0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa},
FALSE},
// AC chroma
{{0, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 119},
{0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41,
0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62, 0x72, 0xd1,
0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26,
0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44,
0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74,
0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a,
0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4,
0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda,
0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4,
0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa},
FALSE}};
const JHUFF_TBL* std_tables = is_dc ? kStandardDCTables : kStandardACTables;
JHUFF_TBL** tables;
if (cinfo->is_decompressor) {
j_decompress_ptr cinfo_d = reinterpret_cast<j_decompress_ptr>(cinfo);
tables = is_dc ? cinfo_d->dc_huff_tbl_ptrs : cinfo_d->ac_huff_tbl_ptrs;
} else {
j_compress_ptr cinfo_c = reinterpret_cast<j_compress_ptr>(cinfo);
tables = is_dc ? cinfo_c->dc_huff_tbl_ptrs : cinfo_c->ac_huff_tbl_ptrs;
}
for (int i = 0; i < 2; ++i) {
if (tables[i] == nullptr) {
tables[i] = jpegli_alloc_huff_table(cinfo);
memcpy(tables[i], &std_tables[i], sizeof(JHUFF_TBL));
ValidateHuffmanTable(cinfo, tables[i], is_dc);
}
}
}
} // namespace jpegli