| /* inftrees.c -- generate Huffman trees for efficient decoding |
| * Copyright (C) 1995-2024 Mark Adler |
| * For conditions of distribution and use, see copyright notice in zlib.h |
| */ |
| |
| #include "zbuild.h" |
| #include "zutil.h" |
| #include "inftrees.h" |
| #include "inflate_p.h" |
| #include "fallback_builtins.h" |
| |
| #if defined(__SSE2__) |
| # include "arch/x86/x86_intrins.h" |
| #elif defined(__ARM_NEON) || defined(__ARM_NEON__) |
| # include "arch/arm/neon_intrins.h" |
| #elif defined(__ALTIVEC__) |
| # include "arch/power/power_intrins.h" |
| #endif |
| |
| const char PREFIX(inflate_copyright)[] = " inflate 1.3.1 Copyright 1995-2024 Mark Adler "; |
| /* |
| If you use the zlib library in a product, an acknowledgment is welcome |
| in the documentation of your product. If for some reason you cannot |
| include such an acknowledgment, I would appreciate that you keep this |
| copyright string in the executable of your product. |
| */ |
| |
| /* Count number of codes for each code length. */ |
| static inline void count_lengths(uint16_t *lens, int codes, uint16_t *count) { |
| /* IBM...made some weird choices for VSX/VMX. Basically vec_ld has an inherent |
| * endianness but we don't want to force VSX to be needed */ |
| static const ALIGNED_(16) uint8_t one[256] = { |
| 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, |
| 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, |
| 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, |
| 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, |
| 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, |
| 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, |
| 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, |
| 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, |
| 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 |
| }; |
| |
| #if defined(__ALTIVEC__) |
| vector unsigned char s1 = vec_splat_u8(0); |
| vector unsigned char s2 = vec_splat_u8(0); |
| |
| if (codes & 1) { |
| s1 = vec_ld(16 * lens[0], one); |
| --codes; |
| ++lens; |
| } |
| |
| while (codes) { |
| s1 = vec_add(s1, vec_ld(16 * lens[0], one)); |
| s2 = vec_add(s2, vec_ld(16 * lens[1], one)); |
| codes -= 2; |
| lens += 2; |
| } |
| |
| vector unsigned short sum_lo = vec_add(vec_unpackh(s1), vec_unpackh(s2)); |
| vector unsigned short sum_hi = vec_add(vec_unpackl(s1), vec_unpackl(s2)); |
| |
| vec_st(sum_lo, 0, &count[0]); |
| vec_st(sum_hi, 0, &count[8]); |
| |
| #elif defined(__ARM_NEON) || defined(__ARM_NEON__) |
| int sym; |
| uint8x16_t s1 = vdupq_n_u8(0); |
| uint8x16_t s2 = vdupq_n_u8(0); |
| |
| if (codes & 1) { |
| s1 = vld1q_u8(&one[16 * lens[0]]); |
| } |
| for (sym = codes & 1; sym < codes; sym += 2) { |
| s1 = vaddq_u8(s1, vld1q_u8(&one[16 * lens[sym]])); |
| s2 = vaddq_u8(s2, vld1q_u8(&one[16 * lens[sym+1]])); |
| } |
| |
| vst1q_u16(&count[0], vaddl_u8(vget_low_u8(s1), vget_low_u8(s2))); |
| vst1q_u16(&count[8], vaddl_u8(vget_high_u8(s1), vget_high_u8(s2))); |
| |
| #elif defined(__SSE2__) |
| int sym; |
| __m128i s1 = _mm_setzero_si128(); |
| __m128i s2 = _mm_setzero_si128(); |
| |
| if (codes & 1) { |
| s1 = _mm_load_si128((const __m128i*)&one[16 * lens[0]]); |
| } |
| for (sym = codes & 1; sym < codes; sym += 2) { |
| s1 = _mm_add_epi8(s1, _mm_load_si128((const __m128i*)&one[16 * lens[sym]])); // vaddq_u8 |
| s2 = _mm_add_epi8(s2, _mm_load_si128((const __m128i*)&one[16 * lens[sym+1]])); |
| } |
| |
| # if defined(__AVX2__) |
| __m256i w1 = _mm256_cvtepu8_epi16(s1); |
| __m256i w2 = _mm256_cvtepu8_epi16(s2); |
| __m256i sum = _mm256_add_epi16(w1, w2); |
| |
| _mm256_storeu_si256((__m256i*)&count[0], sum); |
| # else |
| __m128i zero = _mm_setzero_si128(); |
| |
| __m128i s1_lo = _mm_unpacklo_epi8(s1, zero); |
| __m128i s2_lo = _mm_unpacklo_epi8(s2, zero); |
| __m128i sum_lo = _mm_add_epi16(s1_lo, s2_lo); |
| _mm_storeu_si128((__m128i*)&count[0], sum_lo); |
| |
| __m128i s1_hi = _mm_unpackhi_epi8(s1, zero); |
| __m128i s2_hi = _mm_unpackhi_epi8(s2, zero); |
| __m128i sum_hi = _mm_add_epi16(s1_hi, s2_hi); |
| _mm_storeu_si128((__m128i*)&count[8], sum_hi); |
| # endif |
| #else |
| int len, sym; |
| for (len = 0; len <= MAX_BITS; len++) |
| count[len] = 0; |
| for (sym = 0; sym < codes; sym++) |
| count[lens[sym]]++; |
| Z_UNUSED(one); |
| #endif |
| } |
| |
| /* |
| Build a set of tables to decode the provided canonical Huffman code. |
| The code lengths are lens[0..codes-1]. The result starts at *table, |
| whose indices are 0..2^bits-1. work is a writable array of at least |
| lens shorts, which is used as a work area. type is the type of code |
| to be generated, CODES, LENS, or DISTS. On return, zero is success, |
| -1 is an invalid code, and +1 means that ENOUGH isn't enough. table |
| on return points to the next available entry's address. bits is the |
| requested root table index bits, and on return it is the actual root |
| table index bits. It will differ if the request is greater than the |
| longest code or if it is less than the shortest code. |
| */ |
| int Z_INTERNAL zng_inflate_table(codetype type, uint16_t *lens, unsigned codes, |
| code * *table, unsigned *bits, uint16_t *work) { |
| unsigned len; /* a code's length in bits */ |
| unsigned sym; /* index of code symbols */ |
| unsigned min, max; /* minimum and maximum code lengths */ |
| unsigned root; /* number of index bits for root table */ |
| unsigned curr; /* number of index bits for current table */ |
| unsigned drop; /* code bits to drop for sub-table */ |
| int left; /* number of prefix codes available */ |
| unsigned used; /* code entries in table used */ |
| uint16_t rhuff; /* Reversed huffman code */ |
| unsigned huff; /* Huffman code */ |
| unsigned incr; /* for incrementing code, index */ |
| unsigned fill; /* index for replicating entries */ |
| unsigned low; /* low bits for current root entry */ |
| unsigned mask; /* mask for low root bits */ |
| code here; /* table entry for duplication */ |
| code *next; /* next available space in table */ |
| const uint16_t *base; /* base value table to use */ |
| const uint16_t *extra; /* extra bits table to use */ |
| unsigned match; /* use base and extra for symbol >= match */ |
| uint16_t ALIGNED_(16) count[MAX_BITS+1]; /* number of codes of each length */ |
| uint16_t offs[MAX_BITS+1]; /* offsets in table for each length */ |
| static const uint16_t lbase[31] = { /* Length codes 257..285 base */ |
| 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, |
| 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; |
| static const uint16_t lext[31] = { /* Length codes 257..285 extra */ |
| 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, |
| 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 203, 77}; |
| static const uint16_t dbase[32] = { /* Distance codes 0..29 base */ |
| 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, |
| 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, |
| 8193, 12289, 16385, 24577, 0, 0}; |
| static const uint16_t dext[32] = { /* Distance codes 0..29 extra */ |
| 16, 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, |
| 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, |
| 28, 28, 29, 29, 64, 64}; |
| |
| /* |
| Process a set of code lengths to create a canonical Huffman code. The |
| code lengths are lens[0..codes-1]. Each length corresponds to the |
| symbols 0..codes-1. The Huffman code is generated by first sorting the |
| symbols by length from short to long, and retaining the symbol order |
| for codes with equal lengths. Then the code starts with all zero bits |
| for the first code of the shortest length, and the codes are integer |
| increments for the same length, and zeros are appended as the length |
| increases. For the deflate format, these bits are stored backwards |
| from their more natural integer increment ordering, and so when the |
| decoding tables are built in the large loop below, the integer codes |
| are incremented backwards. |
| |
| This routine assumes, but does not check, that all of the entries in |
| lens[] are in the range 0..MAXBITS. The caller must assure this. |
| 1..MAXBITS is interpreted as that code length. zero means that that |
| symbol does not occur in this code. |
| |
| The codes are sorted by computing a count of codes for each length, |
| creating from that a table of starting indices for each length in the |
| sorted table, and then entering the symbols in order in the sorted |
| table. The sorted table is work[], with that space being provided by |
| the caller. |
| |
| The length counts are used for other purposes as well, i.e. finding |
| the minimum and maximum length codes, determining if there are any |
| codes at all, checking for a valid set of lengths, and looking ahead |
| at length counts to determine sub-table sizes when building the |
| decoding tables. |
| */ |
| |
| /* accumulate lengths for codes (assumes lens[] all in 0..MAXBITS) */ |
| count_lengths(lens, codes, count); |
| |
| /* bound code lengths, force root to be within code lengths */ |
| root = *bits; |
| for (max = MAX_BITS; max >= 1; max--) |
| if (count[max] != 0) break; |
| root = MIN(root, max); |
| if (UNLIKELY(max == 0)) { /* no symbols to code at all */ |
| here.op = (unsigned char)64; /* invalid code marker */ |
| here.bits = (unsigned char)1; |
| here.val = (uint16_t)0; |
| *(*table)++ = here; /* make a table to force an error */ |
| *(*table)++ = here; |
| *bits = 1; |
| return 0; /* no symbols, but wait for decoding to report error */ |
| } |
| for (min = 1; min < max; min++) |
| if (count[min] != 0) break; |
| root = MAX(root, min); |
| |
| /* check for an over-subscribed or incomplete set of lengths */ |
| left = 1; |
| for (len = 1; len <= MAX_BITS; len++) { |
| left <<= 1; |
| left -= count[len]; |
| if (left < 0) return -1; /* over-subscribed */ |
| } |
| if (left > 0 && (type == CODES || max != 1)) |
| return -1; /* incomplete set */ |
| |
| /* generate offsets into symbol table for each length for sorting */ |
| offs[1] = 0; |
| for (len = 1; len < MAX_BITS; len++) |
| offs[len + 1] = offs[len] + count[len]; |
| |
| /* sort symbols by length, by symbol order within each length */ |
| for (sym = 0; sym < codes; sym++) |
| if (lens[sym] != 0) work[offs[lens[sym]]++] = (uint16_t)sym; |
| |
| /* |
| Create and fill in decoding tables. In this loop, the table being |
| filled is at next and has curr index bits. The code being used is huff |
| with length len. That code is converted to an index by dropping drop |
| bits off of the bottom. For codes where len is less than drop + curr, |
| those top drop + curr - len bits are incremented through all values to |
| fill the table with replicated entries. |
| |
| root is the number of index bits for the root table. When len exceeds |
| root, sub-tables are created pointed to by the root entry with an index |
| of the low root bits of huff. This is saved in low to check for when a |
| new sub-table should be started. drop is zero when the root table is |
| being filled, and drop is root when sub-tables are being filled. |
| |
| When a new sub-table is needed, it is necessary to look ahead in the |
| code lengths to determine what size sub-table is needed. The length |
| counts are used for this, and so count[] is decremented as codes are |
| entered in the tables. |
| |
| used keeps track of how many table entries have been allocated from the |
| provided *table space. It is checked for LENS and DIST tables against |
| the constants ENOUGH_LENS and ENOUGH_DISTS to guard against changes in |
| the initial root table size constants. See the comments in inftrees.h |
| for more information. |
| |
| sym increments through all symbols, and the loop terminates when |
| all codes of length max, i.e. all codes, have been processed. This |
| routine permits incomplete codes, so another loop after this one fills |
| in the rest of the decoding tables with invalid code markers. |
| */ |
| |
| /* set up for code type */ |
| switch (type) { |
| case CODES: |
| base = extra = work; /* dummy value--not used */ |
| match = 20; |
| break; |
| case LENS: |
| base = lbase; |
| extra = lext; |
| match = 257; |
| break; |
| default: /* DISTS */ |
| base = dbase; |
| extra = dext; |
| match = 0; |
| } |
| |
| /* initialize state for loop */ |
| rhuff = 0; /* starting code, reversed */ |
| huff = 0; /* starting code */ |
| sym = 0; /* starting code symbol */ |
| len = min; /* starting code length */ |
| next = *table; /* current table to fill in */ |
| curr = root; /* current table index bits */ |
| drop = 0; /* current bits to drop from code for index */ |
| low = (unsigned)(-1); /* trigger new sub-table when len > root */ |
| used = 1U << root; /* use root table entries */ |
| mask = used - 1; /* mask for comparing low */ |
| |
| /* check available table space */ |
| if ((type == LENS && used > ENOUGH_LENS) || |
| (type == DISTS && used > ENOUGH_DISTS)) |
| return 1; |
| |
| /* process all codes and make table entries */ |
| for (;;) { |
| /* create table entry */ |
| here.bits = (unsigned char)(len - drop); |
| if (LIKELY(work[sym] >= match)) { |
| unsigned op = extra[work[sym] - match]; |
| here.op = COMBINE_OP(op, here.bits); |
| here.bits = COMBINE_BITS(here.bits, op); |
| here.val = base[work[sym] - match]; |
| } else if (work[sym] + 1U < match) { |
| here.op = (unsigned char)0; |
| here.val = work[sym]; |
| } else { |
| here.op = (unsigned char)(32 + 64); /* end of block */ |
| here.val = 0; |
| } |
| |
| /* replicate for those indices with low len bits equal to huff */ |
| incr = 1U << (len - drop); |
| fill = 1U << curr; |
| min = fill; /* save offset to next table */ |
| do { |
| fill -= incr; |
| next[(huff >> drop) + fill] = here; |
| } while (fill != 0); |
| |
| /* backwards increment the len-bit code huff */ |
| rhuff = (uint16_t)(rhuff + (0x8000u >> (len - 1))); |
| huff = zng_bitreverse16(rhuff); |
| |
| /* go to next symbol, update count, len */ |
| sym++; |
| if (--(count[len]) == 0) { |
| if (len == max) |
| break; |
| len = lens[work[sym]]; |
| } |
| |
| /* create new sub-table if needed */ |
| if (len > root && (huff & mask) != low) { |
| /* if first time, transition to sub-tables */ |
| if (drop == 0) |
| drop = root; |
| |
| /* increment past last table */ |
| next += min; /* here min is 1 << curr */ |
| |
| /* determine length of next table */ |
| curr = len - drop; |
| left = (int)(1 << curr); |
| while (curr + drop < max) { |
| left -= count[curr + drop]; |
| if (left <= 0) |
| break; |
| curr++; |
| left <<= 1; |
| } |
| |
| /* check for enough space */ |
| used += 1U << curr; |
| if ((type == LENS && used > ENOUGH_LENS) || (type == DISTS && used > ENOUGH_DISTS)) |
| return 1; |
| |
| /* point entry in root table to sub-table */ |
| low = huff & mask; |
| (*table)[low].op = (unsigned char)curr; |
| (*table)[low].bits = (unsigned char)root; |
| (*table)[low].val = (uint16_t)(next - *table); |
| } |
| } |
| |
| /* fill in remaining table entry if code is incomplete (guaranteed to have |
| at most one remaining entry, since if the code is incomplete, the |
| maximum code length that was allowed to get this far is one bit) */ |
| if (UNLIKELY(huff != 0)) { |
| here.op = (unsigned char)64; /* invalid code marker */ |
| here.bits = (unsigned char)(len - drop); |
| here.val = (uint16_t)0; |
| next[huff] = here; |
| } |
| |
| /* set return parameters */ |
| *table += used; |
| *bits = root; |
| return 0; |
| } |
