/* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ /* A (forgetful) hash table to the data seen by the compressor, to help create backward references to previous data. */ #ifndef BROTLI_ENC_HASH_H_ #define BROTLI_ENC_HASH_H_ #include <stdlib.h> /* exit */ #include <string.h> /* memcmp, memset */ #include "../common/constants.h" #include "../common/dictionary.h" #include "../common/platform.h" #include <brotli/types.h> #include "encoder_dict.h" #include "fast_log.h" #include "find_match_length.h" #include "memory.h" #include "quality.h" #include "static_dict.h" #if defined(__cplusplus) || defined(c_plusplus) extern "C" { #endif HasherCommon; #define score_t … static const uint32_t kCutoffTransformsCount = …; /* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */ /* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */ static const uint64_t kCutoffTransforms = …; HasherSearchResult; /* kHashMul32 multiplier has these properties: * The multiplier must be odd. Otherwise we may lose the highest bit. * No long streaks of ones or zeros. * There is no effort to ensure that it is a prime, the oddity is enough for this use. * The number has been tuned heuristically against compression benchmarks. */ static const uint32_t kHashMul32 = …; static const uint64_t kHashMul64 = …; static const uint64_t kHashMul64Long = …; static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) { … } static BROTLI_INLINE void PrepareDistanceCache( int* BROTLI_RESTRICT distance_cache, const int num_distances) { … } #define BROTLI_LITERAL_BYTE_SCORE … #define BROTLI_DISTANCE_BIT_PENALTY … /* Score must be positive after applying maximal penalty. */ #define BROTLI_SCORE_BASE … /* Usually, we always choose the longest backward reference. This function allows for the exception of that rule. If we choose a backward reference that is further away, it will usually be coded with more bits. We approximate this by assuming log2(distance). If the distance can be expressed in terms of the last four distances, we use some heuristic constants to estimate the bits cost. For the first up to four literals we use the bit cost of the literals from the literal cost model, after that we use the average bit cost of the cost model. This function is used to sometimes discard a longer backward reference when it is not much longer and the bit cost for encoding it is more than the saved literals. backward_reference_offset MUST be positive. */ static BROTLI_INLINE score_t BackwardReferenceScore( size_t copy_length, size_t backward_reference_offset) { … } static BROTLI_INLINE score_t BackwardReferenceScoreUsingLastDistance( size_t copy_length) { … } static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance( size_t distance_short_code) { … } static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem( const BrotliEncoderDictionary* dictionary, size_t len, size_t word_idx, const uint8_t* data, size_t max_length, size_t max_backward, size_t max_distance, HasherSearchResult* out) { … } static BROTLI_INLINE void SearchInStaticDictionary( const BrotliEncoderDictionary* dictionary, HasherCommon* common, const uint8_t* data, size_t max_length, size_t max_backward, size_t max_distance, HasherSearchResult* out, BROTLI_BOOL shallow) { … } BackwardMatch; static BROTLI_INLINE void InitBackwardMatch(BackwardMatch* self, size_t dist, size_t len) { … } static BROTLI_INLINE void InitDictionaryBackwardMatch(BackwardMatch* self, size_t dist, size_t len, size_t len_code) { … } static BROTLI_INLINE size_t BackwardMatchLength(const BackwardMatch* self) { … } static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) { … } #define EXPAND_CAT … #define CAT … #define FN … #define HASHER … #define BUCKET_BITS … #define MAX_TREE_SEARCH_DEPTH … #define MAX_TREE_COMP_LENGTH … #include "hash_to_binary_tree_inc.h" /* NOLINT(build/include) */ #undef MAX_TREE_SEARCH_DEPTH #undef MAX_TREE_COMP_LENGTH #undef BUCKET_BITS #undef HASHER /* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */ #define MAX_NUM_MATCHES_H10 … /* For BUCKET_SWEEP_BITS == 0, enabling the dictionary lookup makes compression a little faster (0.5% - 1%) and it compresses 0.15% better on small text and HTML inputs. */ #define HASHER … #define BUCKET_BITS … #define BUCKET_SWEEP_BITS … #define HASH_LEN … #define USE_DICTIONARY … #include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */ #undef BUCKET_SWEEP_BITS #undef USE_DICTIONARY #undef HASHER #define HASHER … #define BUCKET_SWEEP_BITS … #define USE_DICTIONARY … #include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */ #undef USE_DICTIONARY #undef BUCKET_SWEEP_BITS #undef BUCKET_BITS #undef HASHER #define HASHER … #define BUCKET_BITS … #define BUCKET_SWEEP_BITS … #define USE_DICTIONARY … #include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */ #undef USE_DICTIONARY #undef HASH_LEN #undef BUCKET_SWEEP_BITS #undef BUCKET_BITS #undef HASHER #define HASHER … #include "hash_longest_match_inc.h" /* NOLINT(build/include) */ #undef HASHER #define HASHER … #include "hash_longest_match64_inc.h" /* NOLINT(build/include) */ #undef HASHER #define BUCKET_BITS … #define NUM_LAST_DISTANCES_TO_CHECK … #define NUM_BANKS … #define BANK_BITS … #define HASHER … #include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */ #undef HASHER #undef NUM_LAST_DISTANCES_TO_CHECK #define NUM_LAST_DISTANCES_TO_CHECK … #define HASHER … #include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */ #undef HASHER #undef NUM_LAST_DISTANCES_TO_CHECK #undef NUM_BANKS #undef BANK_BITS #define NUM_LAST_DISTANCES_TO_CHECK … #define NUM_BANKS … #define BANK_BITS … #define HASHER … #include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */ #undef HASHER #undef NUM_LAST_DISTANCES_TO_CHECK #undef NUM_BANKS #undef BANK_BITS #undef BUCKET_BITS #define HASHER … #define BUCKET_BITS … #define BUCKET_SWEEP_BITS … #define HASH_LEN … #define USE_DICTIONARY … #include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */ #undef USE_DICTIONARY #undef HASH_LEN #undef BUCKET_SWEEP_BITS #undef BUCKET_BITS #undef HASHER /* fast large window hashers */ #define HASHER … #define CHUNKLEN … #define JUMP … #define NUMBUCKETS … #define MASK … #include "hash_rolling_inc.h" /* NOLINT(build/include) */ #undef JUMP #undef HASHER #define HASHER … #define JUMP … #include "hash_rolling_inc.h" /* NOLINT(build/include) */ #undef MASK #undef NUMBUCKETS #undef JUMP #undef CHUNKLEN #undef HASHER #define HASHER … #define HASHER_A … #define HASHER_B … #include "hash_composite_inc.h" /* NOLINT(build/include) */ #undef HASHER_A #undef HASHER_B #undef HASHER #define HASHER … #define HASHER_A … #define HASHER_B … #include "hash_composite_inc.h" /* NOLINT(build/include) */ #undef HASHER_A #undef HASHER_B #undef HASHER #define HASHER … #define HASHER_A … #define HASHER_B … #include "hash_composite_inc.h" /* NOLINT(build/include) */ #undef HASHER_A #undef HASHER_B #undef HASHER #undef FN #undef CAT #undef EXPAND_CAT #define FOR_SIMPLE_HASHERS(H) … #define FOR_COMPOSITE_HASHERS(H) … #define FOR_GENERIC_HASHERS(H) … #define FOR_ALL_HASHERS(H) … Hasher; /* MUST be invoked before any other method. */ static BROTLI_INLINE void HasherInit(Hasher* hasher) { … } static BROTLI_INLINE void DestroyHasher(MemoryManager* m, Hasher* hasher) { … } static BROTLI_INLINE void HasherReset(Hasher* hasher) { … } static BROTLI_INLINE void HasherSize(const BrotliEncoderParams* params, BROTLI_BOOL one_shot, const size_t input_size, size_t* alloc_size) { … } static BROTLI_INLINE void HasherSetup(MemoryManager* m, Hasher* hasher, BrotliEncoderParams* params, const uint8_t* data, size_t position, size_t input_size, BROTLI_BOOL is_last) { … } static BROTLI_INLINE void InitOrStitchToPreviousBlock( MemoryManager* m, Hasher* hasher, const uint8_t* data, size_t mask, BrotliEncoderParams* params, size_t position, size_t input_size, BROTLI_BOOL is_last) { … } /* NB: when seamless dictionary-ring-buffer copies are implemented, don't forget to add proper guards for non-zero-BROTLI_PARAM_STREAM_OFFSET. */ static BROTLI_INLINE void FindCompoundDictionaryMatch( const PreparedDictionary* self, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, const size_t max_length, const size_t distance_offset, const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) { … } /* NB: when seamless dictionary-ring-buffer copies are implemented, don't forget to add proper guards for non-zero-BROTLI_PARAM_STREAM_OFFSET. */ static BROTLI_INLINE size_t FindAllCompoundDictionaryMatches( const PreparedDictionary* self, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const size_t cur_ix, const size_t min_length, const size_t max_length, const size_t distance_offset, const size_t max_distance, BackwardMatch* matches, size_t match_limit) { … } static BROTLI_INLINE void LookupCompoundDictionaryMatch( const CompoundDictionary* addon, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, const size_t max_length, const size_t max_ring_buffer_distance, const size_t max_distance, HasherSearchResult* sr) { … } static BROTLI_INLINE size_t LookupAllCompoundDictionaryMatches( const CompoundDictionary* addon, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const size_t cur_ix, size_t min_length, const size_t max_length, const size_t max_ring_buffer_distance, const size_t max_distance, BackwardMatch* matches, size_t match_limit) { … } #if defined(__cplusplus) || defined(c_plusplus) } /* extern "C" */ #endif #endif /* BROTLI_ENC_HASH_H_ */
Codebase Browser
chromium