| // Copyright 2019+ Klaus Post. All rights reserved. |
| // License information can be found in the LICENSE file. |
| // Based on work by Yann Collet, released under BSD License. |
| |
| package zstd |
| |
| import ( |
| "errors" |
| "fmt" |
| "math/bits" |
| "runtime" |
| ) |
| |
| // DOption is an option for creating a decoder. |
| type DOption func(*decoderOptions) error |
| |
| // options retains accumulated state of multiple options. |
| type decoderOptions struct { |
| lowMem bool |
| concurrent int |
| maxDecodedSize uint64 |
| maxWindowSize uint64 |
| dicts []*dict |
| ignoreChecksum bool |
| limitToCap bool |
| decodeBufsBelow int |
| } |
| |
| func (o *decoderOptions) setDefault() { |
| *o = decoderOptions{ |
| // use less ram: true for now, but may change. |
| lowMem: true, |
| concurrent: runtime.GOMAXPROCS(0), |
| maxWindowSize: MaxWindowSize, |
| decodeBufsBelow: 128 << 10, |
| } |
| if o.concurrent > 4 { |
| o.concurrent = 4 |
| } |
| o.maxDecodedSize = 64 << 30 |
| } |
| |
| // WithDecoderLowmem will set whether to use a lower amount of memory, |
| // but possibly have to allocate more while running. |
| func WithDecoderLowmem(b bool) DOption { |
| return func(o *decoderOptions) error { o.lowMem = b; return nil } |
| } |
| |
| // WithDecoderConcurrency sets the number of created decoders. |
| // When decoding block with DecodeAll, this will limit the number |
| // of possible concurrently running decodes. |
| // When decoding streams, this will limit the number of |
| // inflight blocks. |
| // When decoding streams and setting maximum to 1, |
| // no async decoding will be done. |
| // When a value of 0 is provided GOMAXPROCS will be used. |
| // By default this will be set to 4 or GOMAXPROCS, whatever is lower. |
| func WithDecoderConcurrency(n int) DOption { |
| return func(o *decoderOptions) error { |
| if n < 0 { |
| return errors.New("concurrency must be at least 1") |
| } |
| if n == 0 { |
| o.concurrent = runtime.GOMAXPROCS(0) |
| } else { |
| o.concurrent = n |
| } |
| return nil |
| } |
| } |
| |
| // WithDecoderMaxMemory allows to set a maximum decoded size for in-memory |
| // non-streaming operations or maximum window size for streaming operations. |
| // This can be used to control memory usage of potentially hostile content. |
| // Maximum is 1 << 63 bytes. Default is 64GiB. |
| func WithDecoderMaxMemory(n uint64) DOption { |
| return func(o *decoderOptions) error { |
| if n == 0 { |
| return errors.New("WithDecoderMaxMemory must be at least 1") |
| } |
| if n > 1<<63 { |
| return errors.New("WithDecoderMaxmemory must be less than 1 << 63") |
| } |
| o.maxDecodedSize = n |
| return nil |
| } |
| } |
| |
| // WithDecoderDicts allows to register one or more dictionaries for the decoder. |
| // |
| // Each slice in dict must be in the [dictionary format] produced by |
| // "zstd --train" from the Zstandard reference implementation. |
| // |
| // If several dictionaries with the same ID are provided, the last one will be used. |
| // |
| // [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format |
| func WithDecoderDicts(dicts ...[]byte) DOption { |
| return func(o *decoderOptions) error { |
| for _, b := range dicts { |
| d, err := loadDict(b) |
| if err != nil { |
| return err |
| } |
| o.dicts = append(o.dicts, d) |
| } |
| return nil |
| } |
| } |
| |
| // WithDecoderDictRaw registers a dictionary that may be used by the decoder. |
| // The slice content can be arbitrary data. |
| func WithDecoderDictRaw(id uint32, content []byte) DOption { |
| return func(o *decoderOptions) error { |
| if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { |
| return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) |
| } |
| o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}) |
| return nil |
| } |
| } |
| |
| // WithDecoderMaxWindow allows to set a maximum window size for decodes. |
| // This allows rejecting packets that will cause big memory usage. |
| // The Decoder will likely allocate more memory based on the WithDecoderLowmem setting. |
| // If WithDecoderMaxMemory is set to a lower value, that will be used. |
| // Default is 512MB, Maximum is ~3.75 TB as per zstandard spec. |
| func WithDecoderMaxWindow(size uint64) DOption { |
| return func(o *decoderOptions) error { |
| if size < MinWindowSize { |
| return errors.New("WithMaxWindowSize must be at least 1KB, 1024 bytes") |
| } |
| if size > (1<<41)+7*(1<<38) { |
| return errors.New("WithMaxWindowSize must be less than (1<<41) + 7*(1<<38) ~ 3.75TB") |
| } |
| o.maxWindowSize = size |
| return nil |
| } |
| } |
| |
| // WithDecodeAllCapLimit will limit DecodeAll to decoding cap(dst)-len(dst) bytes, |
| // or any size set in WithDecoderMaxMemory. |
| // This can be used to limit decoding to a specific maximum output size. |
| // Disabled by default. |
| func WithDecodeAllCapLimit(b bool) DOption { |
| return func(o *decoderOptions) error { |
| o.limitToCap = b |
| return nil |
| } |
| } |
| |
| // WithDecodeBuffersBelow will fully decode readers that have a |
| // `Bytes() []byte` and `Len() int` interface similar to bytes.Buffer. |
| // This typically uses less allocations but will have the full decompressed object in memory. |
| // Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less. |
| // Default is 128KiB. |
| func WithDecodeBuffersBelow(size int) DOption { |
| return func(o *decoderOptions) error { |
| o.decodeBufsBelow = size |
| return nil |
| } |
| } |
| |
| // IgnoreChecksum allows to forcibly ignore checksum checking. |
| func IgnoreChecksum(b bool) DOption { |
| return func(o *decoderOptions) error { |
| o.ignoreChecksum = b |
| return nil |
| } |
| } |