zzg
6cfb5b0319
A shard file is a sequence of length-prefixed CRC'd chunks, each carrying its own dictionary and complete column data. Nothing already written is ever revised, so appending is a write at the end of the file and a torn tail costs exactly the chunk that was in flight. Large text goes to a per-shard .blob sidecar so scans touch timestamps and dictionary codes rather than megabytes of prose. Tests cover round-trip, encoding determinism under Go's randomised map iteration, truncation at every byte offset, random bit flips, torn-tail and bad-CRC recovery, midnight-straddling flushes, and both retention policies.
63 lines
1.2 KiB
Go
63 lines
1.2 KiB
Go
package store
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
)
|
|
|
|
var errTruncated = errors.New("truncated chunk data")
|
|
|
|
// reader is a bounds-checked cursor over a decoded chunk body.
|
|
//
|
|
// Every read is length-checked because chunk bytes come off disk and may be
|
|
// damaged. The decoder's job is to fail cleanly on garbage, never to panic: a
|
|
// query touching a hundred shards should lose the one bad chunk, not the
|
|
// process.
|
|
type reader struct {
|
|
b []byte
|
|
pos int
|
|
}
|
|
|
|
func (r *reader) fixed(n int) ([]byte, error) {
|
|
if n < 0 || r.pos+n > len(r.b) {
|
|
return nil, errTruncated
|
|
}
|
|
b := r.b[r.pos : r.pos+n]
|
|
r.pos += n
|
|
return b, nil
|
|
}
|
|
|
|
func (r *reader) byte() (byte, error) {
|
|
if r.pos >= len(r.b) {
|
|
return 0, errTruncated
|
|
}
|
|
c := r.b[r.pos]
|
|
r.pos++
|
|
return c, nil
|
|
}
|
|
|
|
func (r *reader) uvarint() (uint64, error) {
|
|
v, n := binary.Uvarint(r.b[r.pos:])
|
|
if n <= 0 {
|
|
return 0, errTruncated
|
|
}
|
|
r.pos += n
|
|
return v, nil
|
|
}
|
|
|
|
func (r *reader) str() (string, error) {
|
|
n, err := r.uvarint()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if n > uint64(len(r.b)-r.pos) {
|
|
return "", fmt.Errorf("%w: string claims %d bytes, %d remain", errTruncated, n, len(r.b)-r.pos)
|
|
}
|
|
b, err := r.fixed(int(n))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(b), nil
|
|
}
|