1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
// Copyright (c) 2023+ Klaus Post. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package s2_test
import (
"bytes"
"fmt"
"os"
"github.com/klauspost/compress/s2"
"github.com/klauspost/compress/zstd"
)
func ExampleMakeDict() {
// Read a sample
sample, err := os.ReadFile("../testdata/gettysburg.txt")
if err != nil {
panic(err)
}
fmt.Println("Input size:", len(sample))
// Create a dictionary.
dict := s2.MakeDict(sample, nil)
fmt.Println("Dict size:", len(dict.Bytes()))
encoded := dict.Encode(nil, sample)
if len(encoded) < 20 {
fmt.Println("Encoded size was less than 20 bytes!")
}
// To decode:
decoded, err := dict.Decode(nil, encoded)
if err != nil {
panic(err)
}
if bytes.Equal(decoded, sample) {
fmt.Println("They match!")
}
// OUTPUT:
// Input size: 1548
// Dict size: 1549
// Encoded size was less than 20 bytes!
// They match!
}
func ExampleMakeDict_zstd() {
// Read dictionary generated by zStandard using the command line
// λ zstd -r --train-fastcover -o zstd.dict --maxdict=2048 gosrc\*
// With gosrc containing all the standard library source files.
zdict := []byte("7\xa40콶\xc1\x1bB\x10\x982\xc4\xe9\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0@\xf5<\xda#\"{\xb7\xb6\xdd\xdd\xda\x17\x1b\t\x9b\xbd\x13n{U\xc1k\x11\xc3\x1b\x8b\xfbX\xee\xfe\xcb1\xcai\f\xf6meE\x97\x19\x83\\f\x14\x00\\\tS\x01\x00\x18 \x18\x8f\aT\x1a\xf5\x00\x00\x04\x80O\xd3MIJH\x03q\x98$I\n\xa3\x10B\xc6\x18B\b\x01\x00\x00D\x00\x04\x04\x00\xc0\x00\x00\x004\xcdieĩ@Β \xc7\x14B\n͌\b\x00\x00\x00\x00\x01\x00\x00\x00\x04\x00\x00\x00\b\x00\x00\x00kage types2\n\nimport (\n\t\"cmd/compile/internal/syntax\"\n\t\"strings\"\n\t\"unicode\"\n)\n\n// funcInst type-checks a func\")\n\tif err != nil {\n\t\tt.Fatalf(\"Prepare: %v\", err)\n\t}\n\tdefer stmt.Close()\n\n\tconst n = 10\n\tch := make(chan error, n)\n\tfor i := 0; i < n; i++ {\n\t\tgo func() {\n\t\t\tvar age int\n\t\t\terr := stmt.QueryRowool { return c != nil && c.fd != nil }\n\n// Implementation of the Conn interface.\n\n// Read implements the Conn Read method.\nfunc (c *conn) Read(b []byte) (int, error) {\n\tif !c.ok() {\n\t\treturn 0, t\n\t\t} else {\n\t\t\treturn nil, &FormatError{0, \"invalid magic number\", nil}\n\t\t}\n\t}\n\toffset := int64(4)\n\n\t// Read the number of FatArchHeaders that come after the fat_header.\n\tvar narch uint32\n\terr log.Fatal(err)\n\t\t}\n\t\tf := strings.Fields(line)\n\t\tif len(f) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tswitch f[0] {\n\t\tdefault:\n\t\t\tfmt.Fprintf(os.Stderr, \"?unknown command\\n\")\n\t\t\tcontinue\n\t\tcase \"tags\":\n\t\t\tprefix 00\\x00\\x00\", true},\n\t}\n\n\tfor _, v := range vectors {\n\t\tvar f formatter\n\t\tgot := make([]byte, len(v.want))\n\t\tf.formatNumeric(got, v.in)\n\t\tok := (f.err == nil)\n\t\tif ok != v.ok {\n\t\t\tif v.ok {\n\t\t\t\ttturn true\n\t}\n\treturn false\n}\nfunc rewriteValueARM_OpARMBICconst(v *Value) bool {\n\tv_0 := v.Args[0]\n\t// match: (BICconst [0] x)\n\t// result: x\n\tfor {\n\t\tif auxIntToInt32(v.AuxInt) != 0 {\n\t\t\tbreak\n\tnt) {\n\t\t\t\t\tt.Errorf(\"%5g %s %5g = %5s; want %5s\", x, op, y, got, want)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestFloatArithmeticOverflow(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tprec uint\n\t\t)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n// Copyright 2017 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\npackage ), uintptr(unsafe.Pointer(_p1)), 0)\n\tif e1 != 0 {\n\t\terr = errnoErr(e1)\n\t}\n\treturn\n}\n\n// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\nfunc Sync() (err error) {\n\t_, _, e1 := SyscDLINK = 0x10\n\tMOVEFILE_FAIL_IF_NOT")
// Decode the zstandard dictionary.
insp, err := zstd.InspectDictionary(zdict)
if err != nil {
panic(err)
}
// We are only interested in the contents.
fmt.Println("Dictionary content length:", len(insp.Content()))
// Create a dictionary.
// Assume that files start with "// Copyright (c) 2023".
// Search for the longest match for that.
// This may save a few bytes.
dict := s2.MakeDict(insp.Content(), []byte("// Copyright (c) 2023"))
// b := d.Bytes() will provide a dictionary that can be saved
// and reloaded with s2.NewDict(b).
fmt.Println("Dict size:", len(dict.Bytes()))
// Read a sample. Use this file.
sample, err := os.ReadFile("examples_test.go")
if err != nil {
panic(err)
}
encodedWithDict := dict.Encode(nil, sample)
encodedNoDict := s2.Encode(nil, sample)
// Print a less accurate output that is less likely to change.
// Since we include the (encoded) dictionary itself that will create better than expected compression.
if len(encodedWithDict) < len(encodedNoDict)-1000 {
fmt.Println("Saved more than 1000 bytes")
}
// To decode the content:
decoded, err := dict.Decode(nil, encodedWithDict)
if err != nil {
panic(err)
}
if bytes.Equal(decoded, sample) {
fmt.Println("They match!")
}
// OUTPUT:
// Dictionary content length: 1894
// Dict size: 1896
// Saved more than 1000 bytes
// They match!
}
|
