1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
package brotli
func utf8Position(last uint, c uint, clamp uint) uint {
if c < 128 {
return 0 /* Next one is the 'Byte 1' again. */
} else if c >= 192 { /* Next one is the 'Byte 2' of utf-8 encoding. */
return brotli_min_size_t(1, clamp)
} else {
/* Let's decide over the last byte if this ends the sequence. */
if last < 0xE0 {
return 0 /* Completed two or three byte coding. */ /* Next one is the 'Byte 3' of utf-8 encoding. */
} else {
return brotli_min_size_t(2, clamp)
}
}
}
func decideMultiByteStatsLevel(pos uint, len uint, mask uint, data []byte) uint {
var counts = [3]uint{0} /* should be 2, but 1 compresses better. */
var max_utf8 uint = 1
var last_c uint = 0
var i uint
for i = 0; i < len; i++ {
var c uint = uint(data[(pos+i)&mask])
counts[utf8Position(last_c, c, 2)]++
last_c = c
}
if counts[2] < 500 {
max_utf8 = 1
}
if counts[1]+counts[2] < 25 {
max_utf8 = 0
}
return max_utf8
}
func estimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte, cost []float32) {
var max_utf8 uint = decideMultiByteStatsLevel(pos, uint(len), mask, data)
/* Bootstrap histograms. */
var histogram = [3][256]uint{[256]uint{0}}
var window_half uint = 495
var in_window uint = brotli_min_size_t(window_half, uint(len))
var in_window_utf8 = [3]uint{0}
/* max_utf8 is 0 (normal ASCII single byte modeling),
1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
var i uint
{
var last_c uint = 0
var utf8_pos uint = 0
for i = 0; i < in_window; i++ {
var c uint = uint(data[(pos+i)&mask])
histogram[utf8_pos][c]++
in_window_utf8[utf8_pos]++
utf8_pos = utf8Position(last_c, c, max_utf8)
last_c = c
}
}
/* Compute bit costs with sliding window. */
for i = 0; i < len; i++ {
if i >= window_half {
var c uint
var last_c uint
if i < window_half+1 {
c = 0
} else {
c = uint(data[(pos+i-window_half-1)&mask])
}
if i < window_half+2 {
last_c = 0
} else {
last_c = uint(data[(pos+i-window_half-2)&mask])
}
/* Remove a byte in the past. */
var utf8_pos2 uint = utf8Position(last_c, c, max_utf8)
histogram[utf8_pos2][data[(pos+i-window_half)&mask]]--
in_window_utf8[utf8_pos2]--
}
if i+window_half < len {
var c uint = uint(data[(pos+i+window_half-1)&mask])
var last_c uint = uint(data[(pos+i+window_half-2)&mask])
/* Add a byte in the future. */
var utf8_pos2 uint = utf8Position(last_c, c, max_utf8)
histogram[utf8_pos2][data[(pos+i+window_half)&mask]]++
in_window_utf8[utf8_pos2]++
}
{
var c uint
var last_c uint
if i < 1 {
c = 0
} else {
c = uint(data[(pos+i-1)&mask])
}
if i < 2 {
last_c = 0
} else {
last_c = uint(data[(pos+i-2)&mask])
}
var utf8_pos uint = utf8Position(last_c, c, max_utf8)
var masked_pos uint = (pos + i) & mask
var histo uint = histogram[utf8_pos][data[masked_pos]]
var lit_cost float64
if histo == 0 {
histo = 1
}
lit_cost = fastLog2(in_window_utf8[utf8_pos]) - fastLog2(histo)
lit_cost += 0.02905
if lit_cost < 1.0 {
lit_cost *= 0.5
lit_cost += 0.5
}
/* Make the first bytes more expensive -- seems to help, not sure why.
Perhaps because the entropy source is changing its properties
rapidly in the beginning of the file, perhaps because the beginning
of the data is a statistical "anomaly". */
if i < 2000 {
lit_cost += 0.7 - (float64(2000-i) / 2000.0 * 0.35)
}
cost[i] = float32(lit_cost)
}
}
}
func estimateBitCostsForLiterals(pos uint, len uint, mask uint, data []byte, cost []float32) {
if isMostlyUTF8(data, pos, mask, uint(len), kMinUTF8Ratio) {
estimateBitCostsForLiteralsUTF8(pos, uint(len), mask, data, cost)
return
} else {
var histogram = [256]uint{0}
var window_half uint = 2000
var in_window uint = brotli_min_size_t(window_half, uint(len))
var i uint
/* Bootstrap histogram. */
for i = 0; i < in_window; i++ {
histogram[data[(pos+i)&mask]]++
}
/* Compute bit costs with sliding window. */
for i = 0; i < len; i++ {
var histo uint
if i >= window_half {
/* Remove a byte in the past. */
histogram[data[(pos+i-window_half)&mask]]--
in_window--
}
if i+window_half < len {
/* Add a byte in the future. */
histogram[data[(pos+i+window_half)&mask]]++
in_window++
}
histo = histogram[data[(pos+i)&mask]]
if histo == 0 {
histo = 1
}
{
var lit_cost float64 = fastLog2(in_window) - fastLog2(histo)
lit_cost += 0.029
if lit_cost < 1.0 {
lit_cost *= 0.5
lit_cost += 0.5
}
cost[i] = float32(lit_cost)
}
}
}
}
|
