Created
June 14, 2021 21:01
-
-
Save 101arrowz/a979452d4355992cbf8f257cbffc9edd to your computer and use it in GitHub Desktop.
Generate Zstandard table representation from frequencies
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Will be used in fzstd if the default distributions ever update | |
// Not fast because it's meant to be run once for precomputation | |
// accuracy log | |
const al = 5; | |
// distribution | |
const dist = [1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, | |
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1]; | |
const out = new Uint8Array(1000); | |
let pos = 4; | |
out[0] = al - 5; | |
const msb = (val) => { | |
let bits = 0; | |
for (; (1 << bits) <= val; ++bits); | |
return bits - 1; | |
} | |
let probs = 1 << al; | |
for (const val of dist) { | |
const ev = (val + 1) << (pos & 7); | |
out[pos >> 3] |= ev; | |
out[(pos >> 3) + 1] |= ev >> 8; | |
const bits = msb(probs + 1) + 1; | |
const msv = (1 << bits) - 2 - probs; | |
pos += bits - ((val + 1) < msv); | |
probs -= Math.abs(val); | |
} | |
console.log(out, probs, pos); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment