Brotli.cpp 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946
  1. /*
  2. * Copyright (c) 2022, Michiel Visser <opensource@webmichiel.nl>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/BinarySearch.h>
  7. #include <AK/QuickSort.h>
  8. #include <LibCompress/Brotli.h>
  9. #include <LibCompress/BrotliDictionary.h>
  10. namespace Compress {
  11. ErrorOr<size_t> Brotli::CanonicalCode::read_symbol(LittleEndianInputBitStream& input_stream) const
  12. {
  13. size_t code_bits = 1;
  14. while (code_bits < (1 << 16)) {
  15. // FIXME: This is very inefficient and could greatly be improved by implementing this
  16. // algorithm: https://www.hanshq.net/zip.html#huffdec
  17. size_t index;
  18. if (binary_search(m_symbol_codes.span(), code_bits, &index))
  19. return m_symbol_values[index];
  20. code_bits = (code_bits << 1) | TRY(input_stream.read_bit());
  21. }
  22. return Error::from_string_literal("no matching code found");
  23. }
  24. BrotliDecompressionStream::BrotliDecompressionStream(MaybeOwned<Stream> stream)
  25. : m_input_stream(move(stream))
  26. {
  27. }
  28. ErrorOr<size_t> BrotliDecompressionStream::read_window_length()
  29. {
  30. if (TRY(m_input_stream.read_bit())) {
  31. switch (TRY(m_input_stream.read_bits(3))) {
  32. case 0: {
  33. switch (TRY(m_input_stream.read_bits(3))) {
  34. case 0:
  35. return 17;
  36. case 1:
  37. return Error::from_string_literal("invalid window length");
  38. case 2:
  39. return 10;
  40. case 3:
  41. return 11;
  42. case 4:
  43. return 12;
  44. case 5:
  45. return 13;
  46. case 6:
  47. return 14;
  48. case 7:
  49. return 15;
  50. default:
  51. VERIFY_NOT_REACHED();
  52. }
  53. }
  54. case 1:
  55. return 18;
  56. case 2:
  57. return 19;
  58. case 3:
  59. return 20;
  60. case 4:
  61. return 21;
  62. case 5:
  63. return 22;
  64. case 6:
  65. return 23;
  66. case 7:
  67. return 24;
  68. default:
  69. VERIFY_NOT_REACHED();
  70. }
  71. } else {
  72. return 16;
  73. }
  74. }
  75. ErrorOr<size_t> BrotliDecompressionStream::read_size_number_of_nibbles()
  76. {
  77. switch (TRY(m_input_stream.read_bits(2))) {
  78. case 0:
  79. return 4;
  80. case 1:
  81. return 5;
  82. case 2:
  83. return 6;
  84. case 3:
  85. return 0;
  86. default:
  87. VERIFY_NOT_REACHED();
  88. }
  89. }
  90. ErrorOr<size_t> BrotliDecompressionStream::read_variable_length()
  91. {
  92. // Value Bit Pattern
  93. // ----- -----------
  94. // 1 0
  95. // 2 0001
  96. // 3..4 x0011
  97. // 5..8 xx0101
  98. // 9..16 xxx0111
  99. // 17..32 xxxx1001
  100. // 33..64 xxxxx1011
  101. // 65..128 xxxxxx1101
  102. // 129..256 xxxxxxx1111
  103. if (TRY(m_input_stream.read_bit())) {
  104. switch (TRY(m_input_stream.read_bits(3))) {
  105. case 0:
  106. return 2;
  107. case 1:
  108. return 3 + TRY(m_input_stream.read_bits(1));
  109. case 2:
  110. return 5 + TRY(m_input_stream.read_bits(2));
  111. case 3:
  112. return 9 + TRY(m_input_stream.read_bits(3));
  113. case 4:
  114. return 17 + TRY(m_input_stream.read_bits(4));
  115. case 5:
  116. return 33 + TRY(m_input_stream.read_bits(5));
  117. case 6:
  118. return 65 + TRY(m_input_stream.read_bits(6));
  119. case 7:
  120. return 129 + TRY(m_input_stream.read_bits(7));
  121. default:
  122. VERIFY_NOT_REACHED();
  123. }
  124. } else {
  125. return 1;
  126. }
  127. }
  128. ErrorOr<size_t> Brotli::CanonicalCode::read_complex_prefix_code_length(LittleEndianInputBitStream& stream)
  129. {
  130. // Symbol Code
  131. // ------ ----
  132. // 0 00
  133. // 1 0111
  134. // 2 011
  135. // 3 10
  136. // 4 01
  137. // 5 1111
  138. switch (TRY(stream.read_bits(2))) {
  139. case 0:
  140. return 0;
  141. case 1:
  142. return 4;
  143. case 2:
  144. return 3;
  145. case 3: {
  146. if (TRY(stream.read_bit()) == 0) {
  147. return 2;
  148. } else {
  149. if (TRY(stream.read_bit()) == 0) {
  150. return 1;
  151. } else {
  152. return 5;
  153. }
  154. }
  155. }
  156. default:
  157. VERIFY_NOT_REACHED();
  158. }
  159. }
  160. ErrorOr<Brotli::CanonicalCode> Brotli::CanonicalCode::read_prefix_code(LittleEndianInputBitStream& stream, size_t alphabet_size)
  161. {
  162. size_t hskip = TRY(stream.read_bits(2));
  163. if (hskip == 1)
  164. return TRY(read_simple_prefix_code(stream, alphabet_size));
  165. return TRY(read_complex_prefix_code(stream, alphabet_size, hskip));
  166. }
  167. ErrorOr<Brotli::CanonicalCode> Brotli::CanonicalCode::read_simple_prefix_code(LittleEndianInputBitStream& stream, size_t alphabet_size)
  168. {
  169. CanonicalCode code {};
  170. size_t number_of_symbols = 1 + TRY(stream.read_bits(2));
  171. size_t symbol_size = 0;
  172. while ((1u << symbol_size) < alphabet_size)
  173. symbol_size++;
  174. Vector<size_t> symbols;
  175. for (size_t i = 0; i < number_of_symbols; i++) {
  176. size_t symbol = TRY(stream.read_bits(symbol_size));
  177. symbols.append(symbol);
  178. if (symbol >= alphabet_size)
  179. return Error::from_string_literal("symbol larger than alphabet");
  180. }
  181. if (number_of_symbols == 1) {
  182. code.m_symbol_codes.append(0b1);
  183. code.m_symbol_values = move(symbols);
  184. } else if (number_of_symbols == 2) {
  185. code.m_symbol_codes.extend({ 0b10, 0b11 });
  186. if (symbols[0] > symbols[1])
  187. swap(symbols[0], symbols[1]);
  188. code.m_symbol_values = move(symbols);
  189. } else if (number_of_symbols == 3) {
  190. code.m_symbol_codes.extend({ 0b10, 0b110, 0b111 });
  191. if (symbols[1] > symbols[2])
  192. swap(symbols[1], symbols[2]);
  193. code.m_symbol_values = move(symbols);
  194. } else if (number_of_symbols == 4) {
  195. bool tree_select = TRY(stream.read_bit());
  196. if (tree_select) {
  197. code.m_symbol_codes.extend({ 0b10, 0b110, 0b1110, 0b1111 });
  198. if (symbols[2] > symbols[3])
  199. swap(symbols[2], symbols[3]);
  200. code.m_symbol_values = move(symbols);
  201. } else {
  202. code.m_symbol_codes.extend({ 0b100, 0b101, 0b110, 0b111 });
  203. quick_sort(symbols);
  204. code.m_symbol_values = move(symbols);
  205. }
  206. }
  207. return code;
  208. }
  209. ErrorOr<Brotli::CanonicalCode> Brotli::CanonicalCode::read_complex_prefix_code(LittleEndianInputBitStream& stream, size_t alphabet_size, size_t hskip)
  210. {
  211. // hskip should only be 0, 2 or 3
  212. VERIFY(hskip != 1);
  213. VERIFY(hskip <= 3);
  214. // Read the prefix code_value that is used to encode the actual prefix code_value
  215. size_t const symbol_mapping[18] = { 1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
  216. size_t code_length[18] { 0 };
  217. size_t code_length_counts[6] { 0 };
  218. size_t sum = 0;
  219. size_t number_of_non_zero_symbols = 0;
  220. for (size_t i = hskip; i < 18; i++) {
  221. size_t len = TRY(read_complex_prefix_code_length(stream));
  222. code_length[symbol_mapping[i]] = len;
  223. if (len != 0) {
  224. code_length_counts[len]++;
  225. sum += (32 >> len);
  226. number_of_non_zero_symbols++;
  227. }
  228. if (sum == 32)
  229. break;
  230. else if (sum > 32)
  231. return Error::from_string_literal("invalid prefix code");
  232. }
  233. CanonicalCode temp_code;
  234. if (number_of_non_zero_symbols > 1) {
  235. size_t code_value = 0;
  236. for (size_t bits = 1; bits <= 5; bits++) {
  237. code_value = (code_value + code_length_counts[bits - 1]) << 1;
  238. size_t current_code_value = code_value;
  239. for (size_t i = 0; i < 18; i++) {
  240. size_t len = code_length[i];
  241. if (len == bits) {
  242. temp_code.m_symbol_codes.append((1 << bits) | current_code_value);
  243. temp_code.m_symbol_values.append(i);
  244. current_code_value++;
  245. }
  246. }
  247. }
  248. } else {
  249. for (size_t i = 0; i < 18; i++) {
  250. size_t len = code_length[i];
  251. if (len != 0) {
  252. temp_code.m_symbol_codes.append(1);
  253. temp_code.m_symbol_values.append(i);
  254. break;
  255. }
  256. }
  257. }
  258. // Read the actual prefix code_value
  259. sum = 0;
  260. size_t i = 0;
  261. size_t previous_non_zero_code_length = 8;
  262. size_t last_symbol = 0;
  263. size_t last_repeat = 0;
  264. Vector<size_t> result_symbols;
  265. Vector<size_t> result_lengths;
  266. size_t result_lengths_count[16] { 0 };
  267. while (i < alphabet_size) {
  268. auto symbol = TRY(temp_code.read_symbol(stream));
  269. if (symbol < 16) {
  270. result_symbols.append(i);
  271. result_lengths.append(symbol);
  272. result_lengths_count[symbol]++;
  273. if (symbol != 0) {
  274. previous_non_zero_code_length = symbol;
  275. sum += (32768 >> symbol);
  276. if (sum == 32768)
  277. break;
  278. else if (sum > 32768)
  279. return Error::from_string_literal("invalid prefix code");
  280. }
  281. last_repeat = 0;
  282. i++;
  283. } else if (symbol == 16) {
  284. size_t repeat_count = 0;
  285. if (last_symbol == 16 && last_repeat != 0) {
  286. repeat_count = (4 * (last_repeat - 2));
  287. } else {
  288. last_repeat = 0;
  289. }
  290. repeat_count += 3 + TRY(stream.read_bits(2));
  291. for (size_t rep = 0; rep < (repeat_count - last_repeat); rep++) {
  292. result_symbols.append(i);
  293. result_lengths.append(previous_non_zero_code_length);
  294. result_lengths_count[previous_non_zero_code_length]++;
  295. if (previous_non_zero_code_length != 0) {
  296. sum += (32768 >> previous_non_zero_code_length);
  297. if (sum == 32768)
  298. break;
  299. else if (sum > 32768)
  300. return Error::from_string_literal("invalid prefix code");
  301. }
  302. i++;
  303. if (i >= alphabet_size)
  304. break;
  305. }
  306. if (sum == 32768)
  307. break;
  308. VERIFY(sum < 32768);
  309. last_repeat = repeat_count;
  310. } else if (symbol == 17) {
  311. size_t repeat_count = 0;
  312. if (last_symbol == 17 && last_repeat != 0) {
  313. repeat_count = (8 * (last_repeat - 2));
  314. } else {
  315. last_repeat = 0;
  316. }
  317. repeat_count += 3 + TRY(stream.read_bits(3));
  318. i += (repeat_count - last_repeat);
  319. last_repeat = repeat_count;
  320. }
  321. last_symbol = symbol;
  322. }
  323. result_lengths_count[0] = 0;
  324. CanonicalCode final_code;
  325. size_t code_value = 0;
  326. for (size_t bits = 1; bits < 16; bits++) {
  327. code_value = (code_value + result_lengths_count[bits - 1]) << 1;
  328. size_t current_code_value = code_value;
  329. for (size_t n = 0; n < result_symbols.size(); n++) {
  330. size_t len = result_lengths[n];
  331. if (len == bits) {
  332. final_code.m_symbol_codes.append((1 << bits) | current_code_value);
  333. final_code.m_symbol_values.append(result_symbols[n]);
  334. current_code_value++;
  335. }
  336. }
  337. }
  338. return final_code;
  339. }
  340. static void inverse_move_to_front_transform(Span<u8> v)
  341. {
  342. // RFC 7932 section 7.3
  343. u8 mtf[256];
  344. for (size_t i = 0; i < 256; ++i) {
  345. mtf[i] = (u8)i;
  346. }
  347. for (size_t i = 0; i < v.size(); ++i) {
  348. u8 index = v[i];
  349. u8 value = mtf[index];
  350. v[i] = value;
  351. for (; index; --index) {
  352. mtf[index] = mtf[index - 1];
  353. }
  354. mtf[0] = value;
  355. }
  356. }
  357. ErrorOr<void> BrotliDecompressionStream::read_context_map(size_t number_of_codes, Vector<u8>& context_map, size_t context_map_size)
  358. {
  359. bool use_run_length_encoding = TRY(m_input_stream.read_bit());
  360. size_t run_length_encoding_max = 0;
  361. if (use_run_length_encoding) {
  362. run_length_encoding_max = 1 + TRY(m_input_stream.read_bits(4));
  363. }
  364. auto const code = TRY(CanonicalCode::read_prefix_code(m_input_stream, number_of_codes + run_length_encoding_max));
  365. size_t i = 0;
  366. while (i < context_map_size) {
  367. size_t symbol = TRY(code.read_symbol(m_input_stream));
  368. if (symbol <= run_length_encoding_max) {
  369. size_t repeat_base = 1 << symbol;
  370. size_t repeat_additional = TRY(m_input_stream.read_bits(symbol));
  371. size_t repeat_count = repeat_base + repeat_additional;
  372. while (repeat_count--) {
  373. context_map.append(0);
  374. i++;
  375. }
  376. } else {
  377. size_t value = symbol - run_length_encoding_max;
  378. context_map.append(value);
  379. i++;
  380. }
  381. }
  382. bool inverse_move_to_front = TRY(m_input_stream.read_bit());
  383. if (inverse_move_to_front)
  384. inverse_move_to_front_transform(context_map.span());
  385. return {};
  386. }
  387. ErrorOr<void> BrotliDecompressionStream::read_block_configuration(Block& block)
  388. {
  389. size_t blocks_of_type = TRY(read_variable_length());
  390. block.type = 0;
  391. block.type_previous = 1;
  392. block.number_of_types = blocks_of_type;
  393. if (blocks_of_type == 1) {
  394. block.length = 16 * MiB;
  395. block.type_code = {};
  396. block.length_code = {};
  397. } else {
  398. block.type_code = TRY(CanonicalCode::read_prefix_code(m_input_stream, 2 + blocks_of_type));
  399. block.length_code = TRY(CanonicalCode::read_prefix_code(m_input_stream, 26));
  400. TRY(block_update_length(block));
  401. }
  402. return {};
  403. }
  404. ErrorOr<void> BrotliDecompressionStream::block_update_length(Block& block)
  405. {
  406. size_t const block_length_code_base[26] { 1, 5, 9, 13, 17, 25, 33, 41, 49, 65, 81, 97, 113, 145, 177, 209, 241, 305, 369, 497, 753, 1265, 2289, 4337, 8433, 16625 };
  407. size_t const block_length_code_extra[26] { 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 24 };
  408. size_t symbol = TRY(block.length_code.read_symbol(m_input_stream));
  409. size_t block_length = block_length_code_base[symbol] + TRY(m_input_stream.read_bits(block_length_code_extra[symbol]));
  410. block.length = block_length;
  411. return {};
  412. }
  413. ErrorOr<void> BrotliDecompressionStream::block_read_new_state(Block& block)
  414. {
  415. size_t block_type_symbol = TRY(block.type_code.read_symbol(m_input_stream));
  416. TRY(block_update_length(block));
  417. if (block_type_symbol == 0) {
  418. swap(block.type, block.type_previous);
  419. } else if (block_type_symbol == 1) {
  420. block.type_previous = block.type;
  421. block.type = (block.type + 1) % block.number_of_types;
  422. } else {
  423. block.type_previous = block.type;
  424. block.type = block_type_symbol - 2;
  425. }
  426. return {};
  427. }
  428. size_t BrotliDecompressionStream::literal_code_index_from_context()
  429. {
  430. size_t const context_id_lut0[256] {
  431. 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,
  432. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  433. 8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,
  434. 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,
  435. 12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,
  436. 52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,
  437. 12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,
  438. 60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0,
  439. 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
  440. 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
  441. 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
  442. 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
  443. 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
  444. 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
  445. 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
  446. 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3
  447. };
  448. size_t const context_id_lut1[256] {
  449. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  450. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  451. 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  452. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,
  453. 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  454. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
  455. 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
  456. 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0,
  457. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  458. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  459. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  460. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  461. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  462. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  463. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  464. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
  465. };
  466. size_t const context_id_lut2[256] {
  467. 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  468. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  469. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  470. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  471. 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
  472. 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
  473. 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
  474. 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
  475. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  476. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  477. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  478. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  479. 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
  480. 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
  481. 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
  482. 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7
  483. };
  484. size_t context_mode = m_literal_context_modes[m_literal_block.type];
  485. size_t context_id;
  486. switch (context_mode) {
  487. case 0:
  488. context_id = m_lookback_buffer.value().lookback(1, 0) & 0x3f;
  489. break;
  490. case 1:
  491. context_id = m_lookback_buffer.value().lookback(1, 0) >> 2;
  492. break;
  493. case 2:
  494. context_id = context_id_lut0[m_lookback_buffer.value().lookback(1, 0)] | context_id_lut1[m_lookback_buffer.value().lookback(2, 0)];
  495. break;
  496. case 3:
  497. context_id = (context_id_lut2[m_lookback_buffer.value().lookback(1, 0)] << 3) | context_id_lut2[m_lookback_buffer.value().lookback(2, 0)];
  498. break;
  499. default:
  500. VERIFY_NOT_REACHED();
  501. }
  502. size_t literal_code_index = m_context_mapping_literal[64 * m_literal_block.type + context_id];
  503. return literal_code_index;
  504. }
  505. ErrorOr<Bytes> BrotliDecompressionStream::read_some(Bytes output_buffer)
  506. {
  507. size_t bytes_read = 0;
  508. while (bytes_read < output_buffer.size()) {
  509. if (m_current_state == State::WindowSize) {
  510. size_t window_bits = TRY(read_window_length());
  511. m_window_size = (1 << window_bits) - 16;
  512. m_lookback_buffer = TRY(LookbackBuffer::try_create(m_window_size));
  513. m_current_state = State::Idle;
  514. } else if (m_current_state == State::Idle) {
  515. // If the final block was read, we are done decompressing
  516. if (m_read_final_block)
  517. break;
  518. // RFC 7932 section 9.1
  519. //
  520. // 1 bit: ISLAST, set to 1 if this is the last meta-block
  521. m_read_final_block = TRY(m_input_stream.read_bit());
  522. if (m_read_final_block) {
  523. // 1 bit: ISLASTEMPTY, if set to 1, the meta-block is empty; this
  524. // field is only present if ISLAST bit is set -- if it is 1,
  525. // then the meta-block and the brotli stream ends at that
  526. // bit, with any remaining bits in the last byte of the
  527. // compressed stream filled with zeros (if the fill bits are
  528. // not zero, then the stream should be rejected as invalid)
  529. bool is_last_block_empty = TRY(m_input_stream.read_bit());
  530. // If the last block is empty we are done decompressing
  531. if (is_last_block_empty)
  532. break;
  533. }
  534. // 2 bits: MNIBBLES, number of nibbles to represent the uncompressed
  535. // length
  536. size_t size_number_of_nibbles = TRY(read_size_number_of_nibbles());
  537. // If MNIBBLES is 0, the meta-block is empty, i.e., it does
  538. // not generate any uncompressed data. In this case, the
  539. // rest of the meta-block has the following format:
  540. if (size_number_of_nibbles == 0) {
  541. // 1 bit: reserved, must be zero
  542. bool reserved = TRY(m_input_stream.read_bit());
  543. if (reserved)
  544. return Error::from_string_literal("invalid reserved bit");
  545. // 2 bits: MSKIPBYTES, number of bytes to represent
  546. // metadata length
  547. //
  548. // MSKIPBYTES * 8 bits: MSKIPLEN - 1, where MSKIPLEN is
  549. // the number of metadata bytes; this field is
  550. // only present if MSKIPBYTES is positive;
  551. // otherwise, MSKIPLEN is 0 (if MSKIPBYTES is
  552. // greater than 1, and the last byte is all
  553. // zeros, then the stream should be rejected as
  554. // invalid)
  555. size_t skip_bytes = TRY(m_input_stream.read_bits(2));
  556. if (skip_bytes == 0) {
  557. // 0..7 bits: fill bits until the next byte boundary,
  558. // must be all zeros
  559. u8 remainder = m_input_stream.align_to_byte_boundary();
  560. if (remainder != 0)
  561. return Error::from_string_literal("remainder bits are non-zero");
  562. continue;
  563. }
  564. // MSKIPLEN bytes of metadata, not part of the
  565. // uncompressed data or the sliding window
  566. size_t skip_length = 1 + TRY(m_input_stream.read_bits(8 * skip_bytes));
  567. u8 remainder = m_input_stream.align_to_byte_boundary();
  568. if (remainder != 0)
  569. return Error::from_string_literal("remainder bits are non-zero");
  570. // Discard meta-data bytes
  571. u8 temp_buffer[4096];
  572. Bytes temp_bytes { temp_buffer, 4096 };
  573. while (skip_length > 0) {
  574. Bytes temp_bytes_slice = temp_bytes.slice(0, min(4096, skip_length));
  575. auto metadata_bytes = TRY(m_input_stream.read_some(temp_bytes_slice));
  576. if (metadata_bytes.is_empty())
  577. return Error::from_string_literal("eof");
  578. if (metadata_bytes.last() == 0)
  579. return Error::from_string_literal("invalid stream");
  580. skip_length -= metadata_bytes.size();
  581. }
  582. continue;
  583. }
  584. size_t uncompressed_size = 1 + TRY(m_input_stream.read_bits(4 * size_number_of_nibbles));
  585. // 1 bit: ISUNCOMPRESSED, if set to 1, any bits of compressed data
  586. // up to the next byte boundary are ignored, and the rest of
  587. // the meta-block contains MLEN bytes of literal data; this
  588. // field is only present if the ISLAST bit is not set (if the
  589. // ignored bits are not all zeros, the stream should be
  590. // rejected as invalid)
  591. bool is_uncompressed = false;
  592. if (!m_read_final_block)
  593. is_uncompressed = TRY(m_input_stream.read_bit());
  594. m_bytes_left = uncompressed_size;
  595. if (is_uncompressed) {
  596. u8 remainder = m_input_stream.align_to_byte_boundary();
  597. if (remainder != 0)
  598. return Error::from_string_literal("remainder is non-zero");
  599. m_current_state = State::UncompressedData;
  600. } else {
  601. TRY(read_block_configuration(m_literal_block));
  602. TRY(read_block_configuration(m_insert_and_copy_block));
  603. TRY(read_block_configuration(m_distance_block));
  604. m_postfix_bits = TRY(m_input_stream.read_bits(2));
  605. m_direct_distances = TRY(m_input_stream.read_bits(4)) << m_postfix_bits;
  606. m_literal_context_modes.clear();
  607. for (size_t i = 0; i < m_literal_block.number_of_types; i++) {
  608. size_t context_mode = TRY(m_input_stream.read_bits(2));
  609. m_literal_context_modes.append(context_mode);
  610. }
  611. m_context_mapping_literal.clear();
  612. size_t number_of_literal_codes = TRY(read_variable_length());
  613. if (number_of_literal_codes == 1) {
  614. for (size_t i = 0; i < 64 * m_literal_block.number_of_types; i++)
  615. m_context_mapping_literal.append(0);
  616. } else {
  617. TRY(read_context_map(number_of_literal_codes, m_context_mapping_literal, 64 * m_literal_block.number_of_types));
  618. }
  619. m_context_mapping_distance.clear();
  620. size_t number_of_distance_codes = TRY(read_variable_length());
  621. if (number_of_distance_codes == 1) {
  622. for (size_t i = 0; i < 4 * m_distance_block.number_of_types; i++)
  623. m_context_mapping_distance.append(0);
  624. } else {
  625. TRY(read_context_map(number_of_distance_codes, m_context_mapping_distance, 4 * m_distance_block.number_of_types));
  626. }
  627. m_literal_codes.clear();
  628. for (size_t i = 0; i < number_of_literal_codes; i++) {
  629. m_literal_codes.append(TRY(CanonicalCode::read_prefix_code(m_input_stream, 256)));
  630. }
  631. m_insert_and_copy_codes.clear();
  632. for (size_t i = 0; i < m_insert_and_copy_block.number_of_types; i++) {
  633. m_insert_and_copy_codes.append(TRY(CanonicalCode::read_prefix_code(m_input_stream, 704)));
  634. }
  635. m_distance_codes.clear();
  636. for (size_t i = 0; i < number_of_distance_codes; i++) {
  637. m_distance_codes.append(TRY(CanonicalCode::read_prefix_code(m_input_stream, 16 + m_direct_distances + (48 << m_postfix_bits))));
  638. }
  639. m_current_state = State::CompressedCommand;
  640. }
  641. } else if (m_current_state == State::UncompressedData) {
  642. size_t number_of_fitting_bytes = min(output_buffer.size() - bytes_read, m_bytes_left);
  643. VERIFY(number_of_fitting_bytes > 0);
  644. auto uncompressed_bytes = TRY(m_input_stream.read_some(output_buffer.slice(bytes_read, number_of_fitting_bytes)));
  645. if (uncompressed_bytes.is_empty())
  646. return Error::from_string_literal("eof");
  647. // TODO: Replace the home-grown LookbackBuffer with AK::CircularBuffer.
  648. for (auto c : uncompressed_bytes)
  649. m_lookback_buffer.value().write(c);
  650. m_bytes_left -= uncompressed_bytes.size();
  651. bytes_read += uncompressed_bytes.size();
  652. // If all bytes were read, return to the idle state
  653. if (m_bytes_left == 0)
  654. m_current_state = State::Idle;
  655. } else if (m_current_state == State::CompressedCommand) {
  656. if (m_insert_and_copy_block.length == 0) {
  657. TRY(block_read_new_state(m_insert_and_copy_block));
  658. }
  659. m_insert_and_copy_block.length--;
  660. size_t insert_and_copy_symbol = TRY(m_insert_and_copy_codes[m_insert_and_copy_block.type].read_symbol(m_input_stream));
  661. size_t const insert_length_code_base[11] { 0, 0, 0, 0, 8, 8, 0, 16, 8, 16, 16 };
  662. size_t const copy_length_code_base[11] { 0, 8, 0, 8, 0, 8, 16, 0, 16, 8, 16 };
  663. bool const implicit_zero_distance[11] { true, true, false, false, false, false, false, false, false, false, false };
  664. size_t insert_and_copy_index = insert_and_copy_symbol >> 6;
  665. size_t insert_length_code_offset = (insert_and_copy_symbol >> 3) & 0b111;
  666. size_t copy_length_code_offset = insert_and_copy_symbol & 0b111;
  667. size_t insert_length_code = insert_length_code_base[insert_and_copy_index] + insert_length_code_offset;
  668. size_t copy_length_code = copy_length_code_base[insert_and_copy_index] + copy_length_code_offset;
  669. m_implicit_zero_distance = implicit_zero_distance[insert_and_copy_index];
  670. size_t const insert_length_base[24] { 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594 };
  671. size_t const insert_length_extra[24] { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24 };
  672. size_t const copy_length_base[24] { 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094, 2118 };
  673. size_t const copy_length_extra[24] { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24 };
  674. m_insert_length = insert_length_base[insert_length_code] + TRY(m_input_stream.read_bits(insert_length_extra[insert_length_code]));
  675. m_copy_length = copy_length_base[copy_length_code] + TRY(m_input_stream.read_bits(copy_length_extra[copy_length_code]));
  676. if (m_insert_length > 0) {
  677. m_current_state = State::CompressedLiteral;
  678. } else {
  679. m_current_state = State::CompressedDistance;
  680. }
  681. } else if (m_current_state == State::CompressedLiteral) {
  682. if (m_literal_block.length == 0) {
  683. TRY(block_read_new_state(m_literal_block));
  684. }
  685. m_literal_block.length--;
  686. size_t literal_code_index = literal_code_index_from_context();
  687. size_t literal_value = TRY(m_literal_codes[literal_code_index].read_symbol(m_input_stream));
  688. output_buffer[bytes_read] = literal_value;
  689. m_lookback_buffer.value().write(literal_value);
  690. bytes_read++;
  691. m_insert_length--;
  692. m_bytes_left--;
  693. if (m_bytes_left == 0)
  694. m_current_state = State::Idle;
  695. else if (m_insert_length == 0)
  696. m_current_state = State::CompressedDistance;
  697. } else if (m_current_state == State::CompressedDistance) {
  698. size_t distance_symbol;
  699. if (m_implicit_zero_distance) {
  700. distance_symbol = 0;
  701. } else {
  702. if (m_distance_block.length == 0) {
  703. TRY(block_read_new_state(m_distance_block));
  704. }
  705. m_distance_block.length--;
  706. size_t context_id = clamp(m_copy_length - 2, 0, 3);
  707. size_t distance_code_index = m_context_mapping_distance[4 * m_distance_block.type + context_id];
  708. distance_symbol = TRY(m_distance_codes[distance_code_index].read_symbol(m_input_stream));
  709. }
  710. size_t distance;
  711. bool reuse_previous_distance = false;
  712. if (distance_symbol < 16) {
  713. switch (distance_symbol) {
  714. case 0:
  715. distance = m_distances[0];
  716. reuse_previous_distance = true;
  717. break;
  718. case 1:
  719. distance = m_distances[1];
  720. break;
  721. case 2:
  722. distance = m_distances[2];
  723. break;
  724. case 3:
  725. distance = m_distances[3];
  726. break;
  727. case 4:
  728. distance = m_distances[0] - 1;
  729. break;
  730. case 5:
  731. distance = m_distances[0] + 1;
  732. break;
  733. case 6:
  734. distance = m_distances[0] - 2;
  735. break;
  736. case 7:
  737. distance = m_distances[0] + 2;
  738. break;
  739. case 8:
  740. distance = m_distances[0] - 3;
  741. break;
  742. case 9:
  743. distance = m_distances[0] + 3;
  744. break;
  745. case 10:
  746. distance = m_distances[1] - 1;
  747. break;
  748. case 11:
  749. distance = m_distances[1] + 1;
  750. break;
  751. case 12:
  752. distance = m_distances[1] - 2;
  753. break;
  754. case 13:
  755. distance = m_distances[1] + 2;
  756. break;
  757. case 14:
  758. distance = m_distances[1] - 3;
  759. break;
  760. case 15:
  761. distance = m_distances[1] + 3;
  762. break;
  763. }
  764. } else if (distance_symbol < 16 + m_direct_distances) {
  765. distance = distance_symbol - 15;
  766. } else {
  767. size_t POSTFIX_MASK = (1 << m_postfix_bits) - 1;
  768. size_t ndistbits = 1 + ((distance_symbol - m_direct_distances - 16) >> (m_postfix_bits + 1));
  769. size_t dextra = TRY(m_input_stream.read_bits(ndistbits));
  770. size_t hcode = (distance_symbol - m_direct_distances - 16) >> m_postfix_bits;
  771. size_t lcode = (distance_symbol - m_direct_distances - 16) & POSTFIX_MASK;
  772. size_t offset = ((2 + (hcode & 1)) << ndistbits) - 4;
  773. distance = ((offset + dextra) << m_postfix_bits) + lcode + m_direct_distances + 1;
  774. }
  775. m_distance = distance;
  776. size_t total_written = m_lookback_buffer.value().total_written();
  777. size_t max_lookback = min(total_written, m_window_size);
  778. if (distance > max_lookback) {
  779. size_t word_index = distance - (max_lookback + 1);
  780. m_dictionary_data = TRY(BrotliDictionary::lookup_word(word_index, m_copy_length));
  781. m_copy_length = m_dictionary_data.size();
  782. if (m_copy_length == 0)
  783. m_current_state = State::CompressedCommand;
  784. else
  785. m_current_state = State::CompressedDictionary;
  786. } else {
  787. if (!reuse_previous_distance) {
  788. m_distances[3] = m_distances[2];
  789. m_distances[2] = m_distances[1];
  790. m_distances[1] = m_distances[0];
  791. m_distances[0] = distance;
  792. }
  793. m_current_state = State::CompressedCopy;
  794. }
  795. } else if (m_current_state == State::CompressedCopy) {
  796. u8 copy_value = m_lookback_buffer.value().lookback(m_distance);
  797. output_buffer[bytes_read] = copy_value;
  798. m_lookback_buffer.value().write(copy_value);
  799. bytes_read++;
  800. m_copy_length--;
  801. m_bytes_left--;
  802. if (m_bytes_left == 0)
  803. m_current_state = State::Idle;
  804. else if (m_copy_length == 0)
  805. m_current_state = State::CompressedCommand;
  806. } else if (m_current_state == State::CompressedDictionary) {
  807. size_t offset = m_dictionary_data.size() - m_copy_length;
  808. u8 dictionary_value = m_dictionary_data[offset];
  809. output_buffer[bytes_read] = dictionary_value;
  810. m_lookback_buffer.value().write(dictionary_value);
  811. bytes_read++;
  812. m_copy_length--;
  813. m_bytes_left--;
  814. if (m_bytes_left == 0)
  815. m_current_state = State::Idle;
  816. else if (m_copy_length == 0)
  817. m_current_state = State::CompressedCommand;
  818. }
  819. }
  820. return output_buffer.slice(0, bytes_read);
  821. }
  822. bool BrotliDecompressionStream::is_eof() const
  823. {
  824. return m_read_final_block && m_current_state == State::Idle;
  825. }
  826. }