BinaryMiddleware.js 32 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. */
  4. "use strict";
  5. const memoize = require("../util/memoize");
  6. const SerializerMiddleware = require("./SerializerMiddleware");
  7. /** @typedef {import("./types").BufferSerializableType} BufferSerializableType */
  8. /** @typedef {import("./types").PrimitiveSerializableType} PrimitiveSerializableType */
  9. /*
  10. Format:
  11. File -> Section*
  12. Section -> NullsSection |
  13. BooleansSection |
  14. F64NumbersSection |
  15. I32NumbersSection |
  16. I8NumbersSection |
  17. ShortStringSection |
  18. BigIntSection |
  19. I32BigIntSection |
  20. I8BigIntSection
  21. StringSection |
  22. BufferSection |
  23. NopSection
  24. NullsSection ->
  25. NullHeaderByte | Null2HeaderByte | Null3HeaderByte |
  26. Nulls8HeaderByte 0xnn (n:count - 4) |
  27. Nulls32HeaderByte n:ui32 (n:count - 260) |
  28. BooleansSection -> TrueHeaderByte | FalseHeaderByte | BooleansSectionHeaderByte BooleansCountAndBitsByte
  29. F64NumbersSection -> F64NumbersSectionHeaderByte f64*
  30. I32NumbersSection -> I32NumbersSectionHeaderByte i32*
  31. I8NumbersSection -> I8NumbersSectionHeaderByte i8*
  32. ShortStringSection -> ShortStringSectionHeaderByte ascii-byte*
  33. StringSection -> StringSectionHeaderByte i32:length utf8-byte*
  34. BufferSection -> BufferSectionHeaderByte i32:length byte*
  35. NopSection --> NopSectionHeaderByte
  36. BigIntSection -> BigIntSectionHeaderByte i32:length ascii-byte*
  37. I32BigIntSection -> I32BigIntSectionHeaderByte i32
  38. I8BigIntSection -> I8BigIntSectionHeaderByte i8
  39. ShortStringSectionHeaderByte -> 0b1nnn_nnnn (n:length)
  40. F64NumbersSectionHeaderByte -> 0b001n_nnnn (n:count - 1)
  41. I32NumbersSectionHeaderByte -> 0b010n_nnnn (n:count - 1)
  42. I8NumbersSectionHeaderByte -> 0b011n_nnnn (n:count - 1)
  43. NullsSectionHeaderByte -> 0b0001_nnnn (n:count - 1)
  44. BooleansCountAndBitsByte ->
  45. 0b0000_1xxx (count = 3) |
  46. 0b0001_xxxx (count = 4) |
  47. 0b001x_xxxx (count = 5) |
  48. 0b01xx_xxxx (count = 6) |
  49. 0b1nnn_nnnn (n:count - 7, 7 <= count <= 133)
  50. 0xff n:ui32 (n:count, 134 <= count < 2^32)
  51. StringSectionHeaderByte -> 0b0000_1110
  52. BufferSectionHeaderByte -> 0b0000_1111
  53. NopSectionHeaderByte -> 0b0000_1011
  54. BigIntSectionHeaderByte -> 0b0001_1010
  55. I32BigIntSectionHeaderByte -> 0b0001_1100
  56. I8BigIntSectionHeaderByte -> 0b0001_1011
  57. FalseHeaderByte -> 0b0000_1100
  58. TrueHeaderByte -> 0b0000_1101
  59. RawNumber -> n (n <= 10)
  60. */
  61. const LAZY_HEADER = 0x0b;
  62. const TRUE_HEADER = 0x0c;
  63. const FALSE_HEADER = 0x0d;
  64. const BOOLEANS_HEADER = 0x0e;
  65. const NULL_HEADER = 0x10;
  66. const NULL2_HEADER = 0x11;
  67. const NULL3_HEADER = 0x12;
  68. const NULLS8_HEADER = 0x13;
  69. const NULLS32_HEADER = 0x14;
  70. const NULL_AND_I8_HEADER = 0x15;
  71. const NULL_AND_I32_HEADER = 0x16;
  72. const NULL_AND_TRUE_HEADER = 0x17;
  73. const NULL_AND_FALSE_HEADER = 0x18;
  74. const BIGINT_HEADER = 0x1a;
  75. const BIGINT_I8_HEADER = 0x1b;
  76. const BIGINT_I32_HEADER = 0x1c;
  77. const STRING_HEADER = 0x1e;
  78. const BUFFER_HEADER = 0x1f;
  79. const I8_HEADER = 0x60;
  80. const I32_HEADER = 0x40;
  81. const F64_HEADER = 0x20;
  82. const SHORT_STRING_HEADER = 0x80;
  83. /** Uplift high-order bits */
  84. const NUMBERS_HEADER_MASK = 0xe0; // 0b1010_0000
  85. const NUMBERS_COUNT_MASK = 0x1f; // 0b0001_1111
  86. const SHORT_STRING_LENGTH_MASK = 0x7f; // 0b0111_1111
  87. const HEADER_SIZE = 1;
  88. const I8_SIZE = 1;
  89. const I32_SIZE = 4;
  90. const F64_SIZE = 8;
  91. const MEASURE_START_OPERATION = Symbol("MEASURE_START_OPERATION");
  92. const MEASURE_END_OPERATION = Symbol("MEASURE_END_OPERATION");
  93. /** @typedef {typeof MEASURE_START_OPERATION} MEASURE_START_OPERATION_TYPE */
  94. /** @typedef {typeof MEASURE_END_OPERATION} MEASURE_END_OPERATION_TYPE */
  95. /**
  96. * @param {number} n number
  97. * @returns {0 | 1 | 2} type of number for serialization
  98. */
  99. const identifyNumber = n => {
  100. if (n === (n | 0)) {
  101. if (n <= 127 && n >= -128) return 0;
  102. if (n <= 2147483647 && n >= -2147483648) return 1;
  103. }
  104. return 2;
  105. };
  106. /**
  107. * @param {bigint} n bigint
  108. * @returns {0 | 1 | 2} type of bigint for serialization
  109. */
  110. const identifyBigInt = n => {
  111. if (n <= BigInt(127) && n >= BigInt(-128)) return 0;
  112. if (n <= BigInt(2147483647) && n >= BigInt(-2147483648)) return 1;
  113. return 2;
  114. };
  115. /** @typedef {PrimitiveSerializableType[]} DeserializedType */
  116. /** @typedef {BufferSerializableType[]} SerializedType} */
  117. /** @typedef {{ retainedBuffer?: (x: Buffer) => Buffer }} Context} */
  118. /**
  119. * @template LazyInputValue
  120. * @template LazyOutputValue
  121. * @typedef {import("./SerializerMiddleware").LazyFunction<LazyInputValue, LazyOutputValue, BinaryMiddleware, undefined>} LazyFunction
  122. */
  123. /**
  124. * @extends {SerializerMiddleware<DeserializedType, SerializedType, Context>}
  125. */
  126. class BinaryMiddleware extends SerializerMiddleware {
  127. /**
  128. * @param {DeserializedType} data data
  129. * @param {Context} context context object
  130. * @returns {SerializedType | Promise<SerializedType> | null} serialized data
  131. */
  132. serialize(data, context) {
  133. return this._serialize(data, context);
  134. }
  135. /**
  136. * @param {LazyFunction<DeserializedType, SerializedType>} fn lazy function
  137. * @param {Context} context serialize function
  138. * @returns {LazyFunction<SerializedType, DeserializedType>} new lazy
  139. */
  140. _serializeLazy(fn, context) {
  141. return SerializerMiddleware.serializeLazy(fn, data =>
  142. this._serialize(data, context)
  143. );
  144. }
  145. /**
  146. * @param {DeserializedType} data data
  147. * @param {Context} context context object
  148. * @param {{ leftOverBuffer: Buffer | null, allocationSize: number, increaseCounter: number }} allocationScope allocation scope
  149. * @returns {SerializedType} serialized data
  150. */
  151. _serialize(
  152. data,
  153. context,
  154. allocationScope = {
  155. allocationSize: 1024,
  156. increaseCounter: 0,
  157. leftOverBuffer: null
  158. }
  159. ) {
  160. /** @type {Buffer | null} */
  161. let leftOverBuffer = null;
  162. /** @type {BufferSerializableType[]} */
  163. let buffers = [];
  164. /** @type {Buffer | null} */
  165. let currentBuffer = allocationScope ? allocationScope.leftOverBuffer : null;
  166. allocationScope.leftOverBuffer = null;
  167. let currentPosition = 0;
  168. if (currentBuffer === null) {
  169. currentBuffer = Buffer.allocUnsafe(allocationScope.allocationSize);
  170. }
  171. /**
  172. * @param {number} bytesNeeded bytes needed
  173. */
  174. const allocate = bytesNeeded => {
  175. if (currentBuffer !== null) {
  176. if (currentBuffer.length - currentPosition >= bytesNeeded) return;
  177. flush();
  178. }
  179. if (leftOverBuffer && leftOverBuffer.length >= bytesNeeded) {
  180. currentBuffer = leftOverBuffer;
  181. leftOverBuffer = null;
  182. } else {
  183. currentBuffer = Buffer.allocUnsafe(
  184. Math.max(bytesNeeded, allocationScope.allocationSize)
  185. );
  186. if (
  187. !(allocationScope.increaseCounter =
  188. (allocationScope.increaseCounter + 1) % 4) &&
  189. allocationScope.allocationSize < 16777216
  190. ) {
  191. allocationScope.allocationSize <<= 1;
  192. }
  193. }
  194. };
  195. const flush = () => {
  196. if (currentBuffer !== null) {
  197. if (currentPosition > 0) {
  198. buffers.push(
  199. Buffer.from(
  200. currentBuffer.buffer,
  201. currentBuffer.byteOffset,
  202. currentPosition
  203. )
  204. );
  205. }
  206. if (
  207. !leftOverBuffer ||
  208. leftOverBuffer.length < currentBuffer.length - currentPosition
  209. ) {
  210. leftOverBuffer = Buffer.from(
  211. currentBuffer.buffer,
  212. currentBuffer.byteOffset + currentPosition,
  213. currentBuffer.byteLength - currentPosition
  214. );
  215. }
  216. currentBuffer = null;
  217. currentPosition = 0;
  218. }
  219. };
  220. /**
  221. * @param {number} byte byte
  222. */
  223. const writeU8 = byte => {
  224. /** @type {Buffer} */
  225. (currentBuffer).writeUInt8(byte, currentPosition++);
  226. };
  227. /**
  228. * @param {number} ui32 ui32
  229. */
  230. const writeU32 = ui32 => {
  231. /** @type {Buffer} */
  232. (currentBuffer).writeUInt32LE(ui32, currentPosition);
  233. currentPosition += 4;
  234. };
  235. /** @type {number[]} */
  236. const measureStack = [];
  237. const measureStart = () => {
  238. measureStack.push(buffers.length, currentPosition);
  239. };
  240. /**
  241. * @returns {number} size
  242. */
  243. const measureEnd = () => {
  244. const oldPos = /** @type {number} */ (measureStack.pop());
  245. const buffersIndex = /** @type {number} */ (measureStack.pop());
  246. let size = currentPosition - oldPos;
  247. for (let i = buffersIndex; i < buffers.length; i++) {
  248. size += buffers[i].length;
  249. }
  250. return size;
  251. };
  252. for (let i = 0; i < data.length; i++) {
  253. const thing = data[i];
  254. switch (typeof thing) {
  255. case "function": {
  256. if (!SerializerMiddleware.isLazy(thing)) {
  257. throw new Error(`Unexpected function ${thing}`);
  258. }
  259. /** @type {SerializedType | LazyFunction<SerializedType, DeserializedType> | undefined} */
  260. let serializedData =
  261. SerializerMiddleware.getLazySerializedValue(thing);
  262. if (serializedData === undefined) {
  263. if (SerializerMiddleware.isLazy(thing, this)) {
  264. flush();
  265. allocationScope.leftOverBuffer = leftOverBuffer;
  266. const result =
  267. /** @type {PrimitiveSerializableType[]} */
  268. (thing());
  269. const data = this._serialize(result, context, allocationScope);
  270. leftOverBuffer = allocationScope.leftOverBuffer;
  271. allocationScope.leftOverBuffer = null;
  272. SerializerMiddleware.setLazySerializedValue(thing, data);
  273. serializedData = data;
  274. } else {
  275. serializedData = this._serializeLazy(thing, context);
  276. flush();
  277. buffers.push(serializedData);
  278. break;
  279. }
  280. } else if (typeof serializedData === "function") {
  281. flush();
  282. buffers.push(serializedData);
  283. break;
  284. }
  285. /** @type {number[]} */
  286. const lengths = [];
  287. for (const item of serializedData) {
  288. let last;
  289. if (typeof item === "function") {
  290. lengths.push(0);
  291. } else if (item.length === 0) {
  292. // ignore
  293. } else if (
  294. lengths.length > 0 &&
  295. (last = lengths[lengths.length - 1]) !== 0
  296. ) {
  297. const remaining = 0xffffffff - last;
  298. if (remaining >= item.length) {
  299. lengths[lengths.length - 1] += item.length;
  300. } else {
  301. lengths.push(item.length - remaining);
  302. lengths[lengths.length - 2] = 0xffffffff;
  303. }
  304. } else {
  305. lengths.push(item.length);
  306. }
  307. }
  308. allocate(5 + lengths.length * 4);
  309. writeU8(LAZY_HEADER);
  310. writeU32(lengths.length);
  311. for (const l of lengths) {
  312. writeU32(l);
  313. }
  314. flush();
  315. for (const item of serializedData) {
  316. buffers.push(item);
  317. }
  318. break;
  319. }
  320. case "string": {
  321. const len = Buffer.byteLength(thing);
  322. if (len >= 128 || len !== thing.length) {
  323. allocate(len + HEADER_SIZE + I32_SIZE);
  324. writeU8(STRING_HEADER);
  325. writeU32(len);
  326. currentBuffer.write(thing, currentPosition);
  327. currentPosition += len;
  328. } else if (len >= 70) {
  329. allocate(len + HEADER_SIZE);
  330. writeU8(SHORT_STRING_HEADER | len);
  331. currentBuffer.write(thing, currentPosition, "latin1");
  332. currentPosition += len;
  333. } else {
  334. allocate(len + HEADER_SIZE);
  335. writeU8(SHORT_STRING_HEADER | len);
  336. for (let i = 0; i < len; i++) {
  337. currentBuffer[currentPosition++] = thing.charCodeAt(i);
  338. }
  339. }
  340. break;
  341. }
  342. case "bigint": {
  343. const type = identifyBigInt(thing);
  344. if (type === 0 && thing >= 0 && thing <= BigInt(10)) {
  345. // shortcut for very small bigints
  346. allocate(HEADER_SIZE + I8_SIZE);
  347. writeU8(BIGINT_I8_HEADER);
  348. writeU8(Number(thing));
  349. break;
  350. }
  351. switch (type) {
  352. case 0: {
  353. let n = 1;
  354. allocate(HEADER_SIZE + I8_SIZE * n);
  355. writeU8(BIGINT_I8_HEADER | (n - 1));
  356. while (n > 0) {
  357. currentBuffer.writeInt8(
  358. Number(/** @type {bigint} */ (data[i])),
  359. currentPosition
  360. );
  361. currentPosition += I8_SIZE;
  362. n--;
  363. i++;
  364. }
  365. i--;
  366. break;
  367. }
  368. case 1: {
  369. let n = 1;
  370. allocate(HEADER_SIZE + I32_SIZE * n);
  371. writeU8(BIGINT_I32_HEADER | (n - 1));
  372. while (n > 0) {
  373. currentBuffer.writeInt32LE(
  374. Number(/** @type {bigint} */ (data[i])),
  375. currentPosition
  376. );
  377. currentPosition += I32_SIZE;
  378. n--;
  379. i++;
  380. }
  381. i--;
  382. break;
  383. }
  384. default: {
  385. const value = thing.toString();
  386. const len = Buffer.byteLength(value);
  387. allocate(len + HEADER_SIZE + I32_SIZE);
  388. writeU8(BIGINT_HEADER);
  389. writeU32(len);
  390. currentBuffer.write(value, currentPosition);
  391. currentPosition += len;
  392. break;
  393. }
  394. }
  395. break;
  396. }
  397. case "number": {
  398. const type = identifyNumber(thing);
  399. if (type === 0 && thing >= 0 && thing <= 10) {
  400. // shortcut for very small numbers
  401. allocate(I8_SIZE);
  402. writeU8(thing);
  403. break;
  404. }
  405. /**
  406. * amount of numbers to write
  407. * @type {number}
  408. */
  409. let n = 1;
  410. for (; n < 32 && i + n < data.length; n++) {
  411. const item = data[i + n];
  412. if (typeof item !== "number") break;
  413. if (identifyNumber(item) !== type) break;
  414. }
  415. switch (type) {
  416. case 0:
  417. allocate(HEADER_SIZE + I8_SIZE * n);
  418. writeU8(I8_HEADER | (n - 1));
  419. while (n > 0) {
  420. currentBuffer.writeInt8(
  421. /** @type {number} */ (data[i]),
  422. currentPosition
  423. );
  424. currentPosition += I8_SIZE;
  425. n--;
  426. i++;
  427. }
  428. break;
  429. case 1:
  430. allocate(HEADER_SIZE + I32_SIZE * n);
  431. writeU8(I32_HEADER | (n - 1));
  432. while (n > 0) {
  433. currentBuffer.writeInt32LE(
  434. /** @type {number} */ (data[i]),
  435. currentPosition
  436. );
  437. currentPosition += I32_SIZE;
  438. n--;
  439. i++;
  440. }
  441. break;
  442. case 2:
  443. allocate(HEADER_SIZE + F64_SIZE * n);
  444. writeU8(F64_HEADER | (n - 1));
  445. while (n > 0) {
  446. currentBuffer.writeDoubleLE(
  447. /** @type {number} */ (data[i]),
  448. currentPosition
  449. );
  450. currentPosition += F64_SIZE;
  451. n--;
  452. i++;
  453. }
  454. break;
  455. }
  456. i--;
  457. break;
  458. }
  459. case "boolean": {
  460. let lastByte = thing === true ? 1 : 0;
  461. const bytes = [];
  462. let count = 1;
  463. let n;
  464. for (n = 1; n < 0xffffffff && i + n < data.length; n++) {
  465. const item = data[i + n];
  466. if (typeof item !== "boolean") break;
  467. const pos = count & 0x7;
  468. if (pos === 0) {
  469. bytes.push(lastByte);
  470. lastByte = item === true ? 1 : 0;
  471. } else if (item === true) {
  472. lastByte |= 1 << pos;
  473. }
  474. count++;
  475. }
  476. i += count - 1;
  477. if (count === 1) {
  478. allocate(HEADER_SIZE);
  479. writeU8(lastByte === 1 ? TRUE_HEADER : FALSE_HEADER);
  480. } else if (count === 2) {
  481. allocate(HEADER_SIZE * 2);
  482. writeU8(lastByte & 1 ? TRUE_HEADER : FALSE_HEADER);
  483. writeU8(lastByte & 2 ? TRUE_HEADER : FALSE_HEADER);
  484. } else if (count <= 6) {
  485. allocate(HEADER_SIZE + I8_SIZE);
  486. writeU8(BOOLEANS_HEADER);
  487. writeU8((1 << count) | lastByte);
  488. } else if (count <= 133) {
  489. allocate(HEADER_SIZE + I8_SIZE + I8_SIZE * bytes.length + I8_SIZE);
  490. writeU8(BOOLEANS_HEADER);
  491. writeU8(0x80 | (count - 7));
  492. for (const byte of bytes) writeU8(byte);
  493. writeU8(lastByte);
  494. } else {
  495. allocate(
  496. HEADER_SIZE +
  497. I8_SIZE +
  498. I32_SIZE +
  499. I8_SIZE * bytes.length +
  500. I8_SIZE
  501. );
  502. writeU8(BOOLEANS_HEADER);
  503. writeU8(0xff);
  504. writeU32(count);
  505. for (const byte of bytes) writeU8(byte);
  506. writeU8(lastByte);
  507. }
  508. break;
  509. }
  510. case "object": {
  511. if (thing === null) {
  512. let n;
  513. for (n = 1; n < 0x100000104 && i + n < data.length; n++) {
  514. const item = data[i + n];
  515. if (item !== null) break;
  516. }
  517. i += n - 1;
  518. if (n === 1) {
  519. if (i + 1 < data.length) {
  520. const next = data[i + 1];
  521. if (next === true) {
  522. allocate(HEADER_SIZE);
  523. writeU8(NULL_AND_TRUE_HEADER);
  524. i++;
  525. } else if (next === false) {
  526. allocate(HEADER_SIZE);
  527. writeU8(NULL_AND_FALSE_HEADER);
  528. i++;
  529. } else if (typeof next === "number") {
  530. const type = identifyNumber(next);
  531. if (type === 0) {
  532. allocate(HEADER_SIZE + I8_SIZE);
  533. writeU8(NULL_AND_I8_HEADER);
  534. currentBuffer.writeInt8(next, currentPosition);
  535. currentPosition += I8_SIZE;
  536. i++;
  537. } else if (type === 1) {
  538. allocate(HEADER_SIZE + I32_SIZE);
  539. writeU8(NULL_AND_I32_HEADER);
  540. currentBuffer.writeInt32LE(next, currentPosition);
  541. currentPosition += I32_SIZE;
  542. i++;
  543. } else {
  544. allocate(HEADER_SIZE);
  545. writeU8(NULL_HEADER);
  546. }
  547. } else {
  548. allocate(HEADER_SIZE);
  549. writeU8(NULL_HEADER);
  550. }
  551. } else {
  552. allocate(HEADER_SIZE);
  553. writeU8(NULL_HEADER);
  554. }
  555. } else if (n === 2) {
  556. allocate(HEADER_SIZE);
  557. writeU8(NULL2_HEADER);
  558. } else if (n === 3) {
  559. allocate(HEADER_SIZE);
  560. writeU8(NULL3_HEADER);
  561. } else if (n < 260) {
  562. allocate(HEADER_SIZE + I8_SIZE);
  563. writeU8(NULLS8_HEADER);
  564. writeU8(n - 4);
  565. } else {
  566. allocate(HEADER_SIZE + I32_SIZE);
  567. writeU8(NULLS32_HEADER);
  568. writeU32(n - 260);
  569. }
  570. } else if (Buffer.isBuffer(thing)) {
  571. if (thing.length < 8192) {
  572. allocate(HEADER_SIZE + I32_SIZE + thing.length);
  573. writeU8(BUFFER_HEADER);
  574. writeU32(thing.length);
  575. thing.copy(currentBuffer, currentPosition);
  576. currentPosition += thing.length;
  577. } else {
  578. allocate(HEADER_SIZE + I32_SIZE);
  579. writeU8(BUFFER_HEADER);
  580. writeU32(thing.length);
  581. flush();
  582. buffers.push(thing);
  583. }
  584. }
  585. break;
  586. }
  587. case "symbol": {
  588. if (thing === MEASURE_START_OPERATION) {
  589. measureStart();
  590. } else if (thing === MEASURE_END_OPERATION) {
  591. const size = measureEnd();
  592. allocate(HEADER_SIZE + I32_SIZE);
  593. writeU8(I32_HEADER);
  594. currentBuffer.writeInt32LE(size, currentPosition);
  595. currentPosition += I32_SIZE;
  596. }
  597. break;
  598. }
  599. default: {
  600. throw new Error(
  601. `Unknown typeof "${typeof thing}" in binary middleware`
  602. );
  603. }
  604. }
  605. }
  606. flush();
  607. allocationScope.leftOverBuffer = leftOverBuffer;
  608. // avoid leaking memory
  609. currentBuffer = null;
  610. leftOverBuffer = null;
  611. allocationScope = /** @type {EXPECTED_ANY} */ (undefined);
  612. const _buffers = buffers;
  613. buffers = /** @type {EXPECTED_ANY} */ (undefined);
  614. return _buffers;
  615. }
  616. /**
  617. * @param {SerializedType} data data
  618. * @param {Context} context context object
  619. * @returns {DeserializedType | Promise<DeserializedType>} deserialized data
  620. */
  621. deserialize(data, context) {
  622. return this._deserialize(data, context);
  623. }
  624. /**
  625. * @private
  626. * @param {SerializedType} content content
  627. * @param {Context} context context object
  628. * @returns {LazyFunction<DeserializedType, SerializedType>} lazy function
  629. */
  630. _createLazyDeserialized(content, context) {
  631. return SerializerMiddleware.createLazy(
  632. memoize(() => this._deserialize(content, context)),
  633. this,
  634. undefined,
  635. content
  636. );
  637. }
  638. /**
  639. * @private
  640. * @param {LazyFunction<SerializedType, DeserializedType>} fn lazy function
  641. * @param {Context} context context object
  642. * @returns {LazyFunction<DeserializedType, SerializedType>} new lazy
  643. */
  644. _deserializeLazy(fn, context) {
  645. return SerializerMiddleware.deserializeLazy(fn, data =>
  646. this._deserialize(data, context)
  647. );
  648. }
  649. /**
  650. * @param {SerializedType} data data
  651. * @param {Context} context context object
  652. * @returns {DeserializedType} deserialized data
  653. */
  654. _deserialize(data, context) {
  655. let currentDataItem = 0;
  656. /** @type {BufferSerializableType | null} */
  657. let currentBuffer = data[0];
  658. let currentIsBuffer = Buffer.isBuffer(currentBuffer);
  659. let currentPosition = 0;
  660. const retainedBuffer = context.retainedBuffer || (x => x);
  661. const checkOverflow = () => {
  662. if (currentPosition >= /** @type {Buffer} */ (currentBuffer).length) {
  663. currentPosition = 0;
  664. currentDataItem++;
  665. currentBuffer =
  666. currentDataItem < data.length ? data[currentDataItem] : null;
  667. currentIsBuffer = Buffer.isBuffer(currentBuffer);
  668. }
  669. };
  670. /**
  671. * @param {number} n n
  672. * @returns {boolean} true when in current buffer, otherwise false
  673. */
  674. const isInCurrentBuffer = n =>
  675. currentIsBuffer &&
  676. n + currentPosition <= /** @type {Buffer} */ (currentBuffer).length;
  677. const ensureBuffer = () => {
  678. if (!currentIsBuffer) {
  679. throw new Error(
  680. currentBuffer === null
  681. ? "Unexpected end of stream"
  682. : "Unexpected lazy element in stream"
  683. );
  684. }
  685. };
  686. /**
  687. * Reads n bytes
  688. * @param {number} n amount of bytes to read
  689. * @returns {Buffer} buffer with bytes
  690. */
  691. const read = n => {
  692. ensureBuffer();
  693. const rem =
  694. /** @type {Buffer} */ (currentBuffer).length - currentPosition;
  695. if (rem < n) {
  696. const buffers = [read(rem)];
  697. n -= rem;
  698. ensureBuffer();
  699. while (/** @type {Buffer} */ (currentBuffer).length < n) {
  700. const b = /** @type {Buffer} */ (currentBuffer);
  701. buffers.push(b);
  702. n -= b.length;
  703. currentDataItem++;
  704. currentBuffer =
  705. currentDataItem < data.length ? data[currentDataItem] : null;
  706. currentIsBuffer = Buffer.isBuffer(currentBuffer);
  707. ensureBuffer();
  708. }
  709. buffers.push(read(n));
  710. return Buffer.concat(buffers);
  711. }
  712. const b = /** @type {Buffer} */ (currentBuffer);
  713. const res = Buffer.from(b.buffer, b.byteOffset + currentPosition, n);
  714. currentPosition += n;
  715. checkOverflow();
  716. return res;
  717. };
  718. /**
  719. * Reads up to n bytes
  720. * @param {number} n amount of bytes to read
  721. * @returns {Buffer} buffer with bytes
  722. */
  723. const readUpTo = n => {
  724. ensureBuffer();
  725. const rem =
  726. /** @type {Buffer} */
  727. (currentBuffer).length - currentPosition;
  728. if (rem < n) {
  729. n = rem;
  730. }
  731. const b = /** @type {Buffer} */ (currentBuffer);
  732. const res = Buffer.from(b.buffer, b.byteOffset + currentPosition, n);
  733. currentPosition += n;
  734. checkOverflow();
  735. return res;
  736. };
  737. /**
  738. * @returns {number} U8
  739. */
  740. const readU8 = () => {
  741. ensureBuffer();
  742. /**
  743. * There is no need to check remaining buffer size here
  744. * since {@link checkOverflow} guarantees at least one byte remaining
  745. */
  746. const byte =
  747. /** @type {Buffer} */
  748. (currentBuffer).readUInt8(currentPosition);
  749. currentPosition += I8_SIZE;
  750. checkOverflow();
  751. return byte;
  752. };
  753. /**
  754. * @returns {number} U32
  755. */
  756. const readU32 = () => read(I32_SIZE).readUInt32LE(0);
  757. /**
  758. * @param {number} data data
  759. * @param {number} n n
  760. */
  761. const readBits = (data, n) => {
  762. let mask = 1;
  763. while (n !== 0) {
  764. result.push((data & mask) !== 0);
  765. mask <<= 1;
  766. n--;
  767. }
  768. };
  769. const dispatchTable = Array.from({ length: 256 }).map((_, header) => {
  770. switch (header) {
  771. case LAZY_HEADER:
  772. return () => {
  773. const count = readU32();
  774. const lengths = Array.from({ length: count }).map(() => readU32());
  775. /** @type {(Buffer | LazyFunction<SerializedType, DeserializedType>)[]} */
  776. const content = [];
  777. for (let l of lengths) {
  778. if (l === 0) {
  779. if (typeof currentBuffer !== "function") {
  780. throw new Error("Unexpected non-lazy element in stream");
  781. }
  782. content.push(currentBuffer);
  783. currentDataItem++;
  784. currentBuffer =
  785. currentDataItem < data.length ? data[currentDataItem] : null;
  786. currentIsBuffer = Buffer.isBuffer(currentBuffer);
  787. } else {
  788. do {
  789. const buf = readUpTo(l);
  790. l -= buf.length;
  791. content.push(retainedBuffer(buf));
  792. } while (l > 0);
  793. }
  794. }
  795. result.push(this._createLazyDeserialized(content, context));
  796. };
  797. case BUFFER_HEADER:
  798. return () => {
  799. const len = readU32();
  800. result.push(retainedBuffer(read(len)));
  801. };
  802. case TRUE_HEADER:
  803. return () => result.push(true);
  804. case FALSE_HEADER:
  805. return () => result.push(false);
  806. case NULL3_HEADER:
  807. return () => result.push(null, null, null);
  808. case NULL2_HEADER:
  809. return () => result.push(null, null);
  810. case NULL_HEADER:
  811. return () => result.push(null);
  812. case NULL_AND_TRUE_HEADER:
  813. return () => result.push(null, true);
  814. case NULL_AND_FALSE_HEADER:
  815. return () => result.push(null, false);
  816. case NULL_AND_I8_HEADER:
  817. return () => {
  818. if (currentIsBuffer) {
  819. result.push(
  820. null,
  821. /** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
  822. );
  823. currentPosition += I8_SIZE;
  824. checkOverflow();
  825. } else {
  826. result.push(null, read(I8_SIZE).readInt8(0));
  827. }
  828. };
  829. case NULL_AND_I32_HEADER:
  830. return () => {
  831. result.push(null);
  832. if (isInCurrentBuffer(I32_SIZE)) {
  833. result.push(
  834. /** @type {Buffer} */ (currentBuffer).readInt32LE(
  835. currentPosition
  836. )
  837. );
  838. currentPosition += I32_SIZE;
  839. checkOverflow();
  840. } else {
  841. result.push(read(I32_SIZE).readInt32LE(0));
  842. }
  843. };
  844. case NULLS8_HEADER:
  845. return () => {
  846. const len = readU8() + 4;
  847. for (let i = 0; i < len; i++) {
  848. result.push(null);
  849. }
  850. };
  851. case NULLS32_HEADER:
  852. return () => {
  853. const len = readU32() + 260;
  854. for (let i = 0; i < len; i++) {
  855. result.push(null);
  856. }
  857. };
  858. case BOOLEANS_HEADER:
  859. return () => {
  860. const innerHeader = readU8();
  861. if ((innerHeader & 0xf0) === 0) {
  862. readBits(innerHeader, 3);
  863. } else if ((innerHeader & 0xe0) === 0) {
  864. readBits(innerHeader, 4);
  865. } else if ((innerHeader & 0xc0) === 0) {
  866. readBits(innerHeader, 5);
  867. } else if ((innerHeader & 0x80) === 0) {
  868. readBits(innerHeader, 6);
  869. } else if (innerHeader !== 0xff) {
  870. let count = (innerHeader & 0x7f) + 7;
  871. while (count > 8) {
  872. readBits(readU8(), 8);
  873. count -= 8;
  874. }
  875. readBits(readU8(), count);
  876. } else {
  877. let count = readU32();
  878. while (count > 8) {
  879. readBits(readU8(), 8);
  880. count -= 8;
  881. }
  882. readBits(readU8(), count);
  883. }
  884. };
  885. case STRING_HEADER:
  886. return () => {
  887. const len = readU32();
  888. if (isInCurrentBuffer(len) && currentPosition + len < 0x7fffffff) {
  889. result.push(
  890. /** @type {Buffer} */
  891. (currentBuffer).toString(
  892. undefined,
  893. currentPosition,
  894. currentPosition + len
  895. )
  896. );
  897. currentPosition += len;
  898. checkOverflow();
  899. } else {
  900. result.push(read(len).toString());
  901. }
  902. };
  903. case SHORT_STRING_HEADER:
  904. return () => result.push("");
  905. case SHORT_STRING_HEADER | 1:
  906. return () => {
  907. if (currentIsBuffer && currentPosition < 0x7ffffffe) {
  908. result.push(
  909. /** @type {Buffer} */
  910. (currentBuffer).toString(
  911. "latin1",
  912. currentPosition,
  913. currentPosition + 1
  914. )
  915. );
  916. currentPosition++;
  917. checkOverflow();
  918. } else {
  919. result.push(read(1).toString("latin1"));
  920. }
  921. };
  922. case I8_HEADER:
  923. return () => {
  924. if (currentIsBuffer) {
  925. result.push(
  926. /** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
  927. );
  928. currentPosition++;
  929. checkOverflow();
  930. } else {
  931. result.push(read(1).readInt8(0));
  932. }
  933. };
  934. case BIGINT_I8_HEADER: {
  935. const len = 1;
  936. return () => {
  937. const need = I8_SIZE * len;
  938. if (isInCurrentBuffer(need)) {
  939. for (let i = 0; i < len; i++) {
  940. const value =
  941. /** @type {Buffer} */
  942. (currentBuffer).readInt8(currentPosition);
  943. result.push(BigInt(value));
  944. currentPosition += I8_SIZE;
  945. }
  946. checkOverflow();
  947. } else {
  948. const buf = read(need);
  949. for (let i = 0; i < len; i++) {
  950. const value = buf.readInt8(i * I8_SIZE);
  951. result.push(BigInt(value));
  952. }
  953. }
  954. };
  955. }
  956. case BIGINT_I32_HEADER: {
  957. const len = 1;
  958. return () => {
  959. const need = I32_SIZE * len;
  960. if (isInCurrentBuffer(need)) {
  961. for (let i = 0; i < len; i++) {
  962. const value = /** @type {Buffer} */ (currentBuffer).readInt32LE(
  963. currentPosition
  964. );
  965. result.push(BigInt(value));
  966. currentPosition += I32_SIZE;
  967. }
  968. checkOverflow();
  969. } else {
  970. const buf = read(need);
  971. for (let i = 0; i < len; i++) {
  972. const value = buf.readInt32LE(i * I32_SIZE);
  973. result.push(BigInt(value));
  974. }
  975. }
  976. };
  977. }
  978. case BIGINT_HEADER: {
  979. return () => {
  980. const len = readU32();
  981. if (isInCurrentBuffer(len) && currentPosition + len < 0x7fffffff) {
  982. const value =
  983. /** @type {Buffer} */
  984. (currentBuffer).toString(
  985. undefined,
  986. currentPosition,
  987. currentPosition + len
  988. );
  989. result.push(BigInt(value));
  990. currentPosition += len;
  991. checkOverflow();
  992. } else {
  993. const value = read(len).toString();
  994. result.push(BigInt(value));
  995. }
  996. };
  997. }
  998. default:
  999. if (header <= 10) {
  1000. return () => result.push(header);
  1001. } else if ((header & SHORT_STRING_HEADER) === SHORT_STRING_HEADER) {
  1002. const len = header & SHORT_STRING_LENGTH_MASK;
  1003. return () => {
  1004. if (
  1005. isInCurrentBuffer(len) &&
  1006. currentPosition + len < 0x7fffffff
  1007. ) {
  1008. result.push(
  1009. /** @type {Buffer} */
  1010. (currentBuffer).toString(
  1011. "latin1",
  1012. currentPosition,
  1013. currentPosition + len
  1014. )
  1015. );
  1016. currentPosition += len;
  1017. checkOverflow();
  1018. } else {
  1019. result.push(read(len).toString("latin1"));
  1020. }
  1021. };
  1022. } else if ((header & NUMBERS_HEADER_MASK) === F64_HEADER) {
  1023. const len = (header & NUMBERS_COUNT_MASK) + 1;
  1024. return () => {
  1025. const need = F64_SIZE * len;
  1026. if (isInCurrentBuffer(need)) {
  1027. for (let i = 0; i < len; i++) {
  1028. result.push(
  1029. /** @type {Buffer} */ (currentBuffer).readDoubleLE(
  1030. currentPosition
  1031. )
  1032. );
  1033. currentPosition += F64_SIZE;
  1034. }
  1035. checkOverflow();
  1036. } else {
  1037. const buf = read(need);
  1038. for (let i = 0; i < len; i++) {
  1039. result.push(buf.readDoubleLE(i * F64_SIZE));
  1040. }
  1041. }
  1042. };
  1043. } else if ((header & NUMBERS_HEADER_MASK) === I32_HEADER) {
  1044. const len = (header & NUMBERS_COUNT_MASK) + 1;
  1045. return () => {
  1046. const need = I32_SIZE * len;
  1047. if (isInCurrentBuffer(need)) {
  1048. for (let i = 0; i < len; i++) {
  1049. result.push(
  1050. /** @type {Buffer} */ (currentBuffer).readInt32LE(
  1051. currentPosition
  1052. )
  1053. );
  1054. currentPosition += I32_SIZE;
  1055. }
  1056. checkOverflow();
  1057. } else {
  1058. const buf = read(need);
  1059. for (let i = 0; i < len; i++) {
  1060. result.push(buf.readInt32LE(i * I32_SIZE));
  1061. }
  1062. }
  1063. };
  1064. } else if ((header & NUMBERS_HEADER_MASK) === I8_HEADER) {
  1065. const len = (header & NUMBERS_COUNT_MASK) + 1;
  1066. return () => {
  1067. const need = I8_SIZE * len;
  1068. if (isInCurrentBuffer(need)) {
  1069. for (let i = 0; i < len; i++) {
  1070. result.push(
  1071. /** @type {Buffer} */ (currentBuffer).readInt8(
  1072. currentPosition
  1073. )
  1074. );
  1075. currentPosition += I8_SIZE;
  1076. }
  1077. checkOverflow();
  1078. } else {
  1079. const buf = read(need);
  1080. for (let i = 0; i < len; i++) {
  1081. result.push(buf.readInt8(i * I8_SIZE));
  1082. }
  1083. }
  1084. };
  1085. }
  1086. return () => {
  1087. throw new Error(`Unexpected header byte 0x${header.toString(16)}`);
  1088. };
  1089. }
  1090. });
  1091. /** @type {DeserializedType} */
  1092. let result = [];
  1093. while (currentBuffer !== null) {
  1094. if (typeof currentBuffer === "function") {
  1095. result.push(this._deserializeLazy(currentBuffer, context));
  1096. currentDataItem++;
  1097. currentBuffer =
  1098. currentDataItem < data.length ? data[currentDataItem] : null;
  1099. currentIsBuffer = Buffer.isBuffer(currentBuffer);
  1100. } else {
  1101. const header = readU8();
  1102. dispatchTable[header]();
  1103. }
  1104. }
  1105. // avoid leaking memory in context
  1106. // eslint-disable-next-line prefer-const
  1107. let _result = result;
  1108. result = /** @type {EXPECTED_ANY} */ (undefined);
  1109. return _result;
  1110. }
  1111. }
  1112. module.exports = BinaryMiddleware;
  1113. module.exports.MEASURE_END_OPERATION = MEASURE_END_OPERATION;
  1114. module.exports.MEASURE_START_OPERATION = MEASURE_START_OPERATION;