CachedInputFileSystem.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const { nextTick } = require("process");
  7. /** @typedef {import("./Resolver").FileSystem} FileSystem */
  8. /** @typedef {import("./Resolver").PathLike} PathLike */
  9. /** @typedef {import("./Resolver").PathOrFileDescriptor} PathOrFileDescriptor */
  10. /** @typedef {import("./Resolver").SyncFileSystem} SyncFileSystem */
  11. /** @typedef {FileSystem & SyncFileSystem} BaseFileSystem */
  12. /**
  13. * @template T
  14. * @typedef {import("./Resolver").FileSystemCallback<T>} FileSystemCallback<T>
  15. */
  16. /**
  17. * @param {string} path path
  18. * @returns {string} dirname
  19. */
  20. const dirname = (path) => {
  21. let idx = path.length - 1;
  22. while (idx >= 0) {
  23. const char = path.charCodeAt(idx);
  24. // slash or backslash
  25. if (char === 47 || char === 92) break;
  26. idx--;
  27. }
  28. if (idx < 0) return "";
  29. return path.slice(0, idx);
  30. };
  31. /**
  32. * @template T
  33. * @param {FileSystemCallback<T>[]} callbacks callbacks
  34. * @param {Error | null} err error
  35. * @param {T} result result
  36. */
  37. const runCallbacks = (callbacks, err, result) => {
  38. if (callbacks.length === 1) {
  39. callbacks[0](err, result);
  40. callbacks.length = 0;
  41. return;
  42. }
  43. let error;
  44. for (const callback of callbacks) {
  45. try {
  46. callback(err, result);
  47. } catch (err) {
  48. if (!error) error = err;
  49. }
  50. }
  51. callbacks.length = 0;
  52. if (error) throw error;
  53. };
  54. // eslint-disable-next-line jsdoc/no-restricted-syntax
  55. /** @typedef {Function} EXPECTED_FUNCTION */
  56. // eslint-disable-next-line jsdoc/no-restricted-syntax
  57. /** @typedef {any} EXPECTED_ANY */
  58. class OperationMergerBackend {
  59. /**
  60. * @param {EXPECTED_FUNCTION | undefined} provider async method in filesystem
  61. * @param {EXPECTED_FUNCTION | undefined} syncProvider sync method in filesystem
  62. * @param {BaseFileSystem} providerContext call context for the provider methods
  63. */
  64. constructor(provider, syncProvider, providerContext) {
  65. this._provider = provider;
  66. this._syncProvider = syncProvider;
  67. this._providerContext = providerContext;
  68. this._activeAsyncOperations = new Map();
  69. this.provide = this._provider
  70. ? // Comment to align jsdoc
  71. /**
  72. * @param {PathLike | PathOrFileDescriptor} path path
  73. * @param {object | FileSystemCallback<EXPECTED_ANY> | undefined} options options
  74. * @param {FileSystemCallback<EXPECTED_ANY>=} callback callback
  75. * @returns {EXPECTED_ANY} result
  76. */
  77. (path, options, callback) => {
  78. if (typeof options === "function") {
  79. callback =
  80. /** @type {FileSystemCallback<EXPECTED_ANY>} */
  81. (options);
  82. options = undefined;
  83. }
  84. if (
  85. typeof path !== "string" &&
  86. !Buffer.isBuffer(path) &&
  87. !(path instanceof URL) &&
  88. typeof path !== "number"
  89. ) {
  90. /** @type {EXPECTED_FUNCTION} */
  91. (callback)(
  92. new TypeError("path must be a string, Buffer, URL or number"),
  93. );
  94. return;
  95. }
  96. if (options) {
  97. return /** @type {EXPECTED_FUNCTION} */ (this._provider).call(
  98. this._providerContext,
  99. path,
  100. options,
  101. callback,
  102. );
  103. }
  104. let callbacks = this._activeAsyncOperations.get(path);
  105. if (callbacks) {
  106. callbacks.push(callback);
  107. return;
  108. }
  109. this._activeAsyncOperations.set(path, (callbacks = [callback]));
  110. /** @type {EXPECTED_FUNCTION} */
  111. (provider)(
  112. path,
  113. /**
  114. * @param {Error} err error
  115. * @param {EXPECTED_ANY} result result
  116. */
  117. (err, result) => {
  118. this._activeAsyncOperations.delete(path);
  119. runCallbacks(callbacks, err, result);
  120. },
  121. );
  122. }
  123. : null;
  124. this.provideSync = this._syncProvider
  125. ? // Comment to align jsdoc
  126. /**
  127. * @param {PathLike | PathOrFileDescriptor} path path
  128. * @param {object=} options options
  129. * @returns {EXPECTED_ANY} result
  130. */
  131. (path, options) =>
  132. /** @type {EXPECTED_FUNCTION} */ (this._syncProvider).call(
  133. this._providerContext,
  134. path,
  135. options,
  136. )
  137. : null;
  138. }
  139. purge() {}
  140. purgeParent() {}
  141. }
  142. /*
  143. IDLE:
  144. insert data: goto SYNC
  145. SYNC:
  146. before provide: run ticks
  147. event loop tick: goto ASYNC_ACTIVE
  148. ASYNC:
  149. timeout: run tick, goto ASYNC_PASSIVE
  150. ASYNC_PASSIVE:
  151. before provide: run ticks
  152. IDLE --[insert data]--> SYNC --[event loop tick]--> ASYNC_ACTIVE --[interval tick]-> ASYNC_PASSIVE
  153. ^ |
  154. +---------[insert data]-------+
  155. */
  156. const STORAGE_MODE_IDLE = 0;
  157. const STORAGE_MODE_SYNC = 1;
  158. const STORAGE_MODE_ASYNC = 2;
  159. /**
  160. * @callback Provide
  161. * @param {PathLike | PathOrFileDescriptor} path path
  162. * @param {EXPECTED_ANY} options options
  163. * @param {FileSystemCallback<EXPECTED_ANY>} callback callback
  164. * @returns {void}
  165. */
  166. class CacheBackend {
  167. /**
  168. * @param {number} duration max cache duration of items
  169. * @param {EXPECTED_FUNCTION | undefined} provider async method
  170. * @param {EXPECTED_FUNCTION | undefined} syncProvider sync method
  171. * @param {BaseFileSystem} providerContext call context for the provider methods
  172. */
  173. constructor(duration, provider, syncProvider, providerContext) {
  174. this._duration = duration;
  175. this._provider = provider;
  176. this._syncProvider = syncProvider;
  177. this._providerContext = providerContext;
  178. /** @type {Map<string, FileSystemCallback<EXPECTED_ANY>[]>} */
  179. this._activeAsyncOperations = new Map();
  180. /** @type {Map<string, { err: Error | null, result?: EXPECTED_ANY, level: Set<string> }>} */
  181. this._data = new Map();
  182. /** @type {Set<string>[]} */
  183. this._levels = [];
  184. for (let i = 0; i < 10; i++) this._levels.push(new Set());
  185. for (let i = 5000; i < duration; i += 500) this._levels.push(new Set());
  186. this._currentLevel = 0;
  187. this._tickInterval = Math.floor(duration / this._levels.length);
  188. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC} */
  189. this._mode = STORAGE_MODE_IDLE;
  190. /** @type {NodeJS.Timeout | undefined} */
  191. this._timeout = undefined;
  192. /** @type {number | undefined} */
  193. this._nextDecay = undefined;
  194. // eslint-disable-next-line no-warning-comments
  195. // @ts-ignore
  196. this.provide = provider ? this.provide.bind(this) : null;
  197. // eslint-disable-next-line no-warning-comments
  198. // @ts-ignore
  199. this.provideSync = syncProvider ? this.provideSync.bind(this) : null;
  200. }
  201. /**
  202. * @param {PathLike | PathOrFileDescriptor} path path
  203. * @param {EXPECTED_ANY} options options
  204. * @param {FileSystemCallback<EXPECTED_ANY>} callback callback
  205. * @returns {void}
  206. */
  207. provide(path, options, callback) {
  208. if (typeof options === "function") {
  209. callback = options;
  210. options = undefined;
  211. }
  212. if (
  213. typeof path !== "string" &&
  214. !Buffer.isBuffer(path) &&
  215. !(path instanceof URL) &&
  216. typeof path !== "number"
  217. ) {
  218. callback(new TypeError("path must be a string, Buffer, URL or number"));
  219. return;
  220. }
  221. const strPath = typeof path !== "string" ? path.toString() : path;
  222. if (options) {
  223. return /** @type {EXPECTED_FUNCTION} */ (this._provider).call(
  224. this._providerContext,
  225. path,
  226. options,
  227. callback,
  228. );
  229. }
  230. // When in sync mode we can move to async mode
  231. if (this._mode === STORAGE_MODE_SYNC) {
  232. this._enterAsyncMode();
  233. }
  234. // Check in cache
  235. const cacheEntry = this._data.get(strPath);
  236. if (cacheEntry !== undefined) {
  237. if (cacheEntry.err) return nextTick(callback, cacheEntry.err);
  238. return nextTick(callback, null, cacheEntry.result);
  239. }
  240. // Check if there is already the same operation running
  241. let callbacks = this._activeAsyncOperations.get(strPath);
  242. if (callbacks !== undefined) {
  243. callbacks.push(callback);
  244. return;
  245. }
  246. this._activeAsyncOperations.set(strPath, (callbacks = [callback]));
  247. // Run the operation
  248. /** @type {EXPECTED_FUNCTION} */
  249. (this._provider).call(
  250. this._providerContext,
  251. path,
  252. /**
  253. * @param {Error | null} err error
  254. * @param {EXPECTED_ANY=} result result
  255. */
  256. (err, result) => {
  257. this._activeAsyncOperations.delete(strPath);
  258. this._storeResult(strPath, err, result);
  259. // Enter async mode if not yet done
  260. this._enterAsyncMode();
  261. runCallbacks(
  262. /** @type {FileSystemCallback<EXPECTED_ANY>[]} */ (callbacks),
  263. err,
  264. result,
  265. );
  266. },
  267. );
  268. }
  269. /**
  270. * @param {PathLike | PathOrFileDescriptor} path path
  271. * @param {EXPECTED_ANY} options options
  272. * @returns {EXPECTED_ANY} result
  273. */
  274. provideSync(path, options) {
  275. if (
  276. typeof path !== "string" &&
  277. !Buffer.isBuffer(path) &&
  278. !(path instanceof URL) &&
  279. typeof path !== "number"
  280. ) {
  281. throw new TypeError("path must be a string");
  282. }
  283. const strPath = typeof path !== "string" ? path.toString() : path;
  284. if (options) {
  285. return /** @type {EXPECTED_FUNCTION} */ (this._syncProvider).call(
  286. this._providerContext,
  287. path,
  288. options,
  289. );
  290. }
  291. // In sync mode we may have to decay some cache items
  292. if (this._mode === STORAGE_MODE_SYNC) {
  293. this._runDecays();
  294. }
  295. // Check in cache
  296. const cacheEntry = this._data.get(strPath);
  297. if (cacheEntry !== undefined) {
  298. if (cacheEntry.err) throw cacheEntry.err;
  299. return cacheEntry.result;
  300. }
  301. // Get all active async operations
  302. // This sync operation will also complete them
  303. const callbacks = this._activeAsyncOperations.get(strPath);
  304. this._activeAsyncOperations.delete(strPath);
  305. // Run the operation
  306. // When in idle mode, we will enter sync mode
  307. let result;
  308. try {
  309. result = /** @type {EXPECTED_FUNCTION} */ (this._syncProvider).call(
  310. this._providerContext,
  311. path,
  312. );
  313. } catch (err) {
  314. this._storeResult(strPath, /** @type {Error} */ (err), undefined);
  315. this._enterSyncModeWhenIdle();
  316. if (callbacks) {
  317. runCallbacks(callbacks, /** @type {Error} */ (err), undefined);
  318. }
  319. throw err;
  320. }
  321. this._storeResult(strPath, null, result);
  322. this._enterSyncModeWhenIdle();
  323. if (callbacks) {
  324. runCallbacks(callbacks, null, result);
  325. }
  326. return result;
  327. }
  328. /**
  329. * @param {(string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>)=} what what to purge
  330. */
  331. purge(what) {
  332. if (!what) {
  333. if (this._mode !== STORAGE_MODE_IDLE) {
  334. this._data.clear();
  335. for (const level of this._levels) {
  336. level.clear();
  337. }
  338. this._enterIdleMode();
  339. }
  340. } else if (
  341. typeof what === "string" ||
  342. Buffer.isBuffer(what) ||
  343. what instanceof URL ||
  344. typeof what === "number"
  345. ) {
  346. const strWhat = typeof what !== "string" ? what.toString() : what;
  347. for (const [key, data] of this._data) {
  348. if (key.startsWith(strWhat)) {
  349. this._data.delete(key);
  350. data.level.delete(key);
  351. }
  352. }
  353. if (this._data.size === 0) {
  354. this._enterIdleMode();
  355. }
  356. } else {
  357. for (const [key, data] of this._data) {
  358. for (const item of what) {
  359. const strItem = typeof item !== "string" ? item.toString() : item;
  360. if (key.startsWith(strItem)) {
  361. this._data.delete(key);
  362. data.level.delete(key);
  363. break;
  364. }
  365. }
  366. }
  367. if (this._data.size === 0) {
  368. this._enterIdleMode();
  369. }
  370. }
  371. }
  372. /**
  373. * @param {(string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>)=} what what to purge
  374. */
  375. purgeParent(what) {
  376. if (!what) {
  377. this.purge();
  378. } else if (
  379. typeof what === "string" ||
  380. Buffer.isBuffer(what) ||
  381. what instanceof URL ||
  382. typeof what === "number"
  383. ) {
  384. const strWhat = typeof what !== "string" ? what.toString() : what;
  385. this.purge(dirname(strWhat));
  386. } else {
  387. const set = new Set();
  388. for (const item of what) {
  389. const strItem = typeof item !== "string" ? item.toString() : item;
  390. set.add(dirname(strItem));
  391. }
  392. this.purge(set);
  393. }
  394. }
  395. /**
  396. * @param {string} path path
  397. * @param {Error | null} err error
  398. * @param {EXPECTED_ANY} result result
  399. */
  400. _storeResult(path, err, result) {
  401. if (this._data.has(path)) return;
  402. const level = this._levels[this._currentLevel];
  403. this._data.set(path, { err, result, level });
  404. level.add(path);
  405. }
  406. _decayLevel() {
  407. const nextLevel = (this._currentLevel + 1) % this._levels.length;
  408. const decay = this._levels[nextLevel];
  409. this._currentLevel = nextLevel;
  410. for (const item of decay) {
  411. this._data.delete(item);
  412. }
  413. decay.clear();
  414. if (this._data.size === 0) {
  415. this._enterIdleMode();
  416. } else {
  417. /** @type {number} */
  418. (this._nextDecay) += this._tickInterval;
  419. }
  420. }
  421. _runDecays() {
  422. while (
  423. /** @type {number} */ (this._nextDecay) <= Date.now() &&
  424. this._mode !== STORAGE_MODE_IDLE
  425. ) {
  426. this._decayLevel();
  427. }
  428. }
  429. _enterAsyncMode() {
  430. let timeout = 0;
  431. switch (this._mode) {
  432. case STORAGE_MODE_ASYNC:
  433. return;
  434. case STORAGE_MODE_IDLE:
  435. this._nextDecay = Date.now() + this._tickInterval;
  436. timeout = this._tickInterval;
  437. break;
  438. case STORAGE_MODE_SYNC:
  439. this._runDecays();
  440. // _runDecays may change the mode
  441. if (
  442. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC} */
  443. (this._mode) === STORAGE_MODE_IDLE
  444. ) {
  445. return;
  446. }
  447. timeout = Math.max(
  448. 0,
  449. /** @type {number} */ (this._nextDecay) - Date.now(),
  450. );
  451. break;
  452. }
  453. this._mode = STORAGE_MODE_ASYNC;
  454. const ref = setTimeout(() => {
  455. this._mode = STORAGE_MODE_SYNC;
  456. this._runDecays();
  457. }, timeout);
  458. if (ref.unref) ref.unref();
  459. this._timeout = ref;
  460. }
  461. _enterSyncModeWhenIdle() {
  462. if (this._mode === STORAGE_MODE_IDLE) {
  463. this._mode = STORAGE_MODE_SYNC;
  464. this._nextDecay = Date.now() + this._tickInterval;
  465. }
  466. }
  467. _enterIdleMode() {
  468. this._mode = STORAGE_MODE_IDLE;
  469. this._nextDecay = undefined;
  470. if (this._timeout) clearTimeout(this._timeout);
  471. }
  472. }
  473. /**
  474. * @template {EXPECTED_FUNCTION} Provider
  475. * @template {EXPECTED_FUNCTION} AsyncProvider
  476. * @template FileSystem
  477. * @param {number} duration duration in ms files are cached
  478. * @param {Provider | undefined} provider provider
  479. * @param {AsyncProvider | undefined} syncProvider sync provider
  480. * @param {BaseFileSystem} providerContext provider context
  481. * @returns {OperationMergerBackend | CacheBackend} backend
  482. */
  483. const createBackend = (duration, provider, syncProvider, providerContext) => {
  484. if (duration > 0) {
  485. return new CacheBackend(duration, provider, syncProvider, providerContext);
  486. }
  487. return new OperationMergerBackend(provider, syncProvider, providerContext);
  488. };
  489. module.exports = class CachedInputFileSystem {
  490. /**
  491. * @param {BaseFileSystem} fileSystem file system
  492. * @param {number} duration duration in ms files are cached
  493. */
  494. constructor(fileSystem, duration) {
  495. this.fileSystem = fileSystem;
  496. this._lstatBackend = createBackend(
  497. duration,
  498. this.fileSystem.lstat,
  499. this.fileSystem.lstatSync,
  500. this.fileSystem,
  501. );
  502. const lstat = this._lstatBackend.provide;
  503. this.lstat = /** @type {FileSystem["lstat"]} */ (lstat);
  504. const lstatSync = this._lstatBackend.provideSync;
  505. this.lstatSync = /** @type {SyncFileSystem["lstatSync"]} */ (lstatSync);
  506. this._statBackend = createBackend(
  507. duration,
  508. this.fileSystem.stat,
  509. this.fileSystem.statSync,
  510. this.fileSystem,
  511. );
  512. const stat = this._statBackend.provide;
  513. this.stat = /** @type {FileSystem["stat"]} */ (stat);
  514. const statSync = this._statBackend.provideSync;
  515. this.statSync = /** @type {SyncFileSystem["statSync"]} */ (statSync);
  516. this._readdirBackend = createBackend(
  517. duration,
  518. this.fileSystem.readdir,
  519. this.fileSystem.readdirSync,
  520. this.fileSystem,
  521. );
  522. const readdir = this._readdirBackend.provide;
  523. this.readdir = /** @type {FileSystem["readdir"]} */ (readdir);
  524. const readdirSync = this._readdirBackend.provideSync;
  525. this.readdirSync = /** @type {SyncFileSystem["readdirSync"]} */ (
  526. readdirSync
  527. );
  528. this._readFileBackend = createBackend(
  529. duration,
  530. this.fileSystem.readFile,
  531. this.fileSystem.readFileSync,
  532. this.fileSystem,
  533. );
  534. const readFile = this._readFileBackend.provide;
  535. this.readFile = /** @type {FileSystem["readFile"]} */ (readFile);
  536. const readFileSync = this._readFileBackend.provideSync;
  537. this.readFileSync = /** @type {SyncFileSystem["readFileSync"]} */ (
  538. readFileSync
  539. );
  540. this._readJsonBackend = createBackend(
  541. duration,
  542. // prettier-ignore
  543. this.fileSystem.readJson ||
  544. (this.readFile &&
  545. (
  546. /**
  547. * @param {string} path path
  548. * @param {FileSystemCallback<EXPECTED_ANY>} callback callback
  549. */
  550. (path, callback) => {
  551. this.readFile(path, (err, buffer) => {
  552. if (err) return callback(err);
  553. if (!buffer || buffer.length === 0)
  554. {return callback(new Error("No file content"));}
  555. let data;
  556. try {
  557. data = JSON.parse(buffer.toString("utf8"));
  558. } catch (err_) {
  559. return callback(/** @type {Error} */ (err_));
  560. }
  561. callback(null, data);
  562. });
  563. })
  564. ),
  565. // prettier-ignore
  566. this.fileSystem.readJsonSync ||
  567. (this.readFileSync &&
  568. (
  569. /**
  570. * @param {string} path path
  571. * @returns {EXPECTED_ANY} result
  572. */
  573. (path) => {
  574. const buffer = this.readFileSync(path);
  575. const data = JSON.parse(buffer.toString("utf8"));
  576. return data;
  577. }
  578. )),
  579. this.fileSystem,
  580. );
  581. const readJson = this._readJsonBackend.provide;
  582. this.readJson = /** @type {FileSystem["readJson"]} */ (readJson);
  583. const readJsonSync = this._readJsonBackend.provideSync;
  584. this.readJsonSync = /** @type {SyncFileSystem["readJsonSync"]} */ (
  585. readJsonSync
  586. );
  587. this._readlinkBackend = createBackend(
  588. duration,
  589. this.fileSystem.readlink,
  590. this.fileSystem.readlinkSync,
  591. this.fileSystem,
  592. );
  593. const readlink = this._readlinkBackend.provide;
  594. this.readlink = /** @type {FileSystem["readlink"]} */ (readlink);
  595. const readlinkSync = this._readlinkBackend.provideSync;
  596. this.readlinkSync = /** @type {SyncFileSystem["readlinkSync"]} */ (
  597. readlinkSync
  598. );
  599. this._realpathBackend = createBackend(
  600. duration,
  601. this.fileSystem.realpath,
  602. this.fileSystem.realpathSync,
  603. this.fileSystem,
  604. );
  605. const realpath = this._realpathBackend.provide;
  606. this.realpath = /** @type {FileSystem["realpath"]} */ (realpath);
  607. const realpathSync = this._realpathBackend.provideSync;
  608. this.realpathSync = /** @type {SyncFileSystem["realpathSync"]} */ (
  609. realpathSync
  610. );
  611. }
  612. /**
  613. * @param {(string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>)=} what what to purge
  614. */
  615. purge(what) {
  616. this._statBackend.purge(what);
  617. this._lstatBackend.purge(what);
  618. this._readdirBackend.purgeParent(what);
  619. this._readFileBackend.purge(what);
  620. this._readlinkBackend.purge(what);
  621. this._readJsonBackend.purge(what);
  622. this._realpathBackend.purge(what);
  623. }
  624. };