RealContentHashPlugin.js 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const { SyncBailHook } = require("tapable");
  7. const { CachedSource, CompatSource, RawSource } = require("webpack-sources");
  8. const Compilation = require("../Compilation");
  9. const WebpackError = require("../WebpackError");
  10. const { compareSelect, compareStrings } = require("../util/comparators");
  11. const createHash = require("../util/createHash");
  12. /** @typedef {import("webpack-sources").Source} Source */
  13. /** @typedef {import("../Cache").Etag} Etag */
  14. /** @typedef {import("../Compilation").AssetInfo} AssetInfo */
  15. /** @typedef {import("../Compiler")} Compiler */
  16. /** @typedef {typeof import("../util/Hash")} Hash */
  17. const EMPTY_SET = new Set();
  18. /**
  19. * @template T
  20. * @param {T | T[]} itemOrItems item or items
  21. * @param {Set<T>} list list
  22. */
  23. const addToList = (itemOrItems, list) => {
  24. if (Array.isArray(itemOrItems)) {
  25. for (const item of itemOrItems) {
  26. list.add(item);
  27. }
  28. } else if (itemOrItems) {
  29. list.add(itemOrItems);
  30. }
  31. };
  32. /**
  33. * @template T
  34. * @param {T[]} input list
  35. * @param {(item: T) => Buffer} fn map function
  36. * @returns {Buffer[]} buffers without duplicates
  37. */
  38. const mapAndDeduplicateBuffers = (input, fn) => {
  39. // Buffer.equals compares size first so this should be efficient enough
  40. // If it becomes a performance problem we can use a map and group by size
  41. // instead of looping over all assets.
  42. const result = [];
  43. outer: for (const value of input) {
  44. const buf = fn(value);
  45. for (const other of result) {
  46. if (buf.equals(other)) continue outer;
  47. }
  48. result.push(buf);
  49. }
  50. return result;
  51. };
  52. /**
  53. * Escapes regular expression metacharacters
  54. * @param {string} str String to quote
  55. * @returns {string} Escaped string
  56. */
  57. const quoteMeta = str => str.replace(/[-[\]\\/{}()*+?.^$|]/g, "\\$&");
  58. const cachedSourceMap = new WeakMap();
  59. /**
  60. * @param {Source} source source
  61. * @returns {CachedSource} cached source
  62. */
  63. const toCachedSource = source => {
  64. if (source instanceof CachedSource) {
  65. return source;
  66. }
  67. const entry = cachedSourceMap.get(source);
  68. if (entry !== undefined) return entry;
  69. const newSource = new CachedSource(CompatSource.from(source));
  70. cachedSourceMap.set(source, newSource);
  71. return newSource;
  72. };
  73. /** @typedef {Set<string>} OwnHashes */
  74. /** @typedef {Set<string>} ReferencedHashes */
  75. /** @typedef {Set<string>} Hashes */
  76. /**
  77. * @typedef {object} AssetInfoForRealContentHash
  78. * @property {string} name
  79. * @property {AssetInfo} info
  80. * @property {Source} source
  81. * @property {RawSource | undefined} newSource
  82. * @property {RawSource | undefined} newSourceWithoutOwn
  83. * @property {string} content
  84. * @property {OwnHashes | undefined} ownHashes
  85. * @property {Promise<void> | undefined} contentComputePromise
  86. * @property {Promise<void> | undefined} contentComputeWithoutOwnPromise
  87. * @property {ReferencedHashes | undefined} referencedHashes
  88. * @property {Hashes} hashes
  89. */
  90. /**
  91. * @typedef {object} CompilationHooks
  92. * @property {SyncBailHook<[Buffer[], string], string | void>} updateHash
  93. */
  94. /** @type {WeakMap<Compilation, CompilationHooks>} */
  95. const compilationHooksMap = new WeakMap();
  96. /**
  97. * @typedef {object} RealContentHashPluginOptions
  98. * @property {string | Hash} hashFunction the hash function to use
  99. * @property {string=} hashDigest the hash digest to use
  100. */
  101. const PLUGIN_NAME = "RealContentHashPlugin";
  102. class RealContentHashPlugin {
  103. /**
  104. * @param {Compilation} compilation the compilation
  105. * @returns {CompilationHooks} the attached hooks
  106. */
  107. static getCompilationHooks(compilation) {
  108. if (!(compilation instanceof Compilation)) {
  109. throw new TypeError(
  110. "The 'compilation' argument must be an instance of Compilation"
  111. );
  112. }
  113. let hooks = compilationHooksMap.get(compilation);
  114. if (hooks === undefined) {
  115. hooks = {
  116. updateHash: new SyncBailHook(["content", "oldHash"])
  117. };
  118. compilationHooksMap.set(compilation, hooks);
  119. }
  120. return hooks;
  121. }
  122. /**
  123. * @param {RealContentHashPluginOptions} options options
  124. */
  125. constructor({ hashFunction, hashDigest }) {
  126. this._hashFunction = hashFunction;
  127. this._hashDigest = hashDigest;
  128. }
  129. /**
  130. * Apply the plugin
  131. * @param {Compiler} compiler the compiler instance
  132. * @returns {void}
  133. */
  134. apply(compiler) {
  135. compiler.hooks.compilation.tap(PLUGIN_NAME, compilation => {
  136. const cacheAnalyse = compilation.getCache(
  137. "RealContentHashPlugin|analyse"
  138. );
  139. const cacheGenerate = compilation.getCache(
  140. "RealContentHashPlugin|generate"
  141. );
  142. const hooks = RealContentHashPlugin.getCompilationHooks(compilation);
  143. compilation.hooks.processAssets.tapPromise(
  144. {
  145. name: PLUGIN_NAME,
  146. stage: Compilation.PROCESS_ASSETS_STAGE_OPTIMIZE_HASH
  147. },
  148. async () => {
  149. const assets = compilation.getAssets();
  150. /** @type {AssetInfoForRealContentHash[]} */
  151. const assetsWithInfo = [];
  152. /** @type {Map<string, [AssetInfoForRealContentHash]>} */
  153. const hashToAssets = new Map();
  154. for (const { source, info, name } of assets) {
  155. const cachedSource = toCachedSource(source);
  156. const content = /** @type {string} */ (cachedSource.source());
  157. /** @type {Hashes} */
  158. const hashes = new Set();
  159. addToList(info.contenthash, hashes);
  160. /** @type {AssetInfoForRealContentHash} */
  161. const data = {
  162. name,
  163. info,
  164. source: cachedSource,
  165. newSource: undefined,
  166. newSourceWithoutOwn: undefined,
  167. content,
  168. ownHashes: undefined,
  169. contentComputePromise: undefined,
  170. contentComputeWithoutOwnPromise: undefined,
  171. referencedHashes: undefined,
  172. hashes
  173. };
  174. assetsWithInfo.push(data);
  175. for (const hash of hashes) {
  176. const list = hashToAssets.get(hash);
  177. if (list === undefined) {
  178. hashToAssets.set(hash, [data]);
  179. } else {
  180. list.push(data);
  181. }
  182. }
  183. }
  184. if (hashToAssets.size === 0) return;
  185. const hashRegExp = new RegExp(
  186. Array.from(hashToAssets.keys(), quoteMeta).join("|"),
  187. "g"
  188. );
  189. await Promise.all(
  190. assetsWithInfo.map(async asset => {
  191. const { name, source, content, hashes } = asset;
  192. if (Buffer.isBuffer(content)) {
  193. asset.referencedHashes = EMPTY_SET;
  194. asset.ownHashes = EMPTY_SET;
  195. return;
  196. }
  197. const etag = cacheAnalyse.mergeEtags(
  198. cacheAnalyse.getLazyHashedEtag(source),
  199. [...hashes].join("|")
  200. );
  201. [asset.referencedHashes, asset.ownHashes] =
  202. await cacheAnalyse.providePromise(name, etag, () => {
  203. const referencedHashes = new Set();
  204. const ownHashes = new Set();
  205. const inContent = content.match(hashRegExp);
  206. if (inContent) {
  207. for (const hash of inContent) {
  208. if (hashes.has(hash)) {
  209. ownHashes.add(hash);
  210. continue;
  211. }
  212. referencedHashes.add(hash);
  213. }
  214. }
  215. return [referencedHashes, ownHashes];
  216. });
  217. })
  218. );
  219. /**
  220. * @param {string} hash the hash
  221. * @returns {undefined | ReferencedHashes} the referenced hashes
  222. */
  223. const getDependencies = hash => {
  224. const assets = hashToAssets.get(hash);
  225. if (!assets) {
  226. const referencingAssets = assetsWithInfo.filter(asset =>
  227. /** @type {ReferencedHashes} */ (asset.referencedHashes).has(
  228. hash
  229. )
  230. );
  231. const err = new WebpackError(`RealContentHashPlugin
  232. Some kind of unexpected caching problem occurred.
  233. An asset was cached with a reference to another asset (${hash}) that's not in the compilation anymore.
  234. Either the asset was incorrectly cached, or the referenced asset should also be restored from cache.
  235. Referenced by:
  236. ${referencingAssets
  237. .map(a => {
  238. const match = new RegExp(`.{0,20}${quoteMeta(hash)}.{0,20}`).exec(
  239. a.content
  240. );
  241. return ` - ${a.name}: ...${match ? match[0] : "???"}...`;
  242. })
  243. .join("\n")}`);
  244. compilation.errors.push(err);
  245. return;
  246. }
  247. const hashes = new Set();
  248. for (const { referencedHashes, ownHashes } of assets) {
  249. if (!(/** @type {OwnHashes} */ (ownHashes).has(hash))) {
  250. for (const hash of /** @type {OwnHashes} */ (ownHashes)) {
  251. hashes.add(hash);
  252. }
  253. }
  254. for (const hash of /** @type {ReferencedHashes} */ (
  255. referencedHashes
  256. )) {
  257. hashes.add(hash);
  258. }
  259. }
  260. return hashes;
  261. };
  262. /**
  263. * @param {string} hash the hash
  264. * @returns {string} the hash info
  265. */
  266. const hashInfo = hash => {
  267. const assets = hashToAssets.get(hash);
  268. return `${hash} (${Array.from(
  269. /** @type {AssetInfoForRealContentHash[]} */ (assets),
  270. a => a.name
  271. )})`;
  272. };
  273. /** @type {Set<string>} */
  274. const hashesInOrder = new Set();
  275. for (const hash of hashToAssets.keys()) {
  276. /**
  277. * @param {string} hash the hash
  278. * @param {Set<string>} stack stack of hashes
  279. */
  280. const add = (hash, stack) => {
  281. const deps = getDependencies(hash);
  282. if (!deps) return;
  283. stack.add(hash);
  284. for (const dep of deps) {
  285. if (hashesInOrder.has(dep)) continue;
  286. if (stack.has(dep)) {
  287. throw new Error(
  288. `Circular hash dependency ${Array.from(
  289. stack,
  290. hashInfo
  291. ).join(" -> ")} -> ${hashInfo(dep)}`
  292. );
  293. }
  294. add(dep, stack);
  295. }
  296. hashesInOrder.add(hash);
  297. stack.delete(hash);
  298. };
  299. if (hashesInOrder.has(hash)) continue;
  300. add(hash, new Set());
  301. }
  302. /** @type {Map<string, string>} */
  303. const hashToNewHash = new Map();
  304. /**
  305. * @param {AssetInfoForRealContentHash} asset asset info
  306. * @returns {Etag} etag
  307. */
  308. const getEtag = asset =>
  309. cacheGenerate.mergeEtags(
  310. cacheGenerate.getLazyHashedEtag(asset.source),
  311. Array.from(
  312. /** @type {ReferencedHashes} */ (asset.referencedHashes),
  313. hash => hashToNewHash.get(hash)
  314. ).join("|")
  315. );
  316. /**
  317. * @param {AssetInfoForRealContentHash} asset asset info
  318. * @returns {Promise<void>}
  319. */
  320. const computeNewContent = asset => {
  321. if (asset.contentComputePromise) return asset.contentComputePromise;
  322. return (asset.contentComputePromise = (async () => {
  323. if (
  324. /** @type {OwnHashes} */ (asset.ownHashes).size > 0 ||
  325. [
  326. .../** @type {ReferencedHashes} */ (asset.referencedHashes)
  327. ].some(hash => hashToNewHash.get(hash) !== hash)
  328. ) {
  329. const identifier = asset.name;
  330. const etag = getEtag(asset);
  331. asset.newSource = await cacheGenerate.providePromise(
  332. identifier,
  333. etag,
  334. () => {
  335. const newContent = asset.content.replace(
  336. hashRegExp,
  337. hash => /** @type {string} */ (hashToNewHash.get(hash))
  338. );
  339. return new RawSource(newContent);
  340. }
  341. );
  342. }
  343. })());
  344. };
  345. /**
  346. * @param {AssetInfoForRealContentHash} asset asset info
  347. * @returns {Promise<void>}
  348. */
  349. const computeNewContentWithoutOwn = asset => {
  350. if (asset.contentComputeWithoutOwnPromise) {
  351. return asset.contentComputeWithoutOwnPromise;
  352. }
  353. return (asset.contentComputeWithoutOwnPromise = (async () => {
  354. if (
  355. /** @type {OwnHashes} */ (asset.ownHashes).size > 0 ||
  356. [
  357. .../** @type {ReferencedHashes} */ (asset.referencedHashes)
  358. ].some(hash => hashToNewHash.get(hash) !== hash)
  359. ) {
  360. const identifier = `${asset.name}|without-own`;
  361. const etag = getEtag(asset);
  362. asset.newSourceWithoutOwn = await cacheGenerate.providePromise(
  363. identifier,
  364. etag,
  365. () => {
  366. const newContent = asset.content.replace(
  367. hashRegExp,
  368. hash => {
  369. if (
  370. /** @type {OwnHashes} */
  371. (asset.ownHashes).has(hash)
  372. ) {
  373. return "";
  374. }
  375. return /** @type {string} */ (hashToNewHash.get(hash));
  376. }
  377. );
  378. return new RawSource(newContent);
  379. }
  380. );
  381. }
  382. })());
  383. };
  384. const comparator = compareSelect(a => a.name, compareStrings);
  385. for (const oldHash of hashesInOrder) {
  386. const assets =
  387. /** @type {AssetInfoForRealContentHash[]} */
  388. (hashToAssets.get(oldHash));
  389. assets.sort(comparator);
  390. await Promise.all(
  391. assets.map(asset =>
  392. /** @type {OwnHashes} */ (asset.ownHashes).has(oldHash)
  393. ? computeNewContentWithoutOwn(asset)
  394. : computeNewContent(asset)
  395. )
  396. );
  397. const assetsContent = mapAndDeduplicateBuffers(assets, asset => {
  398. if (/** @type {OwnHashes} */ (asset.ownHashes).has(oldHash)) {
  399. return asset.newSourceWithoutOwn
  400. ? asset.newSourceWithoutOwn.buffer()
  401. : asset.source.buffer();
  402. }
  403. return asset.newSource
  404. ? asset.newSource.buffer()
  405. : asset.source.buffer();
  406. });
  407. let newHash = hooks.updateHash.call(assetsContent, oldHash);
  408. if (!newHash) {
  409. const hash = createHash(this._hashFunction);
  410. if (compilation.outputOptions.hashSalt) {
  411. hash.update(compilation.outputOptions.hashSalt);
  412. }
  413. for (const content of assetsContent) {
  414. hash.update(content);
  415. }
  416. const digest = hash.digest(this._hashDigest);
  417. newHash = /** @type {string} */ (digest.slice(0, oldHash.length));
  418. }
  419. hashToNewHash.set(oldHash, newHash);
  420. }
  421. await Promise.all(
  422. assetsWithInfo.map(async asset => {
  423. await computeNewContent(asset);
  424. const newName = asset.name.replace(
  425. hashRegExp,
  426. hash => /** @type {string} */ (hashToNewHash.get(hash))
  427. );
  428. const infoUpdate = {};
  429. const hash = /** @type {string} */ (asset.info.contenthash);
  430. infoUpdate.contenthash = Array.isArray(hash)
  431. ? hash.map(
  432. hash => /** @type {string} */ (hashToNewHash.get(hash))
  433. )
  434. : /** @type {string} */ (hashToNewHash.get(hash));
  435. if (asset.newSource !== undefined) {
  436. compilation.updateAsset(
  437. asset.name,
  438. asset.newSource,
  439. infoUpdate
  440. );
  441. } else {
  442. compilation.updateAsset(asset.name, asset.source, infoUpdate);
  443. }
  444. if (asset.name !== newName) {
  445. compilation.renameAsset(asset.name, newName);
  446. }
  447. })
  448. );
  449. }
  450. );
  451. });
  452. }
  453. }
  454. module.exports = RealContentHashPlugin;