Skip to content

Commit

Permalink
refactor: use rollup hashing when emitting assets (#10878)
Browse files Browse the repository at this point in the history
  • Loading branch information
@patak-dev
patak-dev authored Nov 13, 2022
1 parent feb9b10 commit 78c77be
Show file tree
Hide file tree
Showing 7 changed files with 52 additions and 81 deletions.
82 changes: 18 additions & 64 deletions packages/vite/src/node/plugins/asset.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import { Buffer } from 'node:buffer'
import * as mrmime from 'mrmime'
import type {
NormalizedOutputOptions,
OutputAsset,
OutputOptions,
PluginContext,
PreRenderedAsset,
Expand All @@ -22,24 +21,19 @@ import type { ResolvedConfig } from '../config'
import { cleanUrl, getHash, joinUrlSegments, normalizePath } from '../utils'
import { FS_PREFIX } from '../constants'

export const assetUrlRE = /__VITE_ASSET__([a-z\d]{8})__(?:\$_(.*?)__)?/g

export const duplicateAssets = new WeakMap<
ResolvedConfig,
Map<string, OutputAsset>
>()
export const assetUrlRE = /__VITE_ASSET__([a-z\d]+)__(?:\$_(.*?)__)?/g

const rawRE = /(\?|&)raw(?:&|$)/
const urlRE = /(\?|&)url(?:&|$)/

const assetCache = new WeakMap<ResolvedConfig, Map<string, string>>()

const assetHashToFilenameMap = new WeakMap<
// chunk.name is the basename for the asset ignoring the directory structure
// For the manifest, we need to preserve the original file path
export const generatedAssets = new WeakMap<
ResolvedConfig,
Map<string, string>
Map<string, { originalName: string }>
>()
// save hashes of the files that has been emitted in build watch
const emittedHashMap = new WeakMap<ResolvedConfig, Set<string>>()

// add own dictionary entry by directly assigning mrmime
export function registerCustomMime(): void {
Expand Down Expand Up @@ -77,10 +71,8 @@ export function renderAssetUrlInJS(

while ((match = assetUrlRE.exec(code))) {
s ||= new MagicString(code)
const [full, hash, postfix = ''] = match
// some internal plugins may still need to emit chunks (e.g. worker) so
// fallback to this.getFileName for that. TODO: remove, not needed
const file = getAssetFilename(hash, config) || ctx.getFileName(hash)
const [full, referenceId, postfix = ''] = match
const file = ctx.getFileName(referenceId)
chunk.viteMetadata.importedAssets.add(cleanUrl(file))
const filename = file + postfix
const replacement = toOutputFilePathInJS(
Expand Down Expand Up @@ -127,18 +119,14 @@ export function renderAssetUrlInJS(
* Also supports loading plain strings with import text from './foo.txt?raw'
*/
export function assetPlugin(config: ResolvedConfig): Plugin {
// assetHashToFilenameMap initialization in buildStart causes getAssetFilename to return undefined
assetHashToFilenameMap.set(config, new Map())

registerCustomMime()

return {
name: 'vite:asset',

buildStart() {
assetCache.set(config, new Map())
emittedHashMap.set(config, new Set())
duplicateAssets.set(config, new Map())
generatedAssets.set(config, new Map())
},

resolveId(id) {
Expand Down Expand Up @@ -253,13 +241,6 @@ function fileToDevUrl(id: string, config: ResolvedConfig) {
return joinUrlSegments(base, rtn.replace(/^\//, ''))
}

export function getAssetFilename(
hash: string,
config: ResolvedConfig
): string | undefined {
return assetHashToFilenameMap.get(config)?.get(hash)
}

export function getPublicAssetFilename(
hash: string,
config: ResolvedConfig
Expand Down Expand Up @@ -458,47 +439,20 @@ async function fileToBuiltUrl(
url = `data:${mimeType};base64,${content.toString('base64')}`
} else {
// emit as asset
// rollup supports `import.meta.ROLLUP_FILE_URL_*`, but it generates code
// that uses runtime url sniffing and it can be verbose when targeting
// non-module format. It also fails to cascade the asset content change
// into the chunk's hash, so we have to do our own content hashing here.
// https://bundlers.tooling.report/hashing/asset-cascade/
// https://github.com/rollup/rollup/issues/3415
const map = assetHashToFilenameMap.get(config)!
const contentHash = getHash(content)
const { search, hash } = parseUrl(id)
const postfix = (search || '') + (hash || '')

const fileName = assetFileNamesToFileName(
resolveAssetFileNames(config),
file,
contentHash,
content
)
if (!map.has(contentHash)) {
map.set(contentHash, fileName)
}
const emittedSet = emittedHashMap.get(config)!
const duplicates = duplicateAssets.get(config)!
const name = normalizePath(path.relative(config.root, file))
if (!emittedSet.has(contentHash)) {
pluginContext.emitFile({
name,
fileName,
type: 'asset',
source: content
})
emittedSet.add(contentHash)
} else {
duplicates.set(name, {
name,
fileName: map.get(contentHash)!,
type: 'asset',
source: content
})
}
const referenceId = pluginContext.emitFile({
// Ignore directory structure for asset file names
name: path.basename(file),

This comment has been minimized.

Sign in to view

Copy link
@lhapaipai

lhapaipai Jan 6, 2023

Hi @patak-dev and thank you for your involvement in the vite projet.

I developed a plugin to allow communication between vite and the symfony framework, and I'm having trouble updating vite v4.

the symfony plugin https://github.com/lhapaipai/vite-plugin-symfony allows you to generate an entrypoints.json file similar to manifest.json but also containing a mapping for the assets: url relative to config.root -> generated url. I could do it easily with vite 3 with a hook on generateBundle because the name of the assets is relative to config.root. the generated entrypoints.json is similar to:

{
  "isProd": true,
  "viteServer": false,
  "entryPoints": {
    "welcome": {
      "js": [
        "/build/assets/welcome.e107d3d9.js"
      ],
      "css": [],
      "preload": [],
      "legacy": false
    }
  },
  "assets": {
    "assets/images/violin.jpg": "/build/assets/violin.a813db3a.jpg"
  },
  "legacy": false
}

but I have difficulties with this same hook under vite 4 because the name is only the basename. the same entrypoints.json generated with vite 4 is

{
  "isProd": true,
  "viteServer": false,
  "entryPoints": {
    "welcome": {
      "js": [
        "/build/assets/welcome-1e67239d.js"
      ],
      "css": [],
      "preload": [],
      "legacy": false
    }
  },
  "assets": {
    "violin.jpg": "/build/assets/violin-7cc8e2c1.jpg"
  },
  "legacy": false
}

not to mention the conflict between 2 assets of the same name, could you explain to me why you are using the basename and do you have another method to successfully perform this mapping ?

thank you very much for the time you could take to answer my message and have a nice day !
type: 'asset',
source: content
})

const originalName = normalizePath(path.relative(config.root, file))
generatedAssets.get(config)!.set(referenceId, { originalName })

url = `__VITE_ASSET__${contentHash}__${postfix ? `$_${postfix}__` : ``}` // TODO_BASE
url = `__VITE_ASSET__${referenceId}__${postfix ? `$_${postfix}__` : ``}` // TODO_BASE
}

cache.set(id, url)
Expand Down
8 changes: 5 additions & 3 deletions packages/vite/src/node/plugins/css.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ import {
assetUrlRE,
checkPublicFile,
fileToUrl,
getAssetFilename,
publicAssetUrlCache,
publicAssetUrlRE,
publicFileToBuiltUrl,
Expand Down Expand Up @@ -475,7 +474,10 @@ export function cssPostPlugin(config: ResolvedConfig): Plugin {
const publicAssetUrlMap = publicAssetUrlCache.get(config)!

// resolve asset URL placeholders to their built file URLs
function resolveAssetUrlsInCss(chunkCSS: string, cssAssetName: string) {
const resolveAssetUrlsInCss = (
chunkCSS: string,
cssAssetName: string
) => {
const encodedPublicUrls = encodePublicUrlsInCSS(config)

const relative = config.base === './' || config.base === ''
Expand All @@ -494,7 +496,7 @@ export function cssPostPlugin(config: ResolvedConfig): Plugin {

// replace asset url references with resolved url.
chunkCSS = chunkCSS.replace(assetUrlRE, (_, fileHash, postfix = '') => {
const filename = getAssetFilename(fileHash, config) + postfix
const filename = this.getFileName(fileHash) + postfix
chunk.viteMetadata.importedAssets.add(cleanUrl(filename))
return toOutputFilePathInCss(
filename,
Expand Down
5 changes: 1 addition & 4 deletions packages/vite/src/node/plugins/html.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import { toOutputFilePathInHtml } from '../build'
import {
assetUrlRE,
checkPublicFile,
getAssetFilename,
getPublicAssetFilename,
publicAssetUrlRE,
urlToBuiltUrl
Expand Down Expand Up @@ -794,9 +793,7 @@ export function buildHtmlPlugin(config: ResolvedConfig): Plugin {
})
// resolve asset url references
result = result.replace(assetUrlRE, (_, fileHash, postfix = '') => {
return (
toOutputAssetFilePath(getAssetFilename(fileHash, config)!) + postfix
)
return toOutputAssetFilePath(this.getFileName(fileHash)) + postfix
})

result = result.replace(publicAssetUrlRE, (_, fileHash) => {
Expand Down
32 changes: 25 additions & 7 deletions packages/vite/src/node/plugins/manifest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type { ResolvedConfig } from '..'
import type { Plugin } from '../plugin'
import { normalizePath } from '../utils'
import { cssEntryFilesCache } from './css'
import { duplicateAssets } from './asset'
import { generatedAssets } from './asset'

export type Manifest = Record<string, ManifestChunk>

Expand Down Expand Up @@ -101,10 +101,10 @@ export function manifestPlugin(config: ResolvedConfig): Plugin {
return manifestChunk
}

function createAsset(chunk: OutputAsset): ManifestChunk {
function createAsset(chunk: OutputAsset, src: string): ManifestChunk {
const manifestChunk: ManifestChunk = {
file: chunk.fileName,
src: chunk.name
src
}

if (cssEntryFiles.has(chunk.name!)) manifestChunk.isEntry = true
Expand All @@ -114,18 +114,36 @@ export function manifestPlugin(config: ResolvedConfig): Plugin {

const cssEntryFiles = cssEntryFilesCache.get(config)!

const fileNameToAssetMeta = new Map<string, { originalName: string }>()
generatedAssets.get(config)!.forEach((asset, referenceId) => {
const fileName = this.getFileName(referenceId)
fileNameToAssetMeta.set(fileName, asset)
})

const fileNameToAsset = new Map<string, ManifestChunk>()

for (const file in bundle) {
const chunk = bundle[file]
if (chunk.type === 'chunk') {
manifest[getChunkName(chunk)] = createChunk(chunk)
} else if (chunk.type === 'asset' && typeof chunk.name === 'string') {
manifest[chunk.name] = createAsset(chunk)
// Add every unique asset to the manifest, keyed by its original name
const src =
fileNameToAssetMeta.get(chunk.fileName)?.originalName ?? chunk.name
const asset = createAsset(chunk, src)
manifest[src] = asset
fileNameToAsset.set(chunk.fileName, asset)
}
}

duplicateAssets.get(config)!.forEach((asset) => {
const chunk = createAsset(asset)
manifest[asset.name!] = chunk
// Add duplicate assets to the manifest
fileNameToAssetMeta.forEach(({ originalName }, fileName) => {
if (!manifest[originalName]) {
const asset = fileNameToAsset.get(fileName)
if (asset) {
manifest[originalName] = asset
}
}
})

outputCount++
Expand Down
2 changes: 1 addition & 1 deletion playground/worker/__tests__/es/es-worker.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ test('normal', async () => {
)
await untilUpdated(
() => page.textContent('.asset-url'),
isBuild ? '/es/assets/vite.svg' : '/es/vite.svg',
isBuild ? '/es/assets/worker_asset.vite.svg' : '/es/vite.svg',
true
)
})
Expand Down
2 changes: 1 addition & 1 deletion playground/worker/__tests__/iife/iife-worker.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ test('normal', async () => {
)
await untilUpdated(
() => page.textContent('.asset-url'),
isBuild ? '/iife/assets/vite.svg' : '/iife/vite.svg',
isBuild ? '/iife/assets/worker_asset.vite.svg' : '/iife/vite.svg',
true
)
})
Expand Down
2 changes: 1 addition & 1 deletion playground/worker/__tests__/relative-base/relative-base-worker.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ test('normal', async () => {
)
await untilUpdated(
() => page.textContent('.asset-url'),
isBuild ? '/other-assets/vite' : '/vite.svg',
isBuild ? '/worker-assets/worker_asset.vite' : '/vite.svg',
true
)
})
Expand Down

0 comments on commit 78c77be

Please sign in to comment.