Use escape sequences to denote special characters

.changeset/smooth-years-speak.md

@@ -0,0 +1,5 @@
+---
+'@sveltejs/kit': patch
+---
+
+[breaking] use hex/unicode escape sequences for encoding special characters in route directory names

documentation/docs/30-advanced/10-advanced-routing.md

@@ -123,27 +123,42 @@ src/routes/[...catchall]/+page.svelte
 
 ### Encoding
 
-Directory names are URI-decoded, meaning that (for example) a directory like `%40[username]` would match characters beginning with `@`:
+Some characters can't be used on the filesystem — `/` on Linux and Mac, `\ / : * ? " < > |` on Windows. The `#` and `%` characters have special meaning in URLs, and the `[ ] ( )` characters have special meaning to SvelteKit, so these also can't be used directly as part of your route.
+
+To use these characters in your routes, you can use hexadecimal escape sequences, which have the format `[x+nn]` where `nn` is a hexadecimal character code:
+
+- `\` — `[x+5c]`
+- `/` — `[x+2f]`
+- `:` — `[x+3a]`
+- `*` — `[x+2a]`
+- `?` — `[x+3f]`
+- `"` — `[x+22]`
+- `<` — `[x+3c]`
+- `>` — `[x+3e]`
+- `|` — `[x+7c]`
+- `#` — `[x+23]`
+- `%` — `[x+25]`
+- `[` — `[x+5b]`
+- `]` — `[x+5d]`
+- `(` — `[x+28]`
+- `)` — `[x+29]`
+
+For example, to create a `/smileys/:-)` route, you would create a `src/routes/smileys/[x+3a]-[x+29]/+page.svelte` file.
+
+You can determine the hexadecimal code for a character with JavaScript:
 
 ```js
-// @filename: ambient.d.ts
-declare global {
-	const assert: {
-		equal: (a: any, b: any) => boolean;
-	};
-}
+':'.charCodeAt(0).toString(16); // '3a', hence '[x+3a]'
+```
 
-export {};
+You can also use Unicode escape sequences. Generally you won't need to as you can use the unencoded character directly, but if — for some reason — you can't have a filename with an emoji in it, for example, then you can use the escaped characters. In other words, these are equivalent:
 
-// @filename: index.js
-// ---cut---
-assert.equal(
-	decodeURIComponent('%40[username]'),
-	'@[username]'
-);
+```
+src/routes/[u+d83e][u+dd2a]/+page.svelte
+src/routes/🤪/+page.svelte
 ```
 
-To express a `%` character, use `%25`, otherwise the result will be malformed.
+The format for a Unicode escape sequence is `[u+nnnn]` where `nnnn` is a valid value between `0000` and `10ffff`. (Unlike JavaScript string escaping, there's no need to use surrogate pairs to represent code points above `ffff`.) To learn more about Unicode encodings, consult [Programming with Unicode](https://unicodebook.readthedocs.io/unicode_encodings.html).
 
 ### Advanced layouts
 

packages/kit/src/core/sync/create_manifest_data/index.js

@@ -102,14 +102,45 @@ function create_routes_and_nodes(cwd, config, fallback) {
 		 * @param {import('types').RouteData | null} parent
 		 */
 		const walk = (depth, id, segment, parent) => {
-			if (/\]\[/.test(id)) {
+			const unescaped = id.replace(/\[([ux])\+([^\]]+)\]/gi, (match, type, code) => {
+				if (match !== match.toLowerCase()) {
+					throw new Error(`Character escape sequence in ${id} must be lowercase`);
+				}
+
+				if (!/[0-9a-f]+/.test(code)) {
+					throw new Error(`Invalid character escape sequence in ${id}`);
+				}
+
+				if (type === 'x') {
+					if (code.length !== 2) {
+						throw new Error(`Hexadecimal escape sequence in ${id} must be two characters`);
+					}
+
+					return String.fromCharCode(parseInt(code, 16));
+				} else {
+					if (code.length < 4 || code.length > 6) {
+						throw new Error(
+							`Unicode escape sequence in ${id} must be between four and six characters`
+						);
+					}
+
+					return String.fromCharCode(parseInt(code, 16));
+				}
+			});
+
+			if (/\]\[/.test(unescaped)) {
 				throw new Error(`Invalid route ${id} — parameters must be separated`);
 			}
 
 			if (count_occurrences('[', id) !== count_occurrences(']', id)) {
 				throw new Error(`Invalid route ${id} — brackets are unbalanced`);
 			}
 
+			if (/#/.test(segment)) {
+				// Vite will barf on files with # in them
+				throw new Error(`Route ${id} should be renamed to ${id.replace(/#/g, '[x+23]')}`);
+			}
+
 			if (/\[\.\.\.\w+\]\/\[\[/.test(id)) {
 				throw new Error(
 					`Invalid route ${id} — an [[optional]] route segment cannot follow a [...rest] route segment`
@@ -464,6 +495,10 @@ function normalize_route_id(id) {
 			// remove groups
 			.replace(/(?<=^|\/)\(.+?\)(?=$|\/)/g, '')
 
+			.replace(/\[[ux]\+([0-9a-f]+)\]/g, (_, x) =>
+				String.fromCharCode(parseInt(x, 16)).replace(/\//g, '%2f')
+			)
+
 			// replace `[param]` with `<*>`, `[param=x]` with `<x>`, and `[[param]]` with `<?*>`
 			.replace(
 				/\[(?:(\[)|(\.\.\.))?.+?(=.+?)?\]\]?/g,

packages/kit/src/core/sync/create_manifest_data/index.spec.js

@@ -181,32 +181,24 @@ test('succeeds when routes does not exist', () => {
 	]);
 });
 
-// TODO some characters will need to be URL-encoded in the filename
 test('encodes invalid characters', () => {
 	const { nodes, routes } = create('samples/encoding');
 
-	// had to remove ? and " because windows
-
-	// const quote = 'samples/encoding/".svelte';
-	const hash = { component: 'samples/encoding/%23/+page.svelte' };
-	// const question_mark = 'samples/encoding/?.svelte';
+	const quote = { component: 'samples/encoding/[x+22]/+page.svelte' };
+	const hash = { component: 'samples/encoding/[x+23]/+page.svelte' };
+	const question_mark = { component: 'samples/encoding/[x+3f]/+page.svelte' };
 
 	assert.equal(nodes.map(simplify_node), [
 		default_layout,
 		default_error,
-		// quote,
-		hash
-		// question_mark
+		quote,
+		hash,
+		question_mark
 	]);
 
 	assert.equal(
-		routes.map((p) => p.pattern),
-		[
-			/^\/$/,
-			// /^\/%22\/?$/,
-			/^\/%23\/?$/
-			// /^\/%3F\/?$/
-		]
+		routes.map((p) => p.pattern.toString()),
+		[/^\/$/, /^\/%3[Ff]\/?$/, /^\/%23\/?$/, /^\/"\/?$/].map((pattern) => pattern.toString())
 	);
 });
 

packages/kit/src/utils/routing.js

@@ -1,6 +1,9 @@
 const param_pattern = /^(\[)?(\.\.\.)?(\w+)(?:=(\w+))?(\])?$/;
 
-/** @param {string} id */
+/**
+ * Creates the regex pattern, extracts parameter names, and generates types for a route
+ * @param {string} id
+ */
 export function parse_route_id(id) {
 	/** @type {string[]} */
 	const names = [];
@@ -21,17 +24,16 @@ export function parse_route_id(id) {
 			: new RegExp(
 					`^${get_route_segments(id)
 						.map((segment, i, segments) => {
-							const decoded_segment = decodeURIComponent(segment);
 							// special case — /[...rest]/ could contain zero segments
-							const rest_match = /^\[\.\.\.(\w+)(?:=(\w+))?\]$/.exec(decoded_segment);
+							const rest_match = /^\[\.\.\.(\w+)(?:=(\w+))?\]$/.exec(segment);
 							if (rest_match) {
 								names.push(rest_match[1]);
 								types.push(rest_match[2]);
 								optional.push(false);
 								return '(?:/(.*))?';
 							}
 							// special case — /[[optional]]/ could contain zero segments
-							const optional_match = /^\[\[(\w+)(?:=(\w+))?\]\]$/.exec(decoded_segment);
+							const optional_match = /^\[\[(\w+)(?:=(\w+))?\]\]$/.exec(segment);
 							if (optional_match) {
 								names.push(optional_match[1]);
 								types.push(optional_match[2]);
@@ -41,14 +43,29 @@ export function parse_route_id(id) {
 
 							const is_last = i === segments.length - 1;
 
-							if (!decoded_segment) {
+							if (!segment) {
 								return;
 							}
 
-							const parts = decoded_segment.split(/\[(.+?)\](?!\])/);
+							const parts = segment.split(/\[(.+?)\](?!\])/);
 							const result = parts
 								.map((content, i) => {
 									if (i % 2) {
+										if (content.startsWith('x+')) {
+											return escape(String.fromCharCode(parseInt(content.slice(2), 16)));
+										}
+
+										if (content.startsWith('u+')) {
+											return escape(
+												String.fromCharCode(
+													...content
+														.slice(2)
+														.split('-')
+														.map((code) => parseInt(code, 16))
+												)
+											);
+										}
+
 										const match = param_pattern.exec(content);
 										if (!match) {
 											throw new Error(
@@ -69,18 +86,7 @@ export function parse_route_id(id) {
 
 									if (is_last && content.includes('.')) add_trailing_slash = false;
 
-									return (
-										content // allow users to specify characters on the file system in an encoded manner
-											.normalize()
-											// '#', '/', and '?' can only appear in URL path segments in an encoded manner.
-											// They will not be touched by decodeURI so need to be encoded here, so
-											// that we can match against them.
-											// We skip '/' since you can't create a file with it on any OS
-											.replace(/#/g, '%23')
-											.replace(/\?/g, '%3F')
-											// escape characters that have special meaning in regex
-											.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-									); // TODO handle encoding
+									return escape(content);
 								})
 								.join('');
 
@@ -143,3 +149,20 @@ export function exec(match, { names, types, optional }, matchers) {
 
 	return params;
 }
+
+/** @param {string} str */
+function escape(str) {
+	return (
+		str
+			.normalize()
+			// escape [ and ] before escaping other characters, since they are used in the replacements
+			.replace(/[[\]]/g, '\\$&')
+			// replace %, /, ? and # with their encoded versions
+			.replace(/%/g, '%25')
+			.replace(/\//g, '%2[Ff]')
+			.replace(/\?/g, '%3[Ff]')
+			.replace(/#/g, '%23')
+			// escape characters that have special meaning in regex
+			.replace(/[.*+?^${}()|\\]/g, '\\$&')
+	);
+}

packages/kit/src/utils/routing.spec.js

@@ -58,20 +58,10 @@ const tests = {
 		names: ['id'],
 		types: ['uuid']
 	},
-	'/%23hash-encoded': {
-		pattern: /^\/%23hash-encoded\/?$/,
-		names: [],
-		types: []
-	},
-	'/%40at-encoded/[id]': {
-		pattern: /^\/@at-encoded\/([^/]+?)\/?$/,
+	'/@-symbol/[id]': {
+		pattern: /^\/@-symbol\/([^/]+?)\/?$/,
 		names: ['id'],
 		types: [undefined]
-	},
-	'/%255bdoubly-encoded': {
-		pattern: /^\/%5bdoubly-encoded\/?$/,
-		names: [],
-		types: []
 	}
 };
 

packages/kit/test/apps/basics/src/routes/encoded/+page.svelte

@@ -3,7 +3,6 @@
 <a href="/encoded/反应">反应</a>
 <a href="/encoded/redirect">Redirect</a>
 <a href="/encoded/@svelte">@svelte</a>
-<a href="/encoded/$SVLT">$SVLT</a>
 <a href="/encoded/test%2520me">test%20me</a>
 <a href="/encoded/test%252fme">test%2fme</a>
 <a href="/encoded/AC%2fDC">AC/DC</a>

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/+layout.svelte

@@ -0,0 +1,16 @@
+<script>
+	import { page } from '$app/stores';
+</script>
+
+<h1>{decodeURIComponent($page.url.pathname.split('/').pop())}</h1>
+<slot />
+
+<a href="/encoded/escape-sequences/:-)">:-)</a>
+<a href="/encoded/escape-sequences/%23">#</a>
+<a href="/encoded/escape-sequences/%2F">/</a>
+<a href="/encoded/escape-sequences/%3f">?</a>
+<a href="/encoded/escape-sequences/苗">苗</a>
+<a href="/encoded/escape-sequences/<">&lt;</a>
+<a href="/encoded/escape-sequences/1<2">1&lt;2</a>
+<a href="/encoded/escape-sequences/🤪">🤪</a>
+<a href="/encoded/escape-sequences/%25">%</a>

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[u+82d7]/+page.svelte

@@ -0,0 +1 @@
+苗

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[u+d83e][u+dd2a]/+page.svelte

@@ -0,0 +1 @@
+🤪

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[x+23]/+page.svelte

@@ -0,0 +1 @@
+#

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[x+25]/+page.svelte

@@ -0,0 +1 @@
+%

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[x+2f]/+page.svelte

@@ -0,0 +1 @@
+/

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[x+3a]-[x+29]/+page.svelte

@@ -0,0 +1 @@
+:-)

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[x+3c]/+page.svelte

@@ -0,0 +1 @@
+<

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[x+3f]/+page.svelte

@@ -0,0 +1 @@
+?

packages/kit/test/apps/basics/src/routes/encoded/escape-sequences/[x][x+3c][y]/+page.svelte

@@ -0,0 +1 @@
+1<2

packages/kit/test/apps/basics/test/test.js

@@ -323,17 +323,42 @@ test.describe('Encoded paths', () => {
 		});
 	});
 
-	test('allows %-encoded characters in directory names', async ({ page, clicknav }) => {
-		await page.goto('/encoded');
-		await clicknav('[href="/encoded/$SVLT"]');
-		expect(await page.textContent('h1')).toBe('$SVLT');
-	});
-
-	test('allows %-encoded characters in filenames', async ({ page, clicknav }) => {
+	test('allows non-ASCII character in parameterized route segment', async ({ page, clicknav }) => {
 		await page.goto('/encoded');
 		await clicknav('[href="/encoded/@svelte"]');
 		expect(await page.textContent('h1')).toBe('@svelte');
 	});
+
+	test('allows characters to be represented as escape sequences', async ({ page, clicknav }) => {
+		await page.goto('/encoded/escape-sequences');
+
+		await clicknav('[href="/encoded/escape-sequences/:-)"]');
+		expect(await page.textContent('h1')).toBe(':-)');
+
+		await clicknav('[href="/encoded/escape-sequences/%23"]');
+		expect(await page.textContent('h1')).toBe('#');
+
+		await clicknav('[href="/encoded/escape-sequences/%2F"]');
+		expect(await page.textContent('h1')).toBe('/');
+
+		await clicknav('[href="/encoded/escape-sequences/%3f"]');
+		expect(await page.textContent('h1')).toBe('?');
+
+		await clicknav('[href="/encoded/escape-sequences/%25"]');
+		expect(await page.textContent('h1')).toBe('%');
+
+		await clicknav('[href="/encoded/escape-sequences/<"]');
+		expect(await page.textContent('h1')).toBe('<');
+
+		await clicknav('[href="/encoded/escape-sequences/1<2"]');
+		expect(await page.textContent('h1')).toBe('1<2');
+
+		await clicknav('[href="/encoded/escape-sequences/苗"]');
+		expect(await page.textContent('h1')).toBe('苗');
+
+		await clicknav('[href="/encoded/escape-sequences/🤪"]');
+		expect(await page.textContent('h1')).toBe('🤪');
+	});
 });
 
 test.describe('$env', () => {