feat(assets): support remote images

.changeset/itchy-pants-grin.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/vercel': patch
+---
+
+Update image support to work with latest version of Astro

.changeset/sour-frogs-shout.md

@@ -0,0 +1,27 @@
+---
+'astro': patch
+---
+
+Added support for optimizing remote images from authorized sources when using `astro:assets`. This comes with two new parameters to specify which domains (`image.domains`) and host patterns (`image.remotePatterns`) are authorized for remote images.
+
+For example, the following configuration will only allow remote images from `astro.build` to be optimized:
+
+```ts
+// astro.config.mjs
+export default defineConfig({
+  image: {
+    domains: ["astro.build"],
+  }
+});
+```
+
+The following configuration will only allow remote images from HTTPS hosts:
+
+```ts
+// astro.config.mjs
+export default defineConfig({
+  image: {
+    remotePatterns: [{ protocol: "https" }],
+  }
+});
+```

packages/astro/package.json

@@ -147,6 +147,7 @@
     "github-slugger": "^2.0.0",
     "gray-matter": "^4.0.3",
     "html-escaper": "^3.0.3",
+    "http-cache-semantics": "^4.1.1",
     "js-yaml": "^4.1.0",
     "kleur": "^4.1.4",
     "magic-string": "^0.30.2",
@@ -186,6 +187,7 @@
     "@types/estree": "^0.0.51",
     "@types/hast": "^2.3.4",
     "@types/html-escaper": "^3.0.0",
+    "@types/http-cache-semantics": "^4.0.1",
     "@types/js-yaml": "^4.0.5",
     "@types/mime": "^2.0.3",
     "@types/mocha": "^9.1.1",

packages/astro/src/@types/astro.ts

@@ -13,6 +13,7 @@ import type { AddressInfo } from 'node:net';
 import type * as rollup from 'rollup';
 import type { TsConfigJson } from 'tsconfig-resolver';
 import type * as vite from 'vite';
+import type { RemotePattern } from '../assets/utils/remotePattern';
 import type { SerializedSSRManifest } from '../core/app/types';
 import type { PageBuildData } from '../core/build/types';
 import type { AstroConfigType } from '../core/config';
@@ -43,6 +44,7 @@ export type {
 	ImageQualityPreset,
 	ImageTransform,
 } from '../assets/types';
+export type { RemotePattern } from '../assets/utils/remotePattern';
 export type { SSRManifest } from '../core/app/types';
 export type { AstroCookies } from '../core/cookies';
 
@@ -366,10 +368,10 @@ export interface ViteUserConfig extends vite.UserConfig {
 	ssr?: vite.SSROptions;
 }
 
-export interface ImageServiceConfig {
+export interface ImageServiceConfig<T extends Record<string, any> = Record<string, any>> {
 	// eslint-disable-next-line @typescript-eslint/ban-types
 	entrypoint: 'astro/assets/services/sharp' | 'astro/assets/services/squoosh' | (string & {});
-	config?: Record<string, any>;
+	config?: T;
 }
 
 /**
@@ -1010,6 +1012,68 @@ export interface AstroUserConfig {
 		 * ```
 		 */
 		service: ImageServiceConfig;
+
+		/**
+		 * @docs
+		 * @name image.domains (Experimental)
+		 * @type {string[]}
+		 * @default `{domains: []}`
+		 * @version 2.10.10
+		 * @description
+		 * Defines a list of permitted image source domains for local image optimization. No other remote images will be optimized by Astro.
+		 *
+		 * This option requires an array of individual domain names as strings. Wildcards are not permitted. Instead, use [`image.remotePatterns`](#imageremotepatterns-experimental) to define a list of allowed source URL patterns.
+		 *
+		 * ```js
+		 * // astro.config.mjs
+		 * {
+		 *   image: {
+		 *     // Example: Allow remote image optimization from a single domain
+		 *     domains: ['astro.build'],
+		 *   },
+		 * }
+		 * ```
+		 */
+		domains?: string[];
+
+		/**
+		 * @docs
+		 * @name image.remotePatterns (Experimental)
+		 * @type {RemotePattern[]}
+		 * @default `{remotePatterns: []}`
+		 * @version 2.10.10
+		 * @description
+		 * Defines a list of permitted image source URL patterns for local image optimization.
+		 *
+		 * `remotePatterns` can be configured with four properties:
+		 * 1. protocol
+		 * 2. hostname
+		 * 3. port
+		 * 4. pathname
+		 *
+		 * ```js
+		 * {
+		 *   image: {
+		 *     // Example: allow processing all images from your aws s3 bucket
+		 *     remotePatterns: [{
+		 *       protocol: 'https',
+		 *       hostname: '**.amazonaws.com',
+		 *     }],
+		 *   },
+		 * }
+		 * ```
+		 *
+		 * You can use wildcards to define the permitted `hostname` and `pathname` values as described below. Otherwise, only the exact values provided will be configured:
+		 * `hostname`:
+		 *   - Start with '**.' to allow all subdomains ('endsWith').
+		 *   - Start with '*.' to allow only one level of subdomain.
+		 *
+		 * `pathname`:
+		 *   - End with '/**' to allow all sub-routes ('startsWith').
+		 *   - End with '/*' to allow only one level of sub-route.
+
+		 */
+		remotePatterns?: Partial<RemotePattern>[];
 	};
 
 	/**

packages/astro/src/assets/build/generate.ts

@@ -0,0 +1,174 @@
+import fs, { readFileSync } from 'node:fs';
+import { basename, join } from 'node:path/posix';
+import type { StaticBuildOptions } from '../../core/build/types.js';
+import { warn } from '../../core/logger/core.js';
+import { prependForwardSlash } from '../../core/path.js';
+import { isServerLikeOutput } from '../../prerender/utils.js';
+import { getConfiguredImageService, isESMImportedImage } from '../internal.js';
+import type { LocalImageService } from '../services/service.js';
+import type { ImageMetadata, ImageTransform } from '../types.js';
+import { loadRemoteImage, type RemoteCacheEntry } from './remote.js';
+
+interface GenerationDataUncached {
+	cached: false;
+	weight: {
+		before: number;
+		after: number;
+	};
+}
+
+interface GenerationDataCached {
+	cached: true;
+}
+
+type GenerationData = GenerationDataUncached | GenerationDataCached;
+
+export async function generateImage(
+	buildOpts: StaticBuildOptions,
+	options: ImageTransform,
+	filepath: string
+): Promise<GenerationData | undefined> {
+	let useCache = true;
+	const assetsCacheDir = new URL('assets/', buildOpts.settings.config.cacheDir);
+
+	// Ensure that the cache directory exists
+	try {
+		await fs.promises.mkdir(assetsCacheDir, { recursive: true });
+	} catch (err) {
+		warn(
+			buildOpts.logging,
+			'astro:assets',
+			`An error was encountered while creating the cache directory. Proceeding without caching. Error: ${err}`
+		);
+		useCache = false;
+	}
+
+	let serverRoot: URL, clientRoot: URL;
+	if (isServerLikeOutput(buildOpts.settings.config)) {
+		serverRoot = buildOpts.settings.config.build.server;
+		clientRoot = buildOpts.settings.config.build.client;
+	} else {
+		serverRoot = buildOpts.settings.config.outDir;
+		clientRoot = buildOpts.settings.config.outDir;
+	}
+
+	const isLocalImage = isESMImportedImage(options.src);
+
+	const finalFileURL = new URL('.' + filepath, clientRoot);
+	const finalFolderURL = new URL('./', finalFileURL);
+
+	// For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server
+	const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json');
+	const cachedFileURL = new URL(cacheFile, assetsCacheDir);
+
+	await fs.promises.mkdir(finalFolderURL, { recursive: true });
+
+	// Check if we have a cached entry first
+	try {
+		if (isLocalImage) {
+			await fs.promises.copyFile(cachedFileURL, finalFileURL);
+
+			return {
+				cached: true,
+			};
+		} else {
+			const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry;
+
+			// If the cache entry is not expired, use it
+			if (JSONData.expires < Date.now()) {
+				await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64'));
+
+				return {
+					cached: true,
+				};
+			}
+		}
+	} catch (e: any) {
+		if (e.code !== 'ENOENT') {
+			throw new Error(`An error was encountered while reading the cache file. Error: ${e}`);
+		}
+		// If the cache file doesn't exist, just move on, and we'll generate it
+	}
+
+	// The original filepath or URL from the image transform
+	const originalImagePath = isLocalImage
+		? (options.src as ImageMetadata).src
+		: (options.src as string);
+
+	let imageData;
+	let resultData: { data: Buffer | undefined; expires: number | undefined } = {
+		data: undefined,
+		expires: undefined,
+	};
+
+	// If the image is local, we can just read it directly, otherwise we need to download it
+	if (isLocalImage) {
+		imageData = await fs.promises.readFile(
+			new URL(
+				'.' +
+					prependForwardSlash(
+						join(buildOpts.settings.config.build.assets, basename(originalImagePath))
+					),
+				serverRoot
+			)
+		);
+	} else {
+		const remoteImage = await loadRemoteImage(originalImagePath);
+		resultData.expires = remoteImage.expires;
+		imageData = remoteImage.data;
+	}
+
+	const imageService = (await getConfiguredImageService()) as LocalImageService;
+	resultData.data = (
+		await imageService.transform(
+			imageData,
+			{ ...options, src: originalImagePath },
+			buildOpts.settings.config.image
+		)
+	).data;
+
+	try {
+		// Write the cache entry
+		if (useCache) {
+			if (isLocalImage) {
+				await fs.promises.writeFile(cachedFileURL, resultData.data);
+			} else {
+				await fs.promises.writeFile(
+					cachedFileURL,
+					JSON.stringify({
+						data: Buffer.from(resultData.data).toString('base64'),
+						expires: resultData.expires,
+					})
+				);
+			}
+		}
+	} catch (e) {
+		warn(
+			buildOpts.logging,
+			'astro:assets',
+			`An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}`
+		);
+	} finally {
+		// Write the final file
+		await fs.promises.writeFile(finalFileURL, resultData.data);
+	}
+
+	return {
+		cached: false,
+		weight: {
+			// Divide by 1024 to get size in kilobytes
+			before: Math.trunc(imageData.byteLength / 1024),
+			after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024),
+		},
+	};
+}
+
+export function getStaticImageList(): Iterable<
+	[string, { path: string; options: ImageTransform }]
+> {
+	if (!globalThis?.astroAsset?.staticImages) {
+		return [];
+	}
+
+	return globalThis.astroAsset.staticImages?.entries();
+}

packages/astro/src/assets/build/remote.ts

@@ -0,0 +1,48 @@
+import CachePolicy from 'http-cache-semantics';
+
+export type RemoteCacheEntry = { data: string; expires: number };
+
+export async function loadRemoteImage(src: string) {
+	const req = new Request(src);
+	const res = await fetch(req);
+
+	if (!res.ok) {
+		throw new Error(
+			`Failed to load remote image ${src}. The request did not return a 200 OK response. (received ${res.status}))`
+		);
+	}
+
+	// calculate an expiration date based on the response's TTL
+	const policy = new CachePolicy(webToCachePolicyRequest(req), webToCachePolicyResponse(res));
+	const expires = policy.storable() ? policy.timeToLive() : 0;
+
+	return {
+		data: Buffer.from(await res.arrayBuffer()),
+		expires: Date.now() + expires,
+	};
+}
+
+function webToCachePolicyRequest({ url, method, headers: _headers }: Request): CachePolicy.Request {
+	let headers: CachePolicy.Headers = {};
+	// Be defensive here due to a cookie header bug in node@18.14.1 + undici
+	try {
+		headers = Object.fromEntries(_headers.entries());
+	} catch {}
+	return {
+		method,
+		url,
+		headers,
+	};
+}
+
+function webToCachePolicyResponse({ status, headers: _headers }: Response): CachePolicy.Response {
+	let headers: CachePolicy.Headers = {};
+	// Be defensive here due to a cookie header bug in node@18.14.1 + undici
+	try {
+		headers = Object.fromEntries(_headers.entries());
+	} catch {}
+	return {
+		status,
+		headers,
+	};
+}