enhance: AVIF support (misskey-dev#9281)

* chore: Make image/avif browsersafe

* server side

* change FileInfoService

* ✌️

* avifはMastodonでは絶望的 see misskey-dev#9283

Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>
Co-authored-by: tamaina <tamaina@hotmail.co.jp>

packages/backend/src/const.ts

@@ -12,6 +12,7 @@ export const FILE_TYPE_BROWSERSAFE = [
  'image/gif',
  'image/jpeg',
  'image/webp',
+ 'image/avif',
  'image/apng',
  'image/bmp',
  'image/tiff',

packages/backend/src/misc/get-file-info.ts

@@ -63,7 +63,18 @@ export async function getFileInfo(path: string, opts: {
  let height: number | undefined;
  let orientation: number | undefined;
 
- if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/bmp', 'image/tiff', 'image/svg+xml', 'image/vnd.adobe.photoshop'].includes(type.mime)) {
+ if ([
+ 'image/png',
+ 'image/gif',
+ 'image/jpeg',
+ 'image/webp',
+ 'image/avif',
+ 'image/apng',
+ 'image/bmp',
+ 'image/tiff',
+ 'image/svg+xml',
+ 'image/vnd.adobe.photoshop',
+ ].includes(type.mime)) {
  const imageSize = await detectImageSize(path).catch(e => {
  warnings.push(`detectImageSize failed: ${e}`);
  return undefined;
@@ -90,7 +101,15 @@ export async function getFileInfo(path: string, opts: {
 
  let blurhash: string | undefined;
 
- if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml'].includes(type.mime)) {
+ if ([
+ 'image/jpeg',
+ 'image/gif',
+ 'image/png',
+ 'image/apng',
+ 'image/webp',
+ 'image/avif',
+ 'image/svg+xml',
+ ].includes(type.mime)) {
  blurhash = await getBlurhash(path).catch(e => {
  warnings.push(`getBlurhash failed: ${e}`);
  return undefined;
@@ -145,7 +164,11 @@ async function detectSensitivity(source: string, mime: string, sensitiveThreshol
  return [sensitive, porn];
  }
 
- if (['image/jpeg', 'image/png', 'image/webp'].includes(mime)) {
+ if ([
+ 'image/jpeg',
+ 'image/png',
+ 'image/webp',
+ ].includes(mime)) {
  const result = await detectSensitive(source);
  if (result) {
  [sensitive, porn] = judgePrediction(result);

packages/backend/src/misc/is-mime-image.ts

@@ -2,7 +2,7 @@ import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
 
 const dictionary = {
  'safe-file': FILE_TYPE_BROWSERSAFE,
- 'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/svg+xml'],
+ 'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'],
 };
 
 export const isMimeImage = (mime: string, type: keyof typeof dictionary): boolean => dictionary[type].includes(mime);

packages/backend/src/models/repositories/drive-file.ts

@@ -61,7 +61,7 @@ export const DriveFileRepository = db.getRepository(DriveFile).extend({
  }
  }
 
- const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/svg+xml'].includes(file.type);
+ const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/avif', 'image/svg+xml'].includes(file.type);
 
  return thumbnail ? (file.thumbnailUrl || (isImage ? (file.webpublicUrl || file.url) : null)) : (file.webpublicUrl || file.url);
  },

packages/backend/src/server/file/send-drive-file.ts

@@ -59,7 +59,7 @@ export default async function(ctx: Koa.Context) {
 
  const convertFile = async () => {
  if (isThumbnail) {
- if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(mime)) {
+ if (['image/jpeg', 'image/webp', 'image/png', 'image/avif', 'image/svg+xml'].includes(mime)) {
  return await convertToWebp(path, 498, 280);
  } else if (mime.startsWith('video/')) {
  return await GenerateVideoThumbnail(path);

packages/backend/src/services/drive/add-file.ts

@@ -48,6 +48,7 @@ async function save(file: DriveFile, path: string, name: string, type: string, h
  if (type === 'image/jpeg') ext = '.jpg';
  if (type === 'image/png') ext = '.png';
  if (type === 'image/webp') ext = '.webp';
+ if (type === 'image/avif') ext = '.avif';
  if (type === 'image/apng') ext = '.apng';
  if (type === 'image/vnd.mozilla.apng') ext = '.apng';
  }
@@ -171,7 +172,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
  }
  }
 
- if (!['image/jpeg', 'image/png', 'image/webp', 'image/svg+xml'].includes(type)) {
+ if (!['image/jpeg', 'image/png', 'image/webp', 'image/avif', 'image/svg+xml'].includes(type)) {
  logger.debug('web image and thumbnail not created (not an required file)');
  return {
  webpublic: null,
@@ -196,7 +197,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
  }
 
  satisfyWebpublic = !!(
- type !== 'image/svg+xml' && type !== 'image/webp' &&
+ type !== 'image/svg+xml' && type !== 'image/webp' && type !== 'image/avif' &&
  !(metadata.exif || metadata.iptc || metadata.xmp || metadata.tifftagPhotoshop) &&
  metadata.width && metadata.width <= 2048 &&
  metadata.height && metadata.height <= 2048
@@ -216,7 +217,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
  logger.info('creating web image');
 
  try {
- if (['image/jpeg', 'image/webp'].includes(type)) {
+ if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
  webpublic = await convertSharpToJpeg(img, 2048, 2048);
  } else if (['image/png'].includes(type)) {
  webpublic = await convertSharpToPng(img, 2048, 2048);
@@ -238,7 +239,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
  let thumbnail: IImage | null = null;
 
  try {
- if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(type)) {
+ if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(type)) {
  thumbnail = await convertSharpToWebp(img, 498, 280);
  } else {
  logger.debug('thumbnail not created (not an required file)');

packages/client/src/const.ts

@@ -7,6 +7,7 @@ export const FILE_TYPE_BROWSERSAFE = [
  'image/gif',
  'image/jpeg',
  'image/webp',
+ 'image/avif',
  'image/apng',
  'image/bmp',
  'image/tiff',

packages/client/src/pages/user/index.photos.vue

@@ -47,6 +47,8 @@ function thumbnail(image: misskey.entities.DriveFile): string {
 onMounted(() => {
  const image = [
  'image/jpeg',
+ 'image/webp',
+ 'image/avif',
  'image/png',
  'image/gif',
  'image/apng',