add ability to override x-robots-tag header

itslenny · itslenny · commit 5970c6335ef6 · 2025-10-07T18:39:39.000-04:00
diff --git a/src/http/routes/object/createObject.ts b/src/http/routes/object/createObject.ts
@@ -32,6 +32,7 @@ interface createObjectRequestInterface extends RequestGenericInterface {
     'content-type': string
     'cache-control'?: string
     'x-upsert'?: string
+    'x-robots-tag'?: string
   }
 }
 
diff --git a/src/http/routes/object/getObject.ts b/src/http/routes/object/getObject.ts
@@ -71,17 +71,18 @@ async function requestHandler(
 
   if (bucket.public) {
     // request is authenticated but we still use the superUser as we don't need to check RLS
-    obj = await request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id, version')
+    obj = await request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id, version, metadata')
   } else {
     // request is authenticated use RLS
-    obj = await request.storage.from(bucketName).findObject(objectName, 'id, version')
+    obj = await request.storage.from(bucketName).findObject(objectName, 'id, version, metadata')
   }
 
   return request.storage.renderer('asset').render(request, response, {
     bucket: storageS3Bucket,
     key: s3Key,
     version: obj.version,
     download,
+    xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
     signal: request.signals.disconnect.signal,
   })
 }
diff --git a/src/http/routes/object/getPublicObject.ts b/src/http/routes/object/getPublicObject.ts
@@ -55,7 +55,7 @@ export default async function routes(fastify: FastifyInstance) {
         request.storage.asSuperUser().findBucket(bucketName, 'id,public', {
           isPublic: true,
         }),
-        request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version'),
+        request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version,metadata'),
       ])
 
       // send the object from s3
@@ -70,6 +70,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
         signal: request.signals.disconnect.signal,
       })
     }
diff --git a/src/http/routes/object/getSignedObject.ts b/src/http/routes/object/getSignedObject.ts
@@ -83,14 +83,15 @@ export default async function routes(fastify: FastifyInstance) {
       const obj = await request.storage
         .asSuperUser()
         .from(bucketName)
-        .findObject(objParts.join('/'), 'id,version')
+        .findObject(objParts.join('/'), 'id,version,metadata')
 
       return request.storage.renderer('asset').render(request, response, {
         bucket: storageS3Bucket,
         key: s3Key,
         version: obj.version,
         download,
         expires: new Date(exp * 1000).toUTCString(),
+        xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
         signal: request.signals.disconnect.signal,
       })
     }
diff --git a/src/http/routes/object/updateObject.ts b/src/http/routes/object/updateObject.ts
@@ -29,6 +29,7 @@ interface updateObjectRequestInterface extends RequestGenericInterface {
     'content-type': string
     'cache-control'?: string
     'x-upsert'?: string
+    'x-robots-tag'?: string
   }
 }
 
diff --git a/src/http/routes/render/renderAuthenticatedImage.ts b/src/http/routes/render/renderAuthenticatedImage.ts
@@ -51,7 +51,7 @@ export default async function routes(fastify: FastifyInstance) {
       const { bucketName } = request.params
       const objectName = request.params['*']
 
-      const obj = await request.storage.from(bucketName).findObject(objectName, 'id,version')
+      const obj = await request.storage.from(bucketName).findObject(objectName, 'id,version,metadata')
 
       const s3Key = request.storage.location.getKeyLocation({
         tenantId: request.tenantId,
@@ -73,6 +73,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
         signal: request.signals.disconnect.signal,
       })
     }
diff --git a/src/http/routes/render/renderPublicImage.ts b/src/http/routes/render/renderPublicImage.ts
@@ -55,7 +55,7 @@ export default async function routes(fastify: FastifyInstance) {
         request.storage.asSuperUser().findBucket(bucketName, 'id,public', {
           isPublic: true,
         }),
-        request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version'),
+        request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version,metadata'),
       ])
 
       const s3Key = `${request.tenantId}/${bucketName}/${objectName}`
@@ -74,6 +74,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
         signal: request.signals.disconnect.signal,
       })
     }
diff --git a/src/http/routes/render/renderSignedImage.ts b/src/http/routes/render/renderSignedImage.ts
@@ -83,7 +83,7 @@ export default async function routes(fastify: FastifyInstance) {
       const obj = await request.storage
         .asSuperUser()
         .from(bucketName)
-        .findObject(objParts.join('/'), 'id,version')
+        .findObject(objParts.join('/'), 'id,version,metadata')
 
       const renderer = request.storage.renderer('image') as ImageRenderer
 
@@ -102,6 +102,7 @@ export default async function routes(fastify: FastifyInstance) {
           version: obj.version,
           download,
           expires: new Date(exp * 1000).toUTCString(),
+          xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
           signal: request.signals.disconnect.signal,
         })
     }
diff --git a/src/storage/backend/adapter.ts b/src/storage/backend/adapter.ts
@@ -31,6 +31,7 @@ export type ObjectMetadata = {
   eTag: string
   contentRange?: string
   httpStatusCode?: number
+  xRobotsTag?: string
 }
 
 export type UploadPart = {
diff --git a/src/storage/renderer/renderer.ts b/src/storage/renderer/renderer.ts
@@ -10,6 +10,7 @@ export interface RenderOptions {
   version: string | undefined
   download?: string
   expires?: string
+  xRobotsTag?: string
   object?: Obj
   signal?: AbortSignal
 }
@@ -80,7 +81,7 @@ export abstract class Renderer {
       .header('ETag', data.metadata.eTag)
       .header('Content-Length', data.metadata.contentLength)
       .header('Last-Modified', data.metadata.lastModified?.toUTCString())
-      .header('X-Robots-Tag', 'none')
+      .header('X-Robots-Tag', options.xRobotsTag || 'none')
 
     if (options.expires) {
       response.header('Expires', options.expires)
diff --git a/src/storage/uploader.ts b/src/storage/uploader.ts
@@ -20,6 +20,7 @@ interface FileUpload {
   mimeType: string
   cacheControl: string
   isTruncated: () => boolean
+  xRobotsTag?: string
   userMetadata?: Record<string, any>
 }
 
@@ -112,6 +113,10 @@ export class Uploader {
         request.signal
       )
 
+      if (request.file.xRobotsTag) {
+        objectMetadata.xRobotsTag = request.file.xRobotsTag
+      }
+
       if (file.isTruncated()) {
         throw ERRORS.EntityTooLarge()
       }
@@ -301,6 +306,7 @@ export async function fileUploadFromRequest(
   }
 ): Promise<FileUpload & { maxFileSize: number }> {
   const contentType = request.headers['content-type']
+  const xRobotsTag = request.headers['x-robots-tag'] as string | undefined
 
   let body: Readable
   let userMetadata: Record<string, any> | undefined
@@ -388,6 +394,7 @@ export async function fileUploadFromRequest(
     isTruncated,
     userMetadata,
     maxFileSize,
+    xRobotsTag,
   }
 }
 
diff --git a/src/test/bucket.test.ts b/src/test/bucket.test.ts
@@ -361,6 +361,7 @@ describe('testing public bucket functionality', () => {
       url: `/object/public/public-bucket/favicon.ico`,
     })
     expect(publicResponse.statusCode).toBe(200)
+    expect(publicResponse.headers['x-robots-tag']).toBe('none')
     expect(publicResponse.headers['etag']).toBe('abc')
     expect(publicResponse.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT')
 
diff --git a/src/test/object.test.ts b/src/test/object.test.ts
@@ -61,6 +61,7 @@ describe('testing GET object', () => {
     })
     expect(response.statusCode).toBe(200)
     expect(response.headers['etag']).toBe('abc')
+    expect(response.headers['x-robots-tag']).toBe('none')
     expect(response.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT')
     expect(S3Backend.prototype.getObject).toBeCalled()
   })
@@ -2119,6 +2120,7 @@ describe('testing retrieving signed URL', () => {
       url: `/object/sign/${urlToSign}?token=${jwtToken}`,
     })
     expect(response.statusCode).toBe(200)
+    expect(response.headers['x-robots-tag']).toBe('none')
     expect(response.headers['etag']).toBe('abc')
     expect(response.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT')
   })
@@ -2521,3 +2523,128 @@ describe('testing list objects', () => {
     expect(responseJSON[1].name).toBe('sadcat-upload23.png')
   })
 })
+
+describe('x-robots-tag header', () => {
+  const X_ROBOTS_TEST_BUCKET = 'X_ROBOTS_TEST_BUCKET'
+  beforeAll(async () => {
+    appInstance = app()
+    await appInstance.inject({
+      method: 'POST',
+      url: `/bucket`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+      payload: {
+        name: X_ROBOTS_TEST_BUCKET,
+      },
+    })
+    appInstance.close()
+  })
+
+  afterAll(async () => {
+    appInstance = app()
+    await appInstance.inject({
+      method: 'POST',
+      url: `/bucket/${X_ROBOTS_TEST_BUCKET}/empty`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    await appInstance.inject({
+      method: 'DELETE',
+      url: `/bucket/${X_ROBOTS_TEST_BUCKET}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    await appInstance.close()
+  })
+
+  test('defaults x-robots-tag header to none if not specified', async () => {
+    const objPath = `${X_ROBOTS_TEST_BUCKET}/test-file-1.txt`
+
+    await appInstance.inject({
+      method: 'POST',
+      url: `/object/${objPath}`,
+      payload: new File(['test'], 'file.txt'),
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+
+    const response = await appInstance.inject({
+      method: 'GET',
+      url: `/object/authenticated/${objPath}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    expect(response.headers['x-robots-tag']).toBe('none')
+  })
+
+  test('uses provided x-robots-tag header if set', async () => {
+    const objPath = `${X_ROBOTS_TEST_BUCKET}/test-file-2.txt`
+
+    await appInstance.inject({
+      method: 'POST',
+      url: `/object/${objPath}`,
+      payload: new File(['test'], 'file.txt'),
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+        'x-robots-tag': 'all',
+      },
+    })
+
+    const response = await appInstance.inject({
+      method: 'GET',
+      url: `/object/authenticated/${objPath}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    expect(response.headers['x-robots-tag']).toBe('all')
+  })
+
+  test('updates x-robots-tag header on upsert', async () => {
+    const objPath = `${X_ROBOTS_TEST_BUCKET}/test-file-3.txt`
+
+    await appInstance.inject({
+      method: 'POST',
+      url: `/object/${objPath}`,
+      payload: new File(['test'], 'file.txt'),
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+        'x-robots-tag': 'max-snippet: 10, notranslate',
+      },
+    })
+
+    const response = await appInstance.inject({
+      method: 'GET',
+      url: `/object/authenticated/${objPath}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    expect(response.headers['x-robots-tag']).toBe('max-snippet: 10, notranslate')
+
+    await appInstance.inject({
+      method: 'POST',
+      url: `/object/${objPath}`,
+      payload: new File(['test'], 'file.txt'),
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+        'x-upsert': 'true',
+        'x-robots-tag': 'nofollow',
+      },
+    })
+
+    const response2 = await appInstance.inject({
+      method: 'GET',
+      url: `/object/authenticated/${objPath}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    expect(response2.headers['x-robots-tag']).toBe('nofollow')
+  })
+})

Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@ interface createObjectRequestInterface extends RequestGenericInterface {`
`32`	`32`	`'content-type': string`
`33`	`33`	`'cache-control'?: string`
`34`	`34`	`'x-upsert'?: string`
	`35`	`+ 'x-robots-tag'?: string`
`35`	`36`	`}`
`36`	`37`	`}`
`37`	`38`
Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ interface updateObjectRequestInterface extends RequestGenericInterface {`
`29`	`29`	`'content-type': string`
`30`	`30`	`'cache-control'?: string`
`31`	`31`	`'x-upsert'?: string`
	`32`	`+ 'x-robots-tag'?: string`
`32`	`33`	`}`
`33`	`34`	`}`
`34`	`35`
Original file line number	Diff line number	Diff line change
`@@ -31,6 +31,7 @@ export type ObjectMetadata = {`
`31`	`31`	`eTag: string`
`32`	`32`	`contentRange?: string`
`33`	`33`	`httpStatusCode?: number`
	`34`	`+ xRobotsTag?: string`
`34`	`35`	`}`
`35`	`36`
`36`	`37`	`export type UploadPart = {`