feat: add new methods Dataset.createItemsPublicUrl & KeyValueStore.createKeysPublicUrl

package.json

@@ -64,6 +64,7 @@
     "dependencies": {
         "@apify/consts": "^2.25.0",
         "@apify/log": "^2.2.6",
+        "@apify/utilities": "^2.18.0",
         "@crawlee/types": "^3.3.0",
         "agentkeepalive": "^4.2.1",
         "async-retry": "^1.3.3",

src/resource_clients/dataset.ts

@@ -1,6 +1,7 @@
 import ow from 'ow';
 
 import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
+import { createStorageContentSignature } from '@apify/utilities';
 
 import type { ApifyApiError } from '../apify_api_error';
 import type { ApiClientSubResourceOptions } from '../base/api_client';
@@ -12,7 +13,7 @@ import {
 } from '../base/resource_client';
 import type { ApifyRequestConfig, ApifyResponse } from '../http_client';
 import type { PaginatedList } from '../utils';
-import { cast, catchNotFoundOrThrow, pluckData } from '../utils';
+import { applyQueryParamsToUrl, cast, catchNotFoundOrThrow, pluckData } from '../utils';
 
 export class DatasetClient<
     Data extends Record<string | number, any> = Record<string | number, unknown>,
@@ -163,6 +164,54 @@ export class DatasetClient<
         return undefined;
     }
 
+    /**
+     * Generates a URL that can be used to access dataset items.
+     *
+     * If the client has permission to access the dataset's URL signing key,
+     * the URL will include a signature to verify its authenticity.
+     *
+     * You can optionally control how long the signed URL should be valid using the `expiresInMillis` option.
+     * This value sets the expiration duration in milliseconds from the time the URL is generated.
+     * If not provided, the URL will not expire.
+     *
+     * Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
+     */
+    async createItemsPublicUrl(options: DatasetClientListItemOptions = {}, expiresInMillis?: number): Promise<string> {
+        ow(
+            options,
+            ow.object.exactShape({
+                clean: ow.optional.boolean,
+                desc: ow.optional.boolean,
+                flatten: ow.optional.array.ofType(ow.string),
+                fields: ow.optional.array.ofType(ow.string),
+                omit: ow.optional.array.ofType(ow.string),
+                limit: ow.optional.number,
+                offset: ow.optional.number,
+                skipEmpty: ow.optional.boolean,
+                skipHidden: ow.optional.boolean,
+                unwind: ow.optional.any(ow.string, ow.array.ofType(ow.string)),
+                view: ow.optional.string,
+            }),
+        );
+
+        const dataset = await this.get();
+
+        let createdItemsPublicUrl = new URL(this._url('items'));
+
+        if (dataset?.urlSigningSecretKey) {
+            const signature = createStorageContentSignature({
+                resourceId: dataset.id,
+                urlSigningSecretKey: dataset.urlSigningSecretKey,
+                expiresInMillis,
+            });
+            createdItemsPublicUrl.searchParams.set('signature', signature);
+        }
+
+        createdItemsPublicUrl = applyQueryParamsToUrl(createdItemsPublicUrl, options);
+
+        return createdItemsPublicUrl.toString();
+    }
+
     private _createPaginationList(response: ApifyResponse, userProvidedDesc: boolean): PaginatedList<Data> {
         return {
             items: response.data,
@@ -191,6 +240,8 @@ export interface Dataset {
     stats: DatasetStats;
     fields: string[];
     generalAccess?: STORAGE_GENERAL_ACCESS | null;
+    urlSigningSecretKey?: string | null;
+    itemsPublicUrl: string;
 }
 
 export interface DatasetStats {

src/resource_clients/key_value_store.ts

@@ -5,6 +5,7 @@ import type { JsonValue } from 'type-fest';
 
 import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
 import log from '@apify/log';
+import { createStorageContentSignature } from '@apify/utilities';
 
 import type { ApifyApiError } from '../apify_api_error';
 import type { ApiClientSubResourceOptions } from '../base/api_client';
@@ -15,7 +16,16 @@ import {
     SMALL_TIMEOUT_MILLIS,
 } from '../base/resource_client';
 import type { ApifyRequestConfig } from '../http_client';
-import { cast, catchNotFoundOrThrow, isBuffer, isNode, isStream, parseDateFields, pluckData } from '../utils';
+import {
+    applyQueryParamsToUrl,
+    cast,
+    catchNotFoundOrThrow,
+    isBuffer,
+    isNode,
+    isStream,
+    parseDateFields,
+    pluckData,
+} from '../utils';
 
 export class KeyValueStoreClient extends ResourceClient {
     /**
@@ -75,6 +85,48 @@ export class KeyValueStoreClient extends ResourceClient {
         return cast(parseDateFields(pluckData(response.data)));
     }
 
+    /**
+     * Generates a URL that can be used to access key-value store keys.
+     *
+     * If the client has permission to access the key-value store's URL signing key,
+     * the URL will include a signature to verify its authenticity.
+     *
+     * You can optionally control how long the signed URL should be valid using the `expiresInMillis` option.
+     * This value sets the expiration duration in milliseconds from the time the URL is generated.
+     * If not provided, the URL will not expire.
+     *
+     * Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
+     *
+     */
+    async createKeysPublicUrl(options: KeyValueClientListKeysOptions = {}, expiresInMillis?: number) {
+        ow(
+            options,
+            ow.object.exactShape({
+                limit: ow.optional.number,
+                exclusiveStartKey: ow.optional.string,
+                collection: ow.optional.string,
+                prefix: ow.optional.string,
+            }),
+        );
+
+        const store = await this.get();
+
+        let createdPublicKeysUrl = new URL(this._url('items'));
+
+        if (store?.urlSigningSecretKey) {
+            const signature = createStorageContentSignature({
+                resourceId: store.id,
+                urlSigningSecretKey: store.urlSigningSecretKey,
+                expiresInMillis,
+            });
+            createdPublicKeysUrl.searchParams.set('signature', signature);
+        }
+
+        createdPublicKeysUrl = applyQueryParamsToUrl(createdPublicKeysUrl, options);
+
+        return createdPublicKeysUrl.toString();
+    }
+
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
      *
@@ -255,6 +307,8 @@ export interface KeyValueStore {
     actRunId?: string;
     stats?: KeyValueStoreStats;
     generalAccess?: STORAGE_GENERAL_ACCESS | null;
+    urlSigningSecretKey?: string | null;
+    keysPublicUrl: string;
 }
 
 export interface KeyValueStoreStats {

src/utils.ts

@@ -260,3 +260,23 @@ export function asArray<T>(value: T | T[]): T[] {
 export type Dictionary<T = unknown> = Record<PropertyKey, T>;
 
 export type DistributiveOptional<T, K extends keyof T> = T extends any ? Omit<T, K> & Partial<Pick<T, K>> : never;
+
+/**
+ * Adds query parameters to a given URL based on the provided options object.
+ */
+export function applyQueryParamsToUrl(
+    url: URL,
+    options?: Record<string, string | number | boolean | string[] | undefined>,
+) {
+    for (const [key, value] of Object.entries(options ?? {})) {
+        // skip undefined values
+        if (value === undefined) continue;
+        // join array values with a comma
+        if (Array.isArray(value)) {
+            url.searchParams.set(key, value.join(','));
+            continue;
+        }
+        url.searchParams.set(key, String(value));
+    }
+    return url;
+}

test/datasets.test.js

@@ -327,5 +327,32 @@ describe('Dataset methods', () => {
             expect(browserRes).toEqual(res);
             validateRequest({}, { datasetId });
         });
+
+        describe('createItemsPublicUrl()', () => {
+            it('should include a signature in the URL when the caller has permission to access the signing secret key', async () => {
+                const datasetId = 'id-with-secret-key';
+                const res = await client.dataset(datasetId).createItemsPublicUrl();
+
+                expect(new URL(res).searchParams.get('signature')).toBeDefined();
+            });
+
+            it('should not include a signature in the URL when the caller lacks permission to access the signing secret key', async () => {
+                const datasetId = 'some-id';
+                const res = await client.dataset(datasetId).createItemsPublicUrl();
+
+                expect(new URL(res).searchParams.get('signature')).toBeNull();
+            });
+
+            it('includes provided options (e.g., limit and prefix) as query parameters', async () => {
+                const datasetId = 'id-with-secret-key';
+                const res = await client.dataset(datasetId).createItemsPublicUrl({ desc: true, limit: 10, offset: 5 });
+                const itemsPublicUrl = new URL(res);
+
+                expect(itemsPublicUrl.searchParams.get('desc')).toBe('true');
+                expect(itemsPublicUrl.searchParams.get('limit')).toBe('10');
+                expect(itemsPublicUrl.searchParams.get('offset')).toBe('5');
+                expect(itemsPublicUrl.searchParams.get('signature')).toBeDefined();
+            });
+        });
     });
 });

test/key_value_stores.test.js

@@ -553,5 +553,31 @@ describe('Key-Value Store methods', () => {
             expect(browserRes).toBeUndefined();
             validateRequest({}, { storeId, key });
         });
+
+        describe('createKeysPublicUrl()', () => {
+            it('should include a signature in the URL when the caller has permission to access the signing secret key', async () => {
+                const storeId = 'id-with-secret-key';
+                const res = await client.keyValueStore(storeId).createKeysPublicUrl();
+
+                expect(new URL(res).searchParams.get('signature')).toBeDefined();
+            });
+
+            it('should not include a signature in the URL when the caller lacks permission to access the signing secret key', async () => {
+                const storeId = 'some-id';
+                const res = await client.keyValueStore(storeId).createKeysPublicUrl();
+
+                expect(new URL(res).searchParams.get('signature')).toBeNull();
+            });
+
+            it('includes provided options (e.g., limit and prefix) as query parameters', async () => {
+                const storeId = 'id-with-secret-key';
+                const res = await client.keyValueStore(storeId).createKeysPublicUrl({ limit: 10, prefix: 'prefix' });
+                const keysPublicUrl = new URL(res);
+
+                expect(keysPublicUrl.searchParams.get('limit')).toBe('10');
+                expect(keysPublicUrl.searchParams.get('prefix')).toBe('prefix');
+                expect(keysPublicUrl.searchParams.get('signature')).toBeDefined();
+            });
+        });
     });
 });

test/mock_server/routes/datasets.js

@@ -7,7 +7,6 @@ const datasets = express.Router();
 const ROUTES = [
     { id: 'get-or-create-dataset', method: 'POST', path: '/' },
     { id: 'list-datasets', method: 'GET', path: '/' },
-    { id: 'get-dataset', method: 'GET', path: '/:datasetId' },
     { id: 'delete-dataset', method: 'DELETE', path: '/:datasetId' },
     { id: 'update-dataset', method: 'PUT', path: '/:datasetId' },
     { id: 'list-items', method: 'GET', path: '/:datasetId/items', type: 'responseJsonMock' },
@@ -17,4 +16,34 @@ const ROUTES = [
 
 addRoutes(datasets, ROUTES);
 
+/**
+ * GET /datasets/:datasetId
+ * Returns a specific dataset by its ID.
+ * If the dataset ID is 'id-with-secret-key', it returns a dataset with a URL signing secret key.
+ * If the dataset ID is '404', it returns a 404 error with a RECORD_NOT_FOUND type.
+ * Otherwise, it returns a dataset with an ID of 'get-dataset' (default).
+ */
+datasets.get('/:datasetId', (req, res) => {
+    const { datasetId } = req.params;
+
+    if (datasetId === 'id-with-secret-key') {
+        return res.json({
+            data: {
+                id: datasetId,
+                urlSigningSecretKey: 'secret-key-for-testing',
+            },
+        });
+    }
+
+    if (datasetId === '404') {
+        return res.status(404).json({ error: { type: 'record-not-found' } });
+    }
+
+    return res.json({
+        data: {
+            id: 'get-dataset',
+        },
+    });
+});
+
 module.exports = datasets;

test/mock_server/routes/key_value_stores.js

@@ -7,7 +7,6 @@ const keyValueStores = express.Router();
 const ROUTES = [
     { id: 'list-stores', method: 'GET', path: '/' },
     { id: 'get-or-create-store', method: 'POST', path: '/' },
-    { id: 'get-store', method: 'GET', path: '/:storeId' },
     { id: 'delete-store', method: 'DELETE', path: '/:storeId' },
     { id: 'update-store', method: 'PUT', path: '/:storeId' },
     { id: 'get-record', method: 'GET', path: '/:storeId/records/:key', type: 'responseJsonMock' },
@@ -24,4 +23,34 @@ const ROUTES = [
 
 addRoutes(keyValueStores, ROUTES);
 
+/**
+ * GET /key-value-stores/:storeId
+ * Returns a specific key-value store by its ID.
+ * If the store ID is 'id-with-secret-key', it returns a store with a URL signing secret key.
+ * If the store ID is '404', it returns a 404 error with a RECORD_NOT_FOUND type.
+ * Otherwise, it returns a store with an ID of 'get-store' (default).
+ */
+keyValueStores.get('/:storeId', (req, res) => {
+    const { storeId } = req.params;
+
+    if (storeId === 'id-with-secret-key') {
+        return res.json({
+            data: {
+                id: storeId,
+                urlSigningSecretKey: 'secret-key-for-testing',
+            },
+        });
+    }
+
+    if (storeId === '404') {
+        return res.status(404).json({ error: { type: 'record-not-found' } });
+    }
+
+    return res.json({
+        data: {
+            id: 'get-store',
+        },
+    });
+});
+
 module.exports = keyValueStores;