Skip to content

feat!: return AsyncIterator for list() #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 10, 2023
Merged

feat!: return AsyncIterator for list() #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
32ad424
feat!: return `AsyncIterator` for `list()`
eduardoboucas Nov 10, 2023
a725749
chore: update README
eduardoboucas Nov 10, 2023
d37ba51
chore: update README
eduardoboucas Nov 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ not there was an object to delete.
await store.delete('my-key')
```

### `list(options?: { cursor?: string, directories?: boolean, paginate?: boolean. prefix?: string }): Promise<{ blobs: BlobResult[], directories: string[] }>`
### `list(options?: { directories?: boolean, paginate?: boolean. prefix?: string }): Promise<{ blobs: BlobResult[], directories: string[] }> | AsyncIterable<{ blobs: BlobResult[], directories: string[] }>`

Returns a list of blobs in a given store.

Expand Down Expand Up @@ -355,6 +355,29 @@ console.log(directories)
Note that we're only interested in entries under the `cats` directory, which is why we're using a trailing slash.
Without it, other keys like `catsuit` would also match.

For performance reasons, the server groups results into pages of up to 1,000 entries. By default, the `list()` method
automatically retrieves all pages, meaning you'll always get the full list of results.

If you'd like to handle this pagination manually, you can supply the `paginate` parameter, which makes `list()` return
an [`AsyncIterator`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncIterator).

```javascript
const blobs = []

for await (const entry of store.list({ paginate: true })) {
blobs.push(...entry.blobs)
}

// [
// { etag: "etag1", key: "cats/garfield.jpg" },
// { etag: "etag2", key: "cats/tom.jpg" },
// { etag: "etag3", key: "mice/jerry.jpg" },
// { etag: "etag4", key: "mice/mickey.jpg" },
// { etag: "etag5", key: "pink-panther.jpg" },
// ]
console.log(blobs)
```

## Contributing

Contributions are welcome! If you encounter any issues or have suggestions for improvements, please open an issue or
Expand Down
210 changes: 139 additions & 71 deletions src/list.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { describe, test, expect, beforeAll, afterEach } from 'vitest'
import { MockFetch } from '../test/mock_fetch.js'

import { getStore } from './main.js'
import type { ListResult } from './store.js'

beforeAll(async () => {
if (semver.lt(nodeVersion, '18.0.0')) {
Expand Down Expand Up @@ -171,7 +172,7 @@ describe('list', () => {
next_cursor: 'cursor_2',
}),
),
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_1&directories=true&context=${storeName}`,
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?directories=true&cursor=cursor_1&context=${storeName}`,
})
.get({
headers: { authorization: `Bearer ${apiToken}` },
Expand All @@ -188,7 +189,7 @@ describe('list', () => {
directories: ['dir3'],
}),
),
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_2&directories=true&context=${storeName}`,
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?directories=true&cursor=cursor_2&context=${storeName}`,
})
.get({
headers: { authorization: `Bearer ${apiToken}` },
Expand Down Expand Up @@ -279,30 +280,47 @@ describe('list', () => {
expect(mockStore.fulfilled).toBeTruthy()
})

test('Paginates manually with `cursor` if `paginate: false`', async () => {
const mockStore = new MockFetch().get({
headers: { authorization: `Bearer ${apiToken}` },
response: new Response(
JSON.stringify({
blobs: [
{
etag: 'etag3',
key: 'key3',
size: 3,
last_modified: '2023-07-18T12:59:06Z',
},
{
etag: 'etag4',
key: 'key4',
size: 4,
last_modified: '2023-07-18T12:59:06Z',
},
],
next_cursor: 'cursor_2',
}),
),
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_1&context=${storeName}`,
})
test('Returns an `AsyncIterator` if `paginate: true`', async () => {
const mockStore = new MockFetch()
.get({
headers: { authorization: `Bearer ${apiToken}` },
response: new Response(
JSON.stringify({
blobs: [
{
etag: 'etag1',
key: 'key1',
size: 1,
last_modified: '2023-07-18T12:59:06Z',
},
{
etag: 'etag2',
key: 'key2',
size: 2,
last_modified: '2023-07-18T12:59:06Z',
},
],
next_cursor: 'cursor_2',
}),
),
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?context=${storeName}`,
})
.get({
headers: { authorization: `Bearer ${apiToken}` },
response: new Response(
JSON.stringify({
blobs: [
{
etag: 'etag3',
key: 'key3',
size: 3,
last_modified: '2023-07-18T12:59:06Z',
},
],
}),
),
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_2&context=${storeName}`,
})

globalThis.fetch = mockStore.fetch

Expand All @@ -311,15 +329,20 @@ describe('list', () => {
token: apiToken,
siteID,
})
const result: ListResult = {
blobs: [],
directories: [],
}

const { blobs } = await store.list({
cursor: 'cursor_1',
paginate: false,
})
for await (const entry of store.list({ paginate: true })) {
result.blobs.push(...entry.blobs)
result.directories.push(...entry.directories)
}

expect(blobs).toEqual([
expect(result.blobs).toEqual([
{ etag: 'etag1', key: 'key1' },
{ etag: 'etag2', key: 'key2' },
{ etag: 'etag3', key: 'key3' },
{ etag: 'etag4', key: 'key4' },
])
expect(mockStore.fulfilled).toBeTruthy()
})
Expand All @@ -346,7 +369,7 @@ describe('list', () => {
last_modified: '2023-07-18T12:59:06Z',
},
],
directories: ['dir1'],
directories: [],
next_cursor: 'cursor_1',
}),
),
Expand All @@ -370,7 +393,7 @@ describe('list', () => {
last_modified: '2023-07-18T12:59:06Z',
},
],
directories: ['dir2'],
directories: [],
next_cursor: 'cursor_2',
}),
),
Expand All @@ -388,7 +411,7 @@ describe('list', () => {
last_modified: '2023-07-18T12:59:06Z',
},
],
directories: ['dir3'],
directories: [],
}),
),
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_2`,
Expand Down Expand Up @@ -430,15 +453,12 @@ describe('list', () => {
{ etag: 'etag5', key: 'key5' },
])

// @ts-expect-error `directories` is not part of the return type
expect(root.directories).toBe(undefined)
expect(root.directories).toEqual([])

const directory = await store.list({ prefix: 'dir2/' })

expect(directory.blobs).toEqual([{ etag: 'etag6', key: 'key6' }])

// @ts-expect-error `directories` is not part of the return type
expect(directory.directories).toBe(undefined)
expect(directory.directories).toEqual([])

expect(mockStore.fulfilled).toBeTruthy()
})
Expand Down Expand Up @@ -491,7 +511,7 @@ describe('list', () => {
next_cursor: 'cursor_2',
}),
),
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_1&directories=true`,
url: `${edgeURL}/${siteID}/${storeName}?directories=true&cursor=cursor_1`,
})
.get({
headers: { authorization: `Bearer ${edgeToken}` },
Expand All @@ -508,7 +528,7 @@ describe('list', () => {
directories: ['dir3'],
}),
),
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_2&directories=true`,
url: `${edgeURL}/${siteID}/${storeName}?directories=true&cursor=cursor_2`,
})
.get({
headers: { authorization: `Bearer ${edgeToken}` },
Expand Down Expand Up @@ -601,30 +621,70 @@ describe('list', () => {
expect(mockStore.fulfilled).toBeTruthy()
})

test('Paginates manually with `cursor` if `paginate: false`', async () => {
const mockStore = new MockFetch().get({
headers: { authorization: `Bearer ${edgeToken}` },
response: new Response(
JSON.stringify({
blobs: [
{
etag: 'etag3',
key: 'key3',
size: 3,
last_modified: '2023-07-18T12:59:06Z',
},
{
etag: 'etag4',
key: 'key4',
size: 4,
last_modified: '2023-07-18T12:59:06Z',
},
],
next_cursor: 'cursor_2',
}),
),
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_1`,
})
test('Returns an `AsyncIterator` if `paginate: true`', async () => {
const mockStore = new MockFetch()
.get({
headers: { authorization: `Bearer ${edgeToken}` },
response: new Response(
JSON.stringify({
blobs: [
{
etag: 'etag1',
key: 'key1',
size: 1,
last_modified: '2023-07-18T12:59:06Z',
},
{
etag: 'etag2',
key: 'key2',
size: 2,
last_modified: '2023-07-18T12:59:06Z',
},
],
next_cursor: 'cursor_2',
}),
),
url: `${edgeURL}/${siteID}/${storeName}`,
})
.get({
headers: { authorization: `Bearer ${edgeToken}` },
response: new Response(
JSON.stringify({
blobs: [
{
etag: 'etag3',
key: 'key3',
size: 3,
last_modified: '2023-07-18T12:59:06Z',
},
{
etag: 'etag4',
key: 'key4',
size: 4,
last_modified: '2023-07-18T12:59:06Z',
},
],
next_cursor: 'cursor_3',
}),
),
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_2`,
})
.get({
headers: { authorization: `Bearer ${edgeToken}` },
response: new Response(
JSON.stringify({
blobs: [
{
etag: 'etag5',
key: 'key5',
size: 5,
last_modified: '2023-07-18T12:59:06Z',
},
],
}),
),
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_3`,
})

globalThis.fetch = mockStore.fetch

Expand All @@ -634,16 +694,24 @@ describe('list', () => {
token: edgeToken,
siteID,
})
const result: ListResult = {
blobs: [],
directories: [],
}

const { blobs } = await store.list({
cursor: 'cursor_1',
paginate: false,
})
for await (const entry of store.list({ paginate: true })) {
result.blobs.push(...entry.blobs)
result.directories.push(...entry.directories)
}

expect(blobs).toEqual([
expect(result.blobs).toEqual([
{ etag: 'etag1', key: 'key1' },
{ etag: 'etag2', key: 'key2' },
{ etag: 'etag3', key: 'key3' },
{ etag: 'etag4', key: 'key4' },
{ etag: 'etag5', key: 'key5' },
])
expect(result.directories).toEqual([])
expect(mockStore.fulfilled).toBeTruthy()
})
})
Expand Down
Loading