Skip to content

Commit

Permalink
feat(gatsby-source-filesystem): add createFileNodeFromBuffer (#14576)
Browse files Browse the repository at this point in the history
  • Loading branch information
superhawk610 authored and wardpeet committed Jun 25, 2019
1 parent b8fe293 commit aa21755
Show file tree
Hide file tree
Showing 6 changed files with 468 additions and 14 deletions.
90 changes: 89 additions & 1 deletion packages/gatsby-source-filesystem/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,11 @@ To filter by the `name` you specified in the config, use `sourceInstanceName`:

## Helper functions

`gatsby-source-filesystem` exports two helper functions:
`gatsby-source-filesystem` exports three helper functions:

- `createFilePath`
- `createRemoteFileNode`
- `createFileNodeFromBuffer`

### createFilePath

Expand Down Expand Up @@ -258,3 +259,90 @@ createRemoteFileNode({
name: "image",
})
```

### createFileNodeFromBuffer

When working with data that isn't already stored in a file, such as when querying binary/blob fields from a database, it's helpful to cache that data to the filesystem in order to use it with other transformers that accept files as input.

The `createFileNodeFromBuffer` helper accepts a `Buffer`, caches its contents to disk, and creates a file node that points to it.

## Example usage

The following example is adapted from the source of [`gatsby-source-mysql`](https://github.com/malcolm-kee/gatsby-source-mysql):

```js
// gatsby-node.js
const createMySqlNodes = require(`./create-nodes`)

exports.sourceNodes = async (
{ actions, createNodeId, store, cache },
config
) => {
const { createNode } = actions
const { conn, queries } = config
const { db, results } = await query(conn, queries)

try {
queries
.map((query, i) => ({ ...query, ___sql: results[i] }))
.forEach(result =>
createMySqlNodes(result, results, createNode, {
createNode,
createNodeId,
store,
cache,
})
)
db.end()
} catch (e) {
console.error(e)
db.end()
}
}

// create-nodes.js
const { createFileNodeFromBuffer } = require(`gatsby-source-filesystem`)
const createNodeHelpers = require(`gatsby-node-helpers`).default

const { createNodeFactory } = createNodeHelpers({ typePrefix: `mysql` })

function attach(node, key, value, ctx) {
if (Buffer.isBuffer(value)) {
ctx.linkChildren.push(parentNodeId =>
createFileNodeFromBuffer({
buffer: value,
store: ctx.store,
cache: ctx.cache,
createNode: ctx.createNode,
createNodeId: ctx.createNodeId,
})
)
value = `Buffer`
}

node[key] = value
}

function createMySqlNodes({ name, __sql, idField, keys }, results, ctx) {
const MySqlNode = createNodeFactory(name)
ctx.linkChildren = []

return __sql.forEach(row => {
if (!keys) keys = Object.keys(row)

const node = { id: row[idField] }

for (const key of keys) {
attach(node, key, row[key], ctx)
}

node = ctx.createNode(node)

for (const link of ctx.linkChildren) {
link(node.id)
}
})
}

module.exports = createMySqlNodes
```
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
jest.mock(`fs-extra`, () => {
return {
ensureDir: jest.fn(() => true),
writeFile: jest.fn((_f, _b, cb) => cb()),
stat: jest.fn(() => {
return {
isDirectory: jest.fn(),
}
}),
}
})
jest.mock(`../create-file-node`, () => {
return {
createFileNode: jest.fn(() => {
return { internal: {} }
}),
}
})

const { ensureDir, writeFile } = require(`fs-extra`)
const { createFileNode } = require(`../create-file-node`)
const createFileNodeFromBuffer = require(`../create-file-node-from-buffer`)

const createMockBuffer = content => {
const buffer = Buffer.alloc(content.length)
buffer.write(content)
return buffer
}

const createMockCache = () => {
return {
get: jest.fn(),
set: jest.fn(),
}
}

const bufferEq = (b1, b2) => Buffer.compare(b1, b2) === 0

describe(`create-file-node-from-buffer`, () => {
const defaultArgs = {
store: {
getState: jest.fn(() => {
return {
program: {
directory: `__whatever__`,
},
}
}),
},
createNode: jest.fn(),
createNodeId: jest.fn(),
}

describe(`functionality`, () => {
afterEach(() => jest.clearAllMocks())

const setup = ({
hash,
buffer = createMockBuffer(`some binary content`),
cache = createMockCache(),
} = {}) =>
createFileNodeFromBuffer({
...defaultArgs,
buffer,
hash,
cache,
})

it(`rejects when the buffer can't be read`, () => {
expect(setup({ buffer: null })).rejects.toEqual(
expect.stringContaining(`bad buffer`)
)
})

it(`caches the buffer's content locally`, async () => {
expect.assertions(2)

let output
writeFile.mockImplementationOnce((_f, buf, cb) => {
output = buf
cb()
})

const buffer = createMockBuffer(`buffer-content`)
await setup({ buffer })

expect(ensureDir).toBeCalledTimes(2)
expect(bufferEq(buffer, output)).toBe(true)
})

it(`uses cached file Promise for buffer with a matching hash`, async () => {
expect.assertions(3)

const cache = createMockCache()

await setup({ cache, hash: `same-hash` })
await setup({ cache, hash: `same-hash` })

expect(cache.get).toBeCalledTimes(1)
expect(cache.set).toBeCalledTimes(1)
expect(writeFile).toBeCalledTimes(1)
})

it(`uses cached file from previous run with a matching hash`, async () => {
expect.assertions(3)

const cache = createMockCache()
cache.get.mockImplementationOnce(() => `cached-file-path`)

await setup({ cache, hash: `cached-hash` })

expect(cache.get).toBeCalledWith(expect.stringContaining(`cached-hash`))
expect(cache.set).not.toBeCalled()
expect(createFileNode).toBeCalledWith(
expect.stringContaining(`cached-file-path`),
expect.any(Function),
expect.any(Object)
)
})
})

describe(`validation`, () => {
it(`throws on invalid inputs: createNode`, () => {
expect(() => {
createFileNodeFromBuffer({
...defaultArgs,
createNode: undefined,
})
}).toThrowErrorMatchingInlineSnapshot(
`"createNode must be a function, was undefined"`
)
})

it(`throws on invalid inputs: createNodeId`, () => {
expect(() => {
createFileNodeFromBuffer({
...defaultArgs,
createNodeId: undefined,
})
}).toThrowErrorMatchingInlineSnapshot(
`"createNodeId must be a function, was undefined"`
)
})

it(`throws on invalid inputs: cache`, () => {
expect(() => {
createFileNodeFromBuffer({
...defaultArgs,
cache: undefined,
})
}).toThrowErrorMatchingInlineSnapshot(
`"cache must be the Gatsby cache, was undefined"`
)
})

it(`throws on invalid inputs: store`, () => {
expect(() => {
createFileNodeFromBuffer({
...defaultArgs,
store: undefined,
})
}).toThrowErrorMatchingInlineSnapshot(
`"store must be the redux store, was undefined"`
)
})
})
})
Loading

0 comments on commit aa21755

Please sign in to comment.