Skip to content
This repository has been archived by the owner on May 22, 2024. It is now read-only.

feat: add in-source-config detection to listFunctions #899

Merged
merged 17 commits into from
Dec 21, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 64 additions & 15 deletions src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import { extname } from 'path'
import { Config } from './config'
import { FeatureFlags, getFlags } from './feature_flags'
import { FunctionSource } from './function'
import { getFunctionsFromPaths } from './runtimes'
import { getFunctionFromPath, getFunctionsFromPaths } from './runtimes'
import { findISCDeclarationsInPath, ISCValues } from './runtimes/node/in_source_config'
import { GetSrcFilesFunction, RuntimeName } from './runtimes/runtime'
import { listFunctionsDirectories, resolveFunctionsDirectories } from './utils/fs'

Expand All @@ -12,6 +13,7 @@ interface ListedFunction {
mainFile: string
runtime: RuntimeName
extension: string
schedule?: string
}

type ListedFunctionFile = ListedFunction & {
Expand All @@ -22,49 +24,96 @@ interface ListFunctionsOptions {
basePath?: string
config?: Config
featureFlags?: FeatureFlags
parseISC?: boolean
}

interface AugmentedFunctionSource extends FunctionSource {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[dust] Currently, this type only differs from FunctionSource in that it also includes ISC. Are you thinking it may be used in the future for other things? If we introduce new options to listFunctions, would that not be under a different conditional check than parseISC? And if so, do you think this should be called something like FunctionSourceWithISC?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the AugmentedFunctionSource type is really the input type to getListedFunction. it's FunctionSource + whatever may need to be added to assemble the return value of listFunctions and listFunction. If there was a new flag, let's say enablePartyMode, then we can extend AugmentedFunctionSource to have an partyModeResult?: Parteyyy property, and getListedFunction could use that to assemble the return value.

inSourceConfig?: ISCValues
}

const augmentWithISC = async (func: FunctionSource): Promise<AugmentedFunctionSource> => {
// ISC is currently only supported in JavaScript and TypeScript functions.
if (func.runtime.name !== 'js') {
return func
}

const inSourceConfig = await findISCDeclarationsInPath(func.mainFile)
return { ...func, inSourceConfig }
}

// List all Netlify Functions main entry files for a specific directory
const listFunctions = async function (
relativeSrcFolders: string | string[],
{ featureFlags: inputFeatureFlags }: { featureFlags?: FeatureFlags } = {},
{
featureFlags: inputFeatureFlags,
config,
parseISC = false,
}: { featureFlags?: FeatureFlags; config?: Config; parseISC?: boolean } = {},
) {
const featureFlags = getFlags(inputFeatureFlags)
const srcFolders = resolveFunctionsDirectories(relativeSrcFolders)
const paths = await listFunctionsDirectories(srcFolders)
const functions = await getFunctionsFromPaths(paths, { featureFlags })
const listedFunctions = [...functions.values()].map(getListedFunction)
return listedFunctions
const functionsMap = await getFunctionsFromPaths(paths, { featureFlags, config })
const functions = [...functionsMap.values()]
const augmentedFunctions = parseISC ? await Promise.all(functions.map(augmentWithISC)) : functions
return augmentedFunctions.map(getListedFunction)
}

// Finds a function at a specific path.
const listFunction = async function (
path: string,
{
featureFlags: inputFeatureFlags,
config,
parseISC = false,
}: { featureFlags?: FeatureFlags; config?: Config; parseISC?: boolean } = {},
) {
const featureFlags = getFlags(inputFeatureFlags)
const func = await getFunctionFromPath(path, { featureFlags, config })
if (!func) {
return
}

const augmentedFunction = parseISC ? await augmentWithISC(func) : func

return getListedFunction(augmentedFunction)
}

// List all Netlify Functions files for a specific directory
const listFunctionsFiles = async function (
relativeSrcFolders: string | string[],
{ basePath, config, featureFlags: inputFeatureFlags }: ListFunctionsOptions = {},
{ basePath, config, featureFlags: inputFeatureFlags, parseISC = false }: ListFunctionsOptions = {},
) {
const featureFlags = getFlags(inputFeatureFlags)
const srcFolders = resolveFunctionsDirectories(relativeSrcFolders)
const paths = await listFunctionsDirectories(srcFolders)
const functions = await getFunctionsFromPaths(paths, { config, featureFlags })
const functionsMap = await getFunctionsFromPaths(paths, { config, featureFlags })
const functions = [...functionsMap.values()]
const augmentedFunctions = parseISC ? await Promise.all(functions.map(augmentWithISC)) : functions
const listedFunctionsFiles = await Promise.all(
[...functions.values()].map((func) => getListedFunctionFiles(func, { basePath, featureFlags })),
augmentedFunctions.map((func) => getListedFunctionFiles(func, { basePath, featureFlags })),
)

return listedFunctionsFiles.flat()
}

const getListedFunction = function ({ runtime, name, mainFile, extension }: FunctionSource): ListedFunction {
return { name, mainFile, runtime: runtime.name, extension }
const getListedFunction = function ({
runtime,
name,
mainFile,
extension,
config,
inSourceConfig,
}: AugmentedFunctionSource): ListedFunction {
return { name, mainFile, runtime: runtime.name, extension, schedule: inSourceConfig?.schedule ?? config.schedule }
}

const getListedFunctionFiles = async function (
func: FunctionSource,
func: AugmentedFunctionSource,
options: { basePath?: string; featureFlags: FeatureFlags },
): Promise<ListedFunctionFile[]> {
const srcFiles = await getSrcFiles({ ...func, ...options })
const { name, mainFile, runtime } = func

return srcFiles.map((srcFile) => ({ srcFile, name, mainFile, runtime: runtime.name, extension: extname(srcFile) }))
return srcFiles.map((srcFile) => ({ ...getListedFunction(func), srcFile, extension: extname(srcFile) }))
}

const getSrcFiles: GetSrcFilesFunction = async function ({ extension, runtime, srcPath, ...args }) {
Expand All @@ -82,6 +131,6 @@ const getSrcFiles: GetSrcFilesFunction = async function ({ extension, runtime, s
})
}

export { listFunctions, listFunctionsFiles }
export { listFunctions, listFunction, listFunctionsFiles }

export { zipFunction, zipFunctions } from './zip'
36 changes: 18 additions & 18 deletions src/runtimes/go/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { SourceFile } from '../../function'
import { cachedLstat, cachedReaddir, FsCache } from '../../utils/fs'
import { nonNullable } from '../../utils/non_nullable'
import { detectBinaryRuntime } from '../detect_runtime'
import { FindFunctionsInPathsFunction, Runtime, ZipFunction } from '../runtime'
import { FindFunctionInPathFunction, FindFunctionsInPathsFunction, Runtime, ZipFunction } from '../runtime'

import { build } from './builder'

Expand All @@ -33,27 +33,27 @@ const detectGoFunction = async ({ fsCache, path }: { fsCache: FsCache; path: str
}

const findFunctionsInPaths: FindFunctionsInPathsFunction = async function ({ featureFlags, fsCache, paths }) {
const functions = await Promise.all(
paths.map(async (path) => {
const runtime = await detectBinaryRuntime({ fsCache, path })
const functions = await Promise.all(paths.map((path) => findFunctionInPath({ featureFlags, fsCache, path })))

if (runtime === 'go') {
return processBinary({ fsCache, path })
}
return functions.filter(nonNullable)
}

if (featureFlags.buildGoSource !== true) {
return
}
const findFunctionInPath: FindFunctionInPathFunction = async function ({ featureFlags, fsCache, path }) {
const runtime = await detectBinaryRuntime({ fsCache, path })

if (runtime === 'go') {
return processBinary({ fsCache, path })
}

const goSourceFile = await detectGoFunction({ fsCache, path })
if (featureFlags.buildGoSource !== true) {
return
}

if (goSourceFile) {
return processSource({ fsCache, mainFile: goSourceFile, path })
}
}),
)
const goSourceFile = await detectGoFunction({ fsCache, path })

return functions.filter(nonNullable)
if (goSourceFile) {
return processSource({ fsCache, mainFile: goSourceFile, path })
}
}

const processBinary = async ({ fsCache, path }: { fsCache: FsCache; path: string }): Promise<SourceFile> => {
Expand Down Expand Up @@ -114,6 +114,6 @@ const zipFunction: ZipFunction = async function ({ config, destFolder, filename,
return { config, path: destPath }
}

const runtime: Runtime = { findFunctionsInPaths, name: 'go', zipFunction }
const runtime: Runtime = { findFunctionsInPaths, findFunctionInPath, name: 'go', zipFunction }

export default runtime
48 changes: 37 additions & 11 deletions src/runtimes/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,16 @@ const findFunctionsInRuntime = async function ({
return { functions: augmentedFunctions, remainingPaths }
}

// An object to cache filesystem operations. This allows different functions
// to perform IO operations on the same file (i.e. getting its stats or its
// contents) without duplicating work.
const makeFsCache = (): FsCache => ({})

// The order of this array determines the priority of the runtimes. If a path
// is used by the first time, it won't be made available to the subsequent
// runtimes.
const RUNTIMES = [jsRuntime, goRuntime, rustRuntime]

/**
* Gets a list of functions found in a list of paths.
*/
Expand All @@ -73,21 +83,13 @@ const getFunctionsFromPaths = async (
featureFlags = defaultFlags,
}: { config?: Config; dedupe?: boolean; featureFlags?: FeatureFlags } = {},
): Promise<FunctionMap> => {
// An object to cache filesystem operations. This allows different functions
// to perform IO operations on the same file (i.e. getting its stats or its
// contents) without duplicating work.
const fsCache = {}

// The order of this array determines the priority of the runtimes. If a path
// is used by the first time, it won't be made available to the subsequent
// runtimes.
const runtimes = [jsRuntime, goRuntime, rustRuntime]
const fsCache = makeFsCache()

// We cycle through the ordered array of runtimes, passing each one of them
// through `findFunctionsInRuntime`. For each iteration, we collect all the
// functions found plus the list of paths that still need to be evaluated,
// using them as the input for the next iteration until the last runtime.
const { functions } = await runtimes.reduce(async (aggregate, runtime) => {
const { functions } = await RUNTIMES.reduce(async (aggregate, runtime) => {
const { functions: aggregateFunctions, remainingPaths: aggregatePaths } = await aggregate
const { functions: runtimeFunctions, remainingPaths: runtimePaths } = await findFunctionsInRuntime({
dedupe,
Expand All @@ -110,4 +112,28 @@ const getFunctionsFromPaths = async (
return new Map(functionsWithConfig)
}

export { getFunctionsFromPaths }
/**
* Gets a list of functions found in a list of paths.
*/
const getFunctionFromPath = async (
path: string,
{ config, featureFlags = defaultFlags }: { config?: Config; featureFlags?: FeatureFlags } = {},
): Promise<FunctionSource | undefined> => {
const fsCache = makeFsCache()

for (const runtime of RUNTIMES) {
// eslint-disable-next-line no-await-in-loop
const func = await runtime.findFunctionInPath({ path, fsCache, featureFlags })
if (func) {
return {
...func,
runtime,
config: getConfigForFunction({ config, func: { ...func, runtime } }),
}
}
}

return undefined
}

export { getFunctionsFromPaths, getFunctionFromPath }
9 changes: 5 additions & 4 deletions src/runtimes/node/finder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import locatePath from 'locate-path'

import { SourceFile } from '../../function'
import { nonNullable } from '../../utils/non_nullable'
import { FindFunctionsInPathsFunction, FindFunctionInPathFunction } from '../runtime'

const pLstat = promisify(lstat)

Expand All @@ -23,8 +24,8 @@ const sortByExtension = (fA: SourceFile, fB: SourceFile) => {
return indexB - indexA
}

const findFunctionsInPaths = async function ({ paths }: { paths: string[] }) {
const functions = await Promise.all(paths.map(getFunctionAtPath))
const findFunctionsInPaths: FindFunctionsInPathsFunction = async function ({ paths, fsCache, featureFlags }) {
const functions = await Promise.all(paths.map((path) => findFunctionInPath({ path, fsCache, featureFlags })))

// It's fine to mutate the array since its scope is local to this function.
const sortedFunctions = functions.filter(nonNullable).sort((fA, fB) => {
Expand All @@ -48,7 +49,7 @@ const findFunctionsInPaths = async function ({ paths }: { paths: string[] }) {
return sortedFunctions
}

const getFunctionAtPath = async function (srcPath: string): Promise<SourceFile | undefined> {
const findFunctionInPath: FindFunctionInPathFunction = async function ({ path: srcPath }) {
const filename = basename(srcPath)

if (filename === 'node_modules') {
Expand Down Expand Up @@ -93,4 +94,4 @@ const getMainFile = async function (srcPath: string, filename: string, stat: Sta
}
}

export { findFunctionsInPaths, getFunctionAtPath }
export { findFunctionsInPaths, findFunctionInPath }
3 changes: 2 additions & 1 deletion src/runtimes/node/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { FeatureFlags } from '../../feature_flags'
import { GetSrcFilesFunction, Runtime, ZipFunction } from '../runtime'

import { getBundler } from './bundlers'
import { findFunctionsInPaths } from './finder'
import { findFunctionsInPaths, findFunctionInPath } from './finder'
import { findISCDeclarationsInPath } from './in_source_config'
import { detectEsModule } from './utils/detect_es_module'
import { createAliases as createPluginsModulesPathAliases, getPluginsModulesPath } from './utils/plugin_modules_path'
Expand Down Expand Up @@ -170,6 +170,7 @@ const zipWithFunctionWithFallback: ZipFunction = async ({ config = {}, ...parame

const runtime: Runtime = {
findFunctionsInPaths,
findFunctionInPath,
getSrcFiles: getSrcFilesWithBundler,
name: 'js',
zipFunction: zipWithFunctionWithFallback,
Expand Down
17 changes: 16 additions & 1 deletion src/runtimes/runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@ type FindFunctionsInPathsFunction = (args: {
paths: string[]
}) => Promise<SourceFile[]>

type FindFunctionInPathFunction = (args: {
featureFlags: FeatureFlags
fsCache: FsCache
path: string
}) => Promise<SourceFile | undefined>

type GetSrcFilesFunction = (
args: {
basePath?: string
Expand Down Expand Up @@ -49,9 +55,18 @@ type ZipFunction = (

interface Runtime {
findFunctionsInPaths: FindFunctionsInPathsFunction
findFunctionInPath: FindFunctionInPathFunction
getSrcFiles?: GetSrcFilesFunction
name: RuntimeName
zipFunction: ZipFunction
}

export { FindFunctionsInPathsFunction, GetSrcFilesFunction, Runtime, RuntimeName, ZipFunction, ZipFunctionResult }
export {
FindFunctionInPathFunction,
FindFunctionsInPathsFunction,
GetSrcFilesFunction,
Runtime,
RuntimeName,
ZipFunction,
ZipFunctionResult,
}
Loading