Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 288a259

Browse files
authored
chore: move withTimeoutOption to core-utils (#3407)
Pulls non-grpc changes out of #3403 to ease the continued merging of master into that branch. - Moves withTimeoutOption into core-utils - Moves TimeoutError into core-utils - Adds missing ts project links - Adds more add-all tests to interface suite - Ignores unpassable tests for non-grpc or core implementations - Normalises mode and mtime in normalise-input function - Dedupes mtime normalisation between core and http client
1 parent d38b0ab commit 288a259

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

98 files changed

+425
-281
lines changed

packages/interface-ipfs-core/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
"is-ipfs": "^2.0.0",
4949
"iso-random-stream": "^1.1.1",
5050
"it-all": "^1.0.4",
51+
"it-buffer-stream": "^1.0.5",
5152
"it-concat": "^1.0.1",
5253
"it-drain": "^1.0.3",
5354
"it-last": "^1.0.4",

packages/interface-ipfs-core/src/add-all.js

+80
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ const { isNode } = require('ipfs-utils/src/env')
1515
const { getDescribe, getIt, expect } = require('./utils/mocha')
1616
const testTimeout = require('./utils/test-timeout')
1717
const uint8ArrayFromString = require('uint8arrays/from-string')
18+
const bufferStream = require('it-buffer-stream')
1819

1920
/** @typedef { import("ipfsd-ctl/src/factory") } Factory */
2021
/**
@@ -420,5 +421,84 @@ module.exports = (common, options) => {
420421
expect(files[0].cid.codec).to.equal('dag-pb')
421422
expect(files[0].size).to.equal(18)
422423
})
424+
425+
it('should support bidirectional streaming', async function () {
426+
let progressInvoked
427+
428+
const handler = (bytes, path) => {
429+
progressInvoked = true
430+
}
431+
432+
const source = async function * () {
433+
yield {
434+
content: 'hello',
435+
path: '/file'
436+
}
437+
438+
await new Promise((resolve) => {
439+
const interval = setInterval(() => {
440+
// we've received a progress result, that means we've received some
441+
// data from the server before we're done sending data to the server
442+
// so the streaming is bidirectional and we can finish up
443+
if (progressInvoked) {
444+
clearInterval(interval)
445+
resolve()
446+
}
447+
}, 10)
448+
})
449+
}
450+
451+
await drain(ipfs.addAll(source(), {
452+
progress: handler,
453+
fileImportConcurrency: 1
454+
}))
455+
456+
expect(progressInvoked).to.be.true()
457+
})
458+
459+
it('should error during add-all stream', async function () {
460+
const source = async function * () {
461+
yield {
462+
content: 'hello',
463+
path: '/file'
464+
}
465+
466+
yield {
467+
content: 'hello',
468+
path: '/file'
469+
}
470+
}
471+
472+
await expect(drain(ipfs.addAll(source(), {
473+
fileImportConcurrency: 1,
474+
chunker: 'rabin-2048--50' // invalid chunker parameters, validated after the stream starts moving
475+
}))).to.eventually.be.rejectedWith(/Chunker parameter avg must be an integer/)
476+
})
477+
478+
it('should add big files', async function () {
479+
const totalSize = 1024 * 1024 * 200
480+
const chunkSize = 1024 * 1024 * 99
481+
482+
const source = async function * () {
483+
yield {
484+
path: '/dir/file-200mb-1',
485+
content: bufferStream(totalSize, {
486+
chunkSize
487+
})
488+
}
489+
490+
yield {
491+
path: '/dir/file-200mb-2',
492+
content: bufferStream(totalSize, {
493+
chunkSize
494+
})
495+
}
496+
}
497+
498+
const results = await all(ipfs.addAll(source()))
499+
500+
expect(await ipfs.files.stat(`/ipfs/${results[0].cid}`)).to.have.property('size', totalSize)
501+
expect(await ipfs.files.stat(`/ipfs/${results[1].cid}`)).to.have.property('size', totalSize)
502+
})
423503
})
424504
}

packages/ipfs-cli/tsconfig.json

+5-2
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,17 @@
99
],
1010
"references": [
1111
{
12-
"path": "../ipfs-core-utils"
12+
"path": "../ipfs-core"
1313
},
1414
{
15-
"path": "../ipfs-core"
15+
"path": "../ipfs-core-utils"
1616
},
1717
{
1818
"path": "../ipfs-http-client"
1919
},
20+
{
21+
"path": "../ipfs-http-gateway"
22+
},
2023
{
2124
"path": "../ipfs-http-server"
2225
}

packages/ipfs-core-utils/package.json

+3
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
},
3939
"license": "MIT",
4040
"dependencies": {
41+
"any-signal": "^2.0.0",
4142
"blob-to-it": "^1.0.1",
4243
"browser-readablestream-to-it": "^1.0.1",
4344
"cids": "^1.0.0",
@@ -48,6 +49,8 @@
4849
"it-peekable": "^1.0.1",
4950
"multiaddr": "^8.0.0",
5051
"multiaddr-to-uri": "^6.0.0",
52+
"parse-duration": "^0.4.4",
53+
"timeout-abort-controller": "^1.1.1",
5154
"uint8arrays": "^1.1.0"
5255
},
5356
"devDependencies": {
+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
'use strict'
2+
3+
class TimeoutError extends Error {
4+
constructor (message = 'request timed out') {
5+
super(message)
6+
this.name = 'TimeoutError'
7+
this.code = TimeoutError.code
8+
}
9+
}
10+
11+
TimeoutError.code = 'ERR_TIMEOUT'
12+
exports.TimeoutError = TimeoutError

packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js

+5-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ const {
88
isBytes,
99
isBlob,
1010
isReadableStream,
11-
isFileObject
11+
isFileObject,
12+
mtimeToObject,
13+
modeToNumber
1214
} = require('./utils')
1315

1416
// eslint-disable-next-line complexity
@@ -105,7 +107,8 @@ module.exports = async function * normaliseInput (input, normaliseContent) {
105107
async function toFileObject (input, normaliseContent) {
106108
// @ts-ignore - Those properties don't exist on most input types
107109
const { path, mode, mtime, content } = input
108-
const file = { path: path || '', mode, mtime }
110+
111+
const file = { path: path || '', mode: modeToNumber(mode), mtime: mtimeToObject(mtime) }
109112

110113
if (content) {
111114
file.content = await normaliseContent(content)

packages/ipfs-core-utils/src/files/normalise-input/utils.js

+86-1
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,94 @@ function isFileObject (obj) {
3535
const isReadableStream = (value) =>
3636
value && typeof value.getReader === 'function'
3737

38+
/**
39+
* @param {any} mtime
40+
* @returns {{secs:number, nsecs:number}|undefined}
41+
*/
42+
function mtimeToObject (mtime) {
43+
if (mtime == null) {
44+
return undefined
45+
}
46+
47+
// Javascript Date
48+
if (mtime instanceof Date) {
49+
const ms = mtime.getTime()
50+
const secs = Math.floor(ms / 1000)
51+
52+
return {
53+
secs: secs,
54+
nsecs: (ms - (secs * 1000)) * 1000
55+
}
56+
}
57+
58+
// { secs, nsecs }
59+
if (Object.prototype.hasOwnProperty.call(mtime, 'secs')) {
60+
return {
61+
secs: mtime.secs,
62+
nsecs: mtime.nsecs
63+
}
64+
}
65+
66+
// UnixFS TimeSpec
67+
if (Object.prototype.hasOwnProperty.call(mtime, 'Seconds')) {
68+
return {
69+
secs: mtime.Seconds,
70+
nsecs: mtime.FractionalNanoseconds
71+
}
72+
}
73+
74+
// process.hrtime()
75+
if (Array.isArray(mtime)) {
76+
return {
77+
secs: mtime[0],
78+
nsecs: mtime[1]
79+
}
80+
}
81+
/*
82+
TODO: https://github.com/ipfs/aegir/issues/487
83+
84+
// process.hrtime.bigint()
85+
if (typeof mtime === 'bigint') {
86+
const secs = mtime / BigInt(1e9)
87+
const nsecs = mtime - (secs * BigInt(1e9))
88+
89+
return {
90+
secs: parseInt(secs),
91+
nsecs: parseInt(nsecs)
92+
}
93+
}
94+
*/
95+
}
96+
97+
/**
98+
* @param {any} mode
99+
* @returns {number|undefined}
100+
*/
101+
function modeToNumber (mode) {
102+
if (mode == null) {
103+
return undefined
104+
}
105+
106+
if (typeof mode === 'number') {
107+
return mode
108+
}
109+
110+
mode = mode.toString()
111+
112+
if (mode.substring(0, 1) === '0') {
113+
// octal string
114+
return parseInt(mode, 8)
115+
}
116+
117+
// decimal string
118+
return parseInt(mode, 10)
119+
}
120+
38121
module.exports = {
39122
isBytes,
40123
isBlob,
41124
isFileObject,
42-
isReadableStream
125+
isReadableStream,
126+
mtimeToObject,
127+
modeToNumber
43128
}

packages/ipfs-core-utils/src/index.js

+6
Original file line numberDiff line numberDiff line change
@@ -1 +1,7 @@
11
'use strict'
2+
3+
/**
4+
* @template {any[]} ARGS
5+
* @template R
6+
* @typedef {(...args: ARGS) => R} Fn
7+
*/
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/* eslint-disable no-unreachable */
2+
'use strict'
3+
4+
const TimeoutController = require('timeout-abort-controller')
5+
const { anySignal } = require('any-signal')
6+
const parseDuration = require('parse-duration').default
7+
const { TimeoutError } = require('./errors')
8+
9+
/**
10+
* @template {any[]} ARGS
11+
* @template {Promise<any> | AsyncIterable<any>} R - The return type of `fn`
12+
* @param {Fn<ARGS, R>} fn
13+
* @param {number} [optionsArgIndex]
14+
* @returns {Fn<ARGS, R>}
15+
*/
16+
function withTimeoutOption (fn, optionsArgIndex) {
17+
// eslint-disable-next-line
18+
return /** @returns {R} */(/** @type {ARGS} */...args) => {
19+
const options = args[optionsArgIndex == null ? args.length - 1 : optionsArgIndex]
20+
if (!options || !options.timeout) return fn(...args)
21+
22+
const timeout = typeof options.timeout === 'string'
23+
? parseDuration(options.timeout)
24+
: options.timeout
25+
26+
const controller = new TimeoutController(timeout)
27+
28+
options.signal = anySignal([options.signal, controller.signal])
29+
30+
const fnRes = fn(...args)
31+
// eslint-disable-next-line promise/param-names
32+
const timeoutPromise = new Promise((_resolve, reject) => {
33+
controller.signal.addEventListener('abort', () => {
34+
reject(new TimeoutError())
35+
})
36+
})
37+
38+
const start = Date.now()
39+
40+
const maybeThrowTimeoutError = () => {
41+
if (controller.signal.aborted) {
42+
throw new TimeoutError()
43+
}
44+
45+
const timeTaken = Date.now() - start
46+
47+
// if we have starved the event loop by adding microtasks, we could have
48+
// timed out already but the TimeoutController will never know because it's
49+
// setTimeout will not fire until we stop adding microtasks
50+
if (timeTaken > timeout) {
51+
controller.abort()
52+
throw new TimeoutError()
53+
}
54+
}
55+
56+
if (fnRes[Symbol.asyncIterator]) {
57+
// @ts-ignore
58+
return (async function * () {
59+
const it = fnRes[Symbol.asyncIterator]()
60+
61+
try {
62+
while (true) {
63+
const { value, done } = await Promise.race([it.next(), timeoutPromise])
64+
65+
if (done) {
66+
break
67+
}
68+
69+
maybeThrowTimeoutError()
70+
71+
yield value
72+
}
73+
} catch (err) {
74+
maybeThrowTimeoutError()
75+
76+
throw err
77+
} finally {
78+
controller.clear()
79+
80+
if (it.return) {
81+
it.return()
82+
}
83+
}
84+
})()
85+
}
86+
87+
// @ts-ignore
88+
return (async () => {
89+
try {
90+
const res = await Promise.race([fnRes, timeoutPromise])
91+
92+
maybeThrowTimeoutError()
93+
94+
return res
95+
} catch (err) {
96+
maybeThrowTimeoutError()
97+
98+
throw err
99+
} finally {
100+
controller.clear()
101+
}
102+
})()
103+
}
104+
}
105+
106+
module.exports = withTimeoutOption

packages/ipfs-core/package.json

-2
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@
5454
"dep-check": "aegir dep-check -i typescript -i interface-ipfs-core"
5555
},
5656
"dependencies": {
57-
"any-signal": "^2.0.0",
5857
"array-shuffle": "^1.0.1",
5958
"bignumber.js": "^9.0.0",
6059
"cbor": "^5.1.0",
@@ -116,7 +115,6 @@
116115
"parse-duration": "^0.4.4",
117116
"peer-id": "^0.14.1",
118117
"streaming-iterables": "^5.0.2",
119-
"timeout-abort-controller": "^1.1.1",
120118
"uint8arrays": "^1.1.0"
121119
},
122120
"devDependencies": {

0 commit comments

Comments
 (0)