Skip to content

Commit

Permalink
feat(async): Better parallelization
Browse files Browse the repository at this point in the history 
Benchmark test 'input as glob pattern' fix
Better cross-platform process spawn handling in benchmarks
Npm test script refactor

fixes #10
  • Loading branch information
@FRSgit
FRSgit authored and Jakub Freisler committed Oct 20, 2019
1 parent 0145d44 commit 2fc9978
Show file tree
Hide file tree
Showing 7 changed files with 158 additions and 481 deletions.
21 changes: 18 additions & 3 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,19 @@
dist: trusty
sudo: required
language: node_js
node_js:
- "node"
- "lts/*"
cache: yarn
matrix:
include:
- name: "Standard linting"
script: yarn standard
node_js: "lts/*"
- name: "Unit tests & coverage"
script: yarn test
node_js:
- "node"
- "lts/*"
- name: "Benchmark tests"
script: yarn test:benchmark
node_js:
- "node"
- "lts/*"
19 changes: 9 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,23 +230,22 @@ FRS-replace a b --content abcd -o foo_replaced.js
FRS-replace a b -i foo.js | <next-command>
```

## Benchmarks
## Benchmarks (Node v10.11.0)
#### input as glob pattern [1000 iterations x 100 repetitions]
| Library (best&nbsp;bolded) | Execution time [s] | Difference percentage (comparing&nbsp;to&nbsp;best&nbsp;time) |
| --- | --- | --- |
| **FRS-replace async** | 0.07656150 | 0.0000% |
| FRS-replace sync | 0.31196953 | 307.4757% |
| replace-in-file | 0.76240075 | 895.8017% |
| replace async | 0.11774627 | 53.7931% |
| replace sync | 0.91518713 | 1095.3620% |
| **FRS-replace async** | 0.15219495 | 0.0000% |
| FRS-replace sync | 0.62943626 | 313.5724% |
| replace-in-file | 1.80808213 | 1088.0040% |
| replace async | 0.21353394 | 40.3029% |
| replace sync | 1.53610279 | 909.2995% |
| replace-string | *N/A* | *N/A* |
#### input & replacement as strings [1000 iterations x 100 repetitions]
| Library (best&nbsp;bolded) | Execution time [s] | Difference percentage (comparing&nbsp;to&nbsp;best&nbsp;time) |
| --- | --- | --- |
| FRS-replace async | 0.00511845 | 77.4972% |
| **FRS-replace sync** | 0.00288368 | 0.0000% |
| FRS-replace async | 0.04848095 | 194.2632% |
| **FRS-replace sync** | 0.01647537 | 0.0000% |
| replace-in-file | *N/A* | *N/A* |
| replace async | *N/A* | *N/A* |
| replace sync | *N/A* | *N/A* |
| replace-string | 0.00292622 | 1.4752% |

| replace-string | 0.01843529 | 11.8961% |
179 changes: 88 additions & 91 deletions benchmark/multiple-file-replace.benchmark.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,43 @@ const content = `aąbcćdeęfg%hi
jklmn
oópqr,stuvwxyZ`
const tmpPrefixes = {
input: 'FRS-replace-replace-in',
output: 'FRS-replace-replace-out'
input: 'FRS-replace-replace-in-',
output: 'FRS-replace-replace-out-'
}
const defaults = {
inputReadOptions: 'utf8',
outputWriteOptions: 'utf8',
inputJoinString: '\n'
}
const repetitionsNo = 100000
const repetitionsNo = 50000
const iterationsNo = 1000
const testInput = {}
const inputFilesNo = 30
const testInput = {
FRSReplace: {
regex,
replacement
},

replace: {
regex,
replacement,
recursive: true,
silent: true
},

replaceAsync: {
regex,
replacement,
async: true,
recursive: true,
silent: true
},

replaceInFile: {
from: regex,
to: replacement
}
}
const testedLibraries = [
'FRS-replace async',
'FRS-replace sync',
Expand All @@ -36,11 +62,9 @@ const testedLibraries = [
'replace-string'
]

let dir, output, input

const readmeContent = fs.readFileSync('./README.md').toString()

let dir; let output; let inputs = []
let perfyResults = ''
let tmpFilesPromise

{
const dirObj = tmp.dirSync() // removing all files similar our tmp
Expand All @@ -53,62 +77,42 @@ let perfyResults = ''
].map(v => v + '*')
)
.forEach(fs.unlinkSync)
}

tap.beforeEach(async () => {
testInput.FRSReplace = {
regex,
replacement
}

testInput.replace = {
regex,
replacement,
recursive: true,
silent: true
}

testInput.replaceAsync = {
regex,
replacement,
async: true,
recursive: true,
silent: true
}

testInput.replaceInFile = {
from: regex,
to: replacement
const promises = []

for (let i = 0; i < inputFilesNo; ++i) {
promises.push(
tmp.file({ prefix: tmpPrefixes.input + i + '-', keep: true, dir })
.then(input => {
inputs.push(input)
return new Promise(resolve => {
fs.appendFile(input.path, content, { encoding: defaults.inputReadOptions }, resolve)
})
})
)
}

cleanInputs()

await tmp.file({ prefix: tmpPrefixes.input, keep: true, dir })
.then(
async f => {
input = f
return new Promise(
resolve => fs.appendFile(f.path, content, { encoding: defaults.inputReadOptions }, resolve)
)
})
})

const cleanInputs = (done) => {
input && input.cleanup()
input = undefined
done && done() // to be runned either by node-tap or manually
tmpFilesPromise = Promise.all(promises)
}

tap.afterEach((done) => {
tap.autoend(false)
tap.beforeEach(() => tmpFilesPromise)
tap.afterEach(done => {
fs.existsSync(output) && fs.unlinkSync(output)
cleanInputs()
done()
})

tap.test(`input as glob pattern [${iterationsNo} iterations x ${repetitionsNo / iterationsNo} repetitions]`, async ct => {
tap.teardown(() => {
inputs.forEach(input => input.cleanup)
inputs = []
const readmeContent = fs.readFileSync('./README.md').toString()

fs.writeFileSync('./README.md', readmeContent.replace(/(##\sBenchmarks \(Node )(?:.*?)(\)\s)[\s\S]*?(?:$|(?:\s##\s))/, `$1${process.version}$2${perfyResults}`))
})

tap.test(`input as glob pattern [${inputFilesNo} files x ${iterationsNo} iterations x ${repetitionsNo / iterationsNo} repetitions]`, async ct => {
const results = await multipleTests(ct, [
{
fn: () => { FRSreplace.async(testInput.FRSReplace) }, // IMPORTANT: test doesn't wait for function to finish, because replace (async) doesn't support that kind of behaviour (https://github.com/harthur/replace/issues/25)
fn: () => FRSreplace.async(testInput.FRSReplace),
before: () => (testInput.FRSReplace.input = `${dir}/${tmpPrefixes.input}*`)
},
{
Expand All @@ -119,12 +123,7 @@ tap.test(`input as glob pattern [${iterationsNo} iterations x ${repetitionsNo /
fn: () => replaceInFile(testInput.replaceInFile),
before: () => (testInput.replaceInFile.files = `${dir}/${tmpPrefixes.input}*`)
},
{
fn: () => replace(testInput.replaceAsync),
before: () => {
testInput.replaceAsync.paths = [dir.replace(/\\/g, '/')]
}
},
undefined, // IMPORTANT: test doesn't checks replace async, because it doesn't returns when (and if) file got replaced(https://github.com/harthur/replace/issues/25)
{
fn: () => replace(testInput.replace),
before: () => {
Expand All @@ -133,12 +132,12 @@ tap.test(`input as glob pattern [${iterationsNo} iterations x ${repetitionsNo /
},
undefined
])
const sortedResults = results.slice().sort(sortByNanoseconds)

ct.is((sortedResults[0].name.indexOf('FRS-replace') !== -1 || (sortedResults[1].name.indexOf('FRS-replace') !== -1 && sortedResults[1].avgPercentageDifference < 5)), true, 'FRS-replace should be the fastest or second, but at most with 5% difference to best')
ct.not(sortedResults[2].name.indexOf('FRS-replace sync'), -1, 'FRS-replace sync should be third (right after async replace)')
const result = outputPerfy(ct, results, results.slice().sort(sortByNumberVariable('fullNanoseconds'))[0])
const sortedResults = result.results.slice().sort(sortByNumberVariable('avgTime'))

outputPerfy(ct, results, sortedResults[0])
ct.is((sortedResults[0].name.indexOf('FRS-replace sync') !== -1 || (sortedResults[1].name.indexOf('FRS-replace sync') !== -1 && sortedResults[1].avgPercentageDifference < 5)), true, 'FRS-replace sync should be the fastest or second, but at most with 5% difference to best')
ct.is(sortedResults[0].name.indexOf('FRS-replace async') !== -1 || sortedResults[1].name.indexOf('FRS-replace async') !== -1, true, 'FRS-replace async should be the fastest or second')

ct.end()
})
Expand All @@ -165,19 +164,14 @@ tap.test(`input & replacement as strings [${iterationsNo} iterations x ${repetit
{ fn: () => replaceString(content, regex.source, replacement) }
])

const result = outputPerfy(ct, results, results.slice().sort(sortByNanoseconds)[0])

const sortedResults = result.results.slice().sort(sortByNanoseconds)
const result = outputPerfy(ct, results, results.slice().sort(sortByNumberVariable('fullNanoseconds'))[0])
const sortedResults = result.results.slice().sort(sortByNumberVariable('avgTime'))

ct.is((sortedResults[0].name.indexOf('FRS-replace') !== -1 || (sortedResults[1].name.indexOf('FRS-replace') !== -1 && sortedResults[1].avgPercentageDifference < 10)), true, 'FRS-replace should be the fastest or second, but at most with 10% difference to best')

ct.end()
})

tap.teardown(() => {
fs.writeFileSync('./README.md', readmeContent.replace(/(##\sBenchmarks\s\s)[\s\S]*?(?:$|(?:\s##\s))/, '$1' + perfyResults))
})

function outputPerfy (t, testResults, best) {
best = best.fullNanoseconds

Expand Down Expand Up @@ -218,7 +212,7 @@ function outputPerfy (t, testResults, best) {
)

perfyResults +=
'#### ' + result.name + '\n' +
'\n### ' + result.name + '\n\n' +
'| Library (best&nbsp;bolded) | Execution time [s] | Difference percentage (comparing&nbsp;to&nbsp;best&nbsp;time) |\n' +
'| --- | --- | --- |\n' +
result.results.reduce(
Expand Down Expand Up @@ -250,13 +244,12 @@ async function multipleTests (t, testCfgs, n, iterations) {

const testCfgLen = testCfgs.length

for (let i = 0; i < n; ++i) {
for (let k = testCfgLen - 1; k >= 0; --k) {
const { v: testCfg, i: index } = testCfgs[k]
const prevResult = results[index]
const libName = testedLibraries[index]

await t.test(`${t.name} - ${libName} #${i}`, async ct => {
for (let k = testCfgLen - 1; k >= 0; --k) {
const { v: testCfg, i: index } = testCfgs[k]
const prevResult = results[index]
const libName = testedLibraries[index]
await t.test(`${t.name} - ${libName}`, async ct => {
for (let i = 0; i < n; ++i) {
testCfg.before && testCfg.before()
const result = await singleTest(libName, testCfg.fn, iterations)

Expand All @@ -270,10 +263,9 @@ async function multipleTests (t, testCfgs, n, iterations) {
}
}
}

ct.end()
})
}
}
ct.end()
})
}

testCfgs.forEach(({ i: index }) => {
Expand Down Expand Up @@ -305,14 +297,19 @@ async function singleTest (name, test, n) {
return result
}

function sortByNanoseconds (a, b) {
if (a.fullNanoseconds === undefined) {
return b.fullNanoseconds === undefined ? 0 : 1
}
function sortByNumberVariable (varName) {
return (a, b) => {
a = a[varName]
b = b[varName]

if (b.fullNanoseconds === undefined) {
return -1
}
if (a === undefined || a === null) {
return b === undefined || b === null ? 0 : 1
}

if (b === undefined || b === null) {
return -1
}

return a.fullNanoseconds - b.fullNanoseconds
return a - b
}
}
Loading

0 comments on commit 2fc9978

Please sign in to comment.