-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathingestor.js
executable file
·186 lines (178 loc) · 7.66 KB
/
ingestor.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
const ingestor = require('commander');
const ingest = require('./ingest');
const ingest_mpu = require('./ingest_mpu');
const ingest_buckets = require('./ingest_buckets');
const readall = require('./readall');
ingestor.version('0.1');
ingestor.command('ingest')
.option('--endpoint <endpoint>', 'endpoint URL')
.option('--bucket <bucket>', 'bucket name')
.option('--profile [profile]', 'aws/credentials profile', 'default')
.option('--workers [n]', 'how many parallel workers', 10, parseInt)
.option('--count [n]', 'how many objects total', 100, parseInt)
.option('--size [n]', 'size of individual objects in bytes', 1000, parseInt)
.option('--prefix [prefix]', 'key prefix', '')
.option('--limit-per-delimiter [limit]',
'max number of object to group in a single delimiter range',
0, parseInt)
.option('--rate-limit [n]',
'limit rate of operations (in op/s)', 0, parseInt)
.option('--csv-stats [filename]', 'output file for stats in CSV format')
.option('--csv-stats-interval [n]',
'interval in seconds between each CSV stats output line',
10, parseInt)
.option('--one-object', 'hammer on a single object', false)
.option('--delete-after-put', 'send deletes after objects are put', false)
.option('--add-tags', 'add a random number of tags', false)
.option('--hash-keys', 'hash keys after the prefix with a MD5 sum to make them unordered', false)
.option('--keys-from-file [path]', 'read keys from file')
.option('--mpu-parts [nbparts]', 'create MPU objects with this many parts',
0, parseInt)
.option('--mpu-fuzz-repeat-complete-prob [probability]',
'repeat an extra time the complete-mpu requests with this probability ' +
'(it can lead to more than one extra complete-mpu for the same request)',
0, parseFloat)
.option('--verbose', 'increase verbosity', false)
.action(options => {
if (!options.endpoint ||
!options.bucket ||
isNaN(options.workers) ||
isNaN(options.count) ||
isNaN(options.size)) {
if (!options.endpoint) {
console.error('option --endpoint is missing');
}
if (!options.bucket) {
console.error('option --bucket is missing');
}
if (isNaN(options.workers)) {
console.error('value of option --workers must be an integer');
}
if (isNaN(options.count)) {
console.error('value of option --count must be an integer');
}
if (isNaN(options.size)) {
console.error('value of option --size must be an integer');
}
ingestor.outputHelp();
process.exit(1);
}
ingest(options, code => process.exit(code));
});
ingestor.command('ingest_mpu')
.option('--endpoint <endpoint>', 'endpoint URL')
.option('--bucket <bucket>', 'bucket name')
.option('--profile [profile]', 'aws/credentials profile', 'default')
.option('--workers [n]', 'how many parallel workers', 10, parseInt)
.option('--parts [nparts]', 'number of parts', 10, parseInt)
.option('--size [n]', 'size of individual parts in bytes', 1000, parseInt)
.option('--prefix [prefix]', 'key prefix', '')
.option('--no-complete', 'do not complete the MPU', false)
.action(options => {
if (!options.endpoint ||
!options.bucket ||
isNaN(options.workers) ||
isNaN(options.parts) ||
isNaN(options.size)) {
if (!options.endpoint) {
console.error('option --endpoint is missing');
}
if (!options.bucket) {
console.error('option --bucket is missing');
}
if (isNaN(options.workers)) {
console.error('value of option --workers must be an integer');
}
if (isNaN(options.parts)) {
console.error('value of option --parts must be an integer');
}
if (isNaN(options.size)) {
console.error('value of option --size must be an integer');
}
ingestor.outputHelp();
process.exit(1);
}
ingest_mpu(options, code => process.exit(code));
});
ingestor.command('ingest_buckets')
.option('--endpoint <endpoint>', 'endpoint URL')
.option('--profile [profile]', 'aws/credentials profile', 'default')
.option('--workers [n]', 'how many parallel workers', 10, parseInt)
.option('--count [n]', 'how many objects total', 100, parseInt)
.option('--prefix [prefix]', 'bucket prefix', '')
.option('--rate-limit [n]',
'limit rate of operations (in op/s)', 0, parseInt)
.option('--csv-stats [filename]', 'output file for stats in CSV format')
.option('--csv-stats-interval [n]',
'interval in seconds between each CSV stats output line',
10, parseInt)
.action(options => {
if (!options.endpoint ||
isNaN(options.workers) ||
isNaN(options.count)) {
if (!options.endpoint) {
console.error('option --endpoint is missing');
}
if (isNaN(options.workers)) {
console.error('value of option --workers must be an integer');
}
if (isNaN(options.count)) {
console.error('value of option --count must be an integer');
}
ingestor.outputHelp();
process.exit(1);
}
ingest_buckets(options, code => process.exit(code));
});
ingestor.command('readall')
.option('--endpoint <endpoint>', 'endpoint URL')
.option('--bucket <bucket>', 'bucket name')
.option('--prefix <prefix>', 'key prefix')
.option('--limit-per-delimiter [limit]',
'max number of object to group in a single delimiter range',
0, parseInt)
.option('--profile [profile]', 'aws/credentials profile', 'default')
.option('--workers [n]', 'how many parallel workers', 10, parseInt)
.option('--count [n]', 'how many objects total', 100, parseInt)
.option('--rate-limit [n]',
'limit rate of operations (in op/s)', 0, parseInt)
.option('--csv-stats [filename]', 'output file for stats in CSV format')
.option('--csv-stats-interval [n]',
'interval in seconds between each CSV stats output line',
10, parseInt)
.option('--random',
'randomize reads, while still reading all keys exactly once',
false)
.option('--keys-from-file [path]', 'read keys from file')
.action(options => {
if (!options.endpoint ||
!options.bucket ||
!options.prefix ||
isNaN(options.workers) ||
isNaN(options.count)) {
if (!options.endpoint) {
console.error('option --endpoint is missing');
}
if (!options.bucket) {
console.error('option --bucket is missing');
}
if (!options.prefix) {
console.error('option --prefix is missing');
}
if (isNaN(options.workers)) {
console.error('value of option --workers must be an integer');
}
if (isNaN(options.count)) {
console.error('value of option --count must be an integer');
}
ingestor.outputHelp();
process.exit(1);
}
readall(options, code => process.exit(code));
});
const commandName = process.argv[2];
if (!ingestor.commands.find(cmd => cmd._name === commandName)) {
ingestor.outputHelp();
process.exit(1);
}
ingestor.parse(process.argv);