-
Notifications
You must be signed in to change notification settings - Fork 171
/
MetaServer.prp
1824 lines (1546 loc) · 80.4 KB
/
MetaServer.prp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#
# $Id#
#
# Copyright 2008-2014,2016 Quantcast Corporation. All rights reserved.
#
# Author: Mike Ovsiannikov
#
# This file is part of Quantcast File System (QFS).
#
# Licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# The metaserver configuration.
# Additional information can be found in
# https://github.com/quantcast/qfs/wiki/Configuration-Reference.
# Client listener port.
metaServer.clientPort = 20000
# Client connection listener ip address to bind to.
# Use :: to bind to ipv6 address any.
# Default is empty, treated as 0.0.0.0 ipv4 address any, unless the following
# parameter metaServer.clientIpV6Only set to 1
# metaServer.clientIp =
# Accept ipv4 client connections.
# Default is 0, enable acception ipv4 connection. Only has effect if
# metaServer.clientIp left empty, or set to ::
# metaServer.clientIpV6Only = 0
# Chunk server connection listener ip address to bind to.
# Use :: to bind to ipv6 address any.
# Default is empty, treated as 0.0.0.0 ipv4 address any, unless the following
# parameter metaServer.chunkServerIpV6Only set to 1
# metaServer.chunkServerIp =
# Accept ipv4 chunk servers connections.
# Default is 0, enable acception ipv4 connection. Only has effect if
# metaServer.chunkServerIp left empty, or set to ::
# metaServer.chunkServerIpV6Only = 0
# Chunk server listener port.
metaServer.chunkServerPort = 30000
# Meta serve transactions log directory.
metaServer.logDir = meta/transaction_logs
# Meta server checkpoint directory.
metaServer.cpDir = meta/checkpoint
# Allow to automatically create an empty file system if checkpoint file does
# not exist.
# The default is 0, as under the normal circumstances where the file system
# content is of value, completely losing checkpoint, transaction log, and
# automatically creating an empty fs will have the same effect as conventional
# "mkfs". All chunks (blocks) will get deleted, and restoring the checkpoint
# and logs later won't be sufficient to recover the data.
# Use "-c" command line option to create new empty file system. For example:
# metaserver -c MetaServer.prp
# metaServer.createEmptyFs = 0
# Root directory permissions -- used only when the new file system created.
# metaServer.rootDirUser = 0
# metaServer.rootDirGroup = 0
# metaServer.rootDirMode = 0755
# Defaults for checkpoint and transaction log without permissions conversion on
# startup.
# metaServer.defaultLoadUser = 0
# metaServer.defaultLoadGroup = 0
# metaServer.defaultLoadFileMode = 0644
# metaServer.defaultLoadDirMode = 0755
# The size of the "client" thread pool.
# When set to greater than 0, dedicated threads to do client network io, request
# parsing, and response assembly are created. The thread pool size should
# usually be (at least one) less than the number of CPUs. "Client" threads help
# with processing large amount of ["short"] requests where more cpu used for
# context switch, network io, request parsing, and response assembly, than
# the cpu for the request processing itself. For example i-node attribute
# lookup, or write append chunk allocations that can be satisfied from the write
# append allocation cache.
# Default is 0 -- no dedicated "client" threads.
# metaServer.clientThreadCount = 0
# Meta server threads affinity.
# Presently only supported on linux.
# The first cpu index to set thread affinity to.
# The main thread will be assigned to the cpu at the specified index, then the
# next "client" thread will be assigned to the cpu index plus one and so on.
# For example with 2 client threads and start cpu index 0 the threads affinity
# would be 0 1 2 respectively.
# Useful on machines with more than one multi-core processor with shared dram
# cache. Assigning the threads to the same processor might help minimize dram
# cache misses.
# Default is off (start index less than 0) no thread affinity set.
# metaServer.clientThreadStartCpuAffinity = -1
# Meta server process max. locked memory.
# If set to a value greater than 0 then locked memory limit will be set to the
# specified value, and mlock(MCL_CURRENT|MCL_FUTURE) invoked.
# On linux running under non root user setting locked memory "hard" limit
# greater or equal to the specified value required. ulimit -l can be used for
# example.
# Default is 0 -- no memory locking.
# metaServer.maxLockedMemory = 0
# Size of [network] io buffer pool.
# The default buffer size is 4K, therefore the amount of memory is
# 4K * metaServer.bufferPool.partitionBuffers.
# All io buffers are allocated at startup.
# If memory locking enabled io buffers are locked in memory at startup.
# Default is 256K or 1GB on 64 bit system, and 32K or 128MB on 32 bit system.
# metaServer.bufferPool.partitionBuffers = 262144
# ==============================================================================
# The parameters below this line can be changed at runtime by editing the
# configuration file and sending meta server process HUP signal.
# Note that to restore parameter to default at run time the default value must
# be explicitly specified in the configuration file. In other words commenting
# out the parameter will not have any effect until restart.
# WORM mode.
# *****************************************************************************
# * This parameter is deprecated, and has no effect, please use qfsadmin or
# * qfstoggleworm to set WORM mode.
# * WORM mode is now stored in checkpoint and transaction logs.
# *****************************************************************************
# Write once, read many mode.
# In this mode only modification of files ".tmp" (without quotes) suffix is
# allowed.
# Typically the application would create and write the file with ".tmp" suffix,
# and then rename it so the destination file name will not have ".tmp " suffix.
# To delete a file without ".tmp" suffix the mode can be temporary turned off
# by the administrator. "qfstoggleworm" utility, or temporary configuration
# modification can be used to do that.
# Default is 0.
# metaServer.wormMode = 0
# Mininum number of connected / functional chunk servers before the file system
# can be used.
# Default is 1.
# metaServer.minChunkservers = 1
# Wait 30 sec for chunk servers to connect back after restarting, before file
# system considered fully functional.
metaServer.recoveryInterval = 30
# Ignore master/slave chunk server assignment for write append.
# Master/slave assignment can help with append replication 2, to avoid
# theoretically possible IO buffers resource deadlock when chunk server A is
# "slave" in one "AB" synchronous append replication chain, and chunk server B
# is "master" in another chunk "BA" synchronous replication.
# In practice such deadlocks should be rare enough to matter, and, if occur,
# are resolved by replication timeout mechanism.
# The downside of using master/slave assignment is that presently it only works
# with replication 2, and only half of the chunk server population will be
# accepting client's append requests.
# Default is "on" -- ignore.
# metaServer.appendPlacementIgnoreMasterSlave = 1
# For write append use the low order bit of the IP address for the chunk servers
# master/slave assignment. This scheme is works well if least significant bit of
# ip address uniformly distributes masters and slaves withing the rack,
# especially with "in rack" placement for append.
# Default is 0. Assign master / slave to keep number of masters and slaves
# equal. The obvious downside of this is that the assignment depends on the
# chunk servers connection order.
# metaServer.assignMasterByIp = 0
# Chunk server executables md5 sums white list.
# The chunk server sends its executable md5sum when it connects to the meta
# server. If the following space separated list is not empty and does not
# contain the chunk server executable md5 sum then the chunk server is
# instructed to exit or restart itself.
# This might be useful for upgrade or versions control.
# While the chunk server is connected to the meta server no md5 sum verification
# performed.
# Default is empty list.
# metaServer.chunkServerMd5sums =
# Unique file system id -- some name that uniquely identifies distributed file
# system instance.
# This is used to protect data loss / and or corruption in the case where chunk
# server(s) connect to the "wrong" meta server.
# The meta server will not accept connections from the chunk servers with a
# different "cluster key".
# Default is empty string.
metaServer.clusterKey = my-fs-unique-identifier
# Assign rack id by ip prefix -- ip address treated as strings.
# Valid rack id range is from 0 to 65535. Entries with rack ids outside of this
# range have no effect.
# The prefix can be positioned with trailing ??
# For example: 10.6.34.2?
# The rack id assigned on chunk server connect, and will not change until the
# chunk server re-connect. Therefore the configuration file changes will not
# have any effect until the chunk servers re-connect.
# Default is empty -- use rack id assigned in the chunk server config.
# metaServer.rackPrefixes =
# Example:
# 10.6.1.* -- rack 1, 10.6.2.* -- rack 2, 10.6.4.1? -- rack 4 etc.
# metaServer.rackPrefixes = 10.6.1. 1 10.6.2. 2 10.6.4.1? 4 10.6.4.1 5
# "Static" placement weights of the racks. The more weight and more chunk
# servers are in the rack the more likely the rack will be chosen for chunk
# allocation.
# Default is empty -- all weight are default to 1.
# metaServer.rackWeights =
# Example: Racks 1 and 2 have weight 1, rack 3 -- 0.9, rack 4 weight 1.2,
# rack 5 weight 1.5. All other rack weights are 1.
# metaServer.rackWeights = 1 1 2 1 3 0.9 4 1.2 5 1.5
# Various timeout settings.
# Extend write lease expiration time by 30 sec. in the case of the write master
# disconnect, to give it a chance to re-connect.
# Default is 30 sec. Production value is 60 sec.
# metaServer.leaseOwnerDownExpireDelay = 30
# Re-replication or recovery delay in seconds on chunk server down, to give
# chunk server a chance to re-connect.
# Default is 120 sec.
# metaServer.serverDownReplicationDelay = 120
# Chunk server heartbeat interval.
# Default is 30 sec.
# metaServer.chunkServer.heartbeatInterval = 30
# Chunk server operations timeouts.
# Heartbeat timeout results in declaring chunk server non operational, and
# closing connection.
# All other operations timeout are interpreted as the operation failure.
# The values are in seconds.
# The defaults:
# metaServer.chunkServer.heartbeatTimeout = 60
# metaServer.chunkServer.chunkReallocTimeout = 75
# metaServer.chunkServer.chunkAllocTimeout = 40
# metaServer.chunkServer.chunkReallocTimeout = 75
# metaServer.chunkServer.makeStableTimeout = 330
# metaServer.chunkServer.replicationTimeout = 330
# The current production values.
# metaServer.chunkServer.heartbeatInterval = 18
# metaServer.chunkServer.heartbeatTimeout = 30
# metaServer.chunkServer.chunkReallocTimeout = 18
# metaServer.chunkServer.chunkAllocTimeout = 18
# metaServer.chunkServer.makeStableTimeout = 60
# Other chunk server operations timeout.
# metaServer.chunkServer.requestTimeout = 600
# Chunk server space utilization placement threshold.
# Chunk servers with space utilization over this threshold are not considered
# as candidates for the chunk placement.
# Default is 0.95 or 95%.
# metaServer.maxSpaceUtilizationThreshold = 0.95
# Unix style permissions
# Space separated list of ip addresses of hosts where root user is allowed.
# Empty list means that root user is allowed on any host.
# Default is empty.
# metaServer.rootHosts =
# File modification time update resolution. Increasing the value will reduce
# number of corresponding transaction log writes with large files.
# Default is 1 sec.
# metaServer.MTimeUpdateResolution = 1
# Files access time update resolution in seconds. Increasing the value will
# reduce number of corresponding transaction log writes.
# File access time update considered when chunk or object store block read
# lease acquire or read lease renew completes successfully.
# Setting to a negative value turns off access time updates.
# Default is -1 -- no file access time update.
# metaServer.ATimeUpdateResolution = -1
# Directories access time update resolution in seconds. Increasing the value
# will reduce number of corresponding transaction log writes.
# Directory access time update considered every time when directory "read" /
# "listed", i.e. "read dir" or "read dir plus" RPC issued.
# Setting to a negative value turns off access time updates.
# Default is -1 -- n directories access time update.
# metaServer.dirATimeUpdateResolution = -1
# --------------- File create limits. ------------------------------------------
#
# Disallow specific file types. The list is space separate file type ids.
# Default is empty list. All valid file types are allowed.
# metaServer.createFileTypeExclude =
# Limit number of data stripes for all file types.If create attempt exceeds
# the limit the meta server returns "permission denied".
# Default is the max supported by the compile time constants.
# metaServer.maxDataStripeCount = 511
# Limit number of recovery stripes for all file types. If create attempt exceeds
# the limit the meta server returns "permission denied".
# Default is 32.
# Max supported by the compile time constants in common/kfstypes.h is 127.
# metaServer.maxRecoveryStripeCount = 32
# Limit number of data stripes for files with recovery.
# Default is 64.
# Max supported by the compile time constants in common/kfstypes.h is 511.
# metaServer.maxRSDataStripeCount = 64
# Max number of replicas for "regular / replicated" file with no recovery.
# If create, or change replication requests exceeds this limit then the meta
# server replaces the value with the value specified.
# metaServer.maxReplicasPerFile = 64
# Max number of replicas for RS (file with recovery).
# If create, or change replication requests exceeds this limit then the meta
# server replaces the value with the value specified.
# metaServer.maxReplicasPerRSFile = 64
# Force effective user to root. This effectively turns off all permissions
# control.
# Default is off.
# metaServer.forceEUserToRoot = 0
# Client backward compatibility.
# Defaults are no user and no group -- no backward compatibility.
# metaServer.defaultUser = 0xFFFFFFFF
# metaServer.defaultGroup = 0xFFFFFFFF
# metaServer.defaultFileMode = 0644
# metaServer.defaultDirMode = 0755
# The chunk server disconnects history size. Useful for monitoring.
# Default is 4096 slots / disconnect events.
# metaServer.maxDownServersHistorySize = 4096
# Space and placement re-balancing.
# Space re-balancing is controlled by the next two parameters (thresholds) below.
# Re-balancing constantly scans all chunks in the system and checks chunk
# placement within the replication or RS groups, and moves chunks from chunk
# servers that are above metaServer.maxRebalanceSpaceUtilThreshold to the chunk
# servers that are below metaServer.minRebalanceSpaceUtilThreshold.
# Default is 1 -- on.
# metaServer.rebalancingEnabled = 1
# Space re-balancing thresholds.
# Move chunk from the servers that exceed the
# metaServer.maxRebalanceSpaceUtilThreshold
# Default is 0.82
# metaServer.maxRebalanceSpaceUtilThreshold = 0.82
# Move chunks to server below metaServer.minRebalanceSpaceUtilThreshold.
# Default is 0.72.
# metaServer.minRebalanceSpaceUtilThreshold = 0.72
# Time interval in seconds between replication queues scans.
# The more often the scan is scheduled the more cpu can potentially use.
# Default is 5 sec.
# metaServer.replicationCheckInterval = 5
# Re-balance scan depth.
# Max number of chunks to scan in one partial scan. The more chunks are scanned
# the more cpu re-balance will use, and the "faster" it will scan the chunks.
# metaServer.maxRebalanceScan = 1024
# Single re-balance partial scan time limit.
# Default is 0.03 sec.
# metaServer.maxRebalanceRunTime = 0.03
# Minimum time between two consecutive re-balance partial scans.
# Default is 0.512 sec.
# metaServer.rebalanceRunInterval = 0.512
# Max. number of a single client connection requests in flight.
# The higher value might reduce cpu and alleviate "head of the line blocking"
# when single client connection shared between multiple concurrent file readers
# and writers, potentially at the cost of reducing "fairness" between the client
# connections. Increasing the value could also reduce number of context
# switches, and os scheduling overhead with the "client" threads enabled.
# Default is 16 if the "client" threads are enabled, and 1 otherwise.
# metaServer.clientSM.maxPendingOps = 16
# ------------------ Chunk placement parameters --------------------------------
# The metaServer.sortCandidatesByLoadAvg and
# metaServer.sortCandidatesBySpaceUtilization are mutially exclusive.
# metaServer.sortCandidatesBySpaceUtilization takes precedence over
# metaServer.sortCandidatesByLoadAvg if both set to 1
# When allocating (placing) a chunk prefer chunk servers with lower "load"
# metric over the chunk servers with the higher "load" metric.
# For the write intensive file systems turning this mode on is
# recommended.
# Default is 0. Do not take chunk server "load" metric into the account.
# metaServer.sortCandidatesByLoadAvg = 0
# When allocating (placing) a chunk prefer chunk servers with lower disk space
# utilization.
# Default is 0. Do not take space utilization into the account.
# metaServer.sortCandidatesBySpaceUtilization = 0
# When allocating (placing) a chunk do not consider chunk server with the "load"
# exceeding average load multiplied by metaServer.maxGoodCandidateLoadRatio.
# Default is 4.
# metaServer.maxGoodCandidateLoadRatio = 4
# When allocating (placing) a chunk do not consider chunk server with the "load"
# exceeding average "master" chunk server load multiplied by
# metaServer.maxGoodMasterLoadRatio if the chunk server is used as master (head
# or synchronous replication chain).
# Default is 4.
# metaServer.maxGoodMasterLoadRatio = 4
# When allocating (placing) a chunk do not consider chunk server with the "load"
# exceeding average "slave" load multiplied by metaServer.maxGoodSlaveLoadRatio
# if the chunk server is used as slave.
# Default is 4.
# metaServer.maxGoodSlaveLoadRatio = 4
# When allocating (placing) a chunk do not consider chunk server with the
# average number of chunks opened for write per drive (disk) exceeding average
# number of chunks opened for write across all disks / chunks servers multiplied
# by metaServer.maxWritesPerDriveRatio.
# Default is 1.5.
# metaServer.maxWritesPerDriveRatio = 1.5
# When allocating (placing) a chunk do not consider chunk server running on the
# same host as writer if the average number of chunks opened for write per drive
# (disk) exceeding average number of chunks opened for write across all disks /
# chunks servers multiplied by metaServer.maxLocalPlacementWeight.
# Default is 1.0.
# metaServer.maxLocalPlacementWeight = 1.0
# "In rack" placement for append and non append chunk allocations.
# Place chunk replicas on the same rack to save cross rack bandwidth at the cost
# of reduced reliability. Useful for temporary / scratch file systems.
# Default is 0.
# metaServer.inRackPlacementForAppend = 0
# "In rack" placement for non append files.
# Default is 0 - place replicas and chunks from the same RS blocks on different
# racks.
# metaServer.inRackPlacement = 0
# Limit number of re-replications (this does not include RS chunk recovery),
# that the given chunk server can be used as replication "source".
# Default is 10.
# metaServer.maxConcurrentReadReplicationsPerNode = 10
# Limit max concurrent chunk re-replications and RS recoveries per chunk server.
# Default is 5.
# metaServer.maxConcurrentWriteReplicationsPerNode = 5
#-------------------------------------------------------------------------------
# Order chunk replicas locations by the chunk "load average" metric in "get
# alloc" responses. The read client logic attempts to use replicas in this
# order.
# Default is 0. The replicas locations are shuffled randomly.
# metaServer.getAllocOrderServersByLoad = 0
# Delay recovery for the chunks that are past the logical end of file in files
# with Reed-Solomon redundant encoding.
# The delay is required to avoid starting recovery while the file is being
# written into, and the chunk sizes aren't known / final. The writer can stop
# writing into a file, and the corresponding chunks write leases might timed
# out, and will be automatically revoked. The existing writer logic sets logical
# EOF when it closes the file, before that the logical file size remains 0
# during write. (Unless it is re-write which is currently for all practical
# purposes not supported with RS files). The timeout below should be set to
# at least the max. practical file "write" time.
# Setting the timeout to a very large value will prevent processing the chunks
# sitting in the replication delayed queue from the "abandoned" files, i.e.
# files that the writer wrote something and then exited without closing the
# file.
# The parameter and the corresponding "delay" logic will likely be removed in
# future releases, and replaced with the write lease renew logic.
# Default is 6 hours or 21600 seconds.
# metaServer.pastEofRecoveryDelay = 21600
# -------------------------- Periodic checkpointing ----------------------------
# If set to -1 checkpoint is disabled. In such case "logcompactor" can be used
# periodically create new checkpoint from the transaction logs.
# Default is 3600 sec.
# metaServer.checkpoint.interval = 3600
# Checkpoint lock file name. Can be used to serialize checkpoint write and load
# with external programs, for example logcompactor.
# Default is empty -- no lock file used.
# metaServer.checkpoint.lockFileName =
# Max consecutive checkpoint write failures.
# Meta server will exit if checkpoint write fails
# metaServer.checkpoint.maxFailedCount times in the row for any reason (not
# enough disk space for example).
# Default is 2.
# metaServer.checkpoint.maxFailedCount = 2
# Checkpoint write timeout. Max time the checkpoint write can take before
# declaring write failure.
# Default is 3600 sec.
# metaServer.checkpoint.writeTimeoutSec = 3600
# Use synchronous mode to write checkpoint, i.e. tell host os to flush all data
# to disk prior to write system call return.
# The main purpose is to reduce the number of "dirty" / unwritten pages in the
# host os vm subsystem / file system buffer cache, therefore reducing memory
# contention and lowering the chances of paging out meta server and other
# processes with no memory locking.
# Default is on.
# metaServer.checkpoint.writeSync = 1
# Checkpoint write buffer size.
# The buffer size should be adequate with synchronous write mode enabled,
# especially if journal and data of host's file system are on the same spinning
# media device, in order to minimize the number of seeks.
# Default is 16MB.
# metaServer.checkpoint.writeBufferSize = 16777216
# --------------------------------- Audit log ----------------------------------
# All request headers and response status are logged.
# The audit log records are null ('\0') separated.
# The log could be useful for debugging and audit purposes.
# The logging require some cpu, but the main resource consumption is disk io.
# Default is off.
# metaServer.clientSM.auditLogging = 0
# Colon (:) separated file name prefixes to store log segments.
# Default is empty list.
# metaServer.auditLogWriter.logFilePrefixes =
# Maximum log segment size.
# Default is -1 -- unlimited.
# metaServer.auditLogWriter.maxLogFileSize = -1
# Maximum number of log segments.
# Default is -1 -- unlimited.
# metaServer.auditLogWriter.maxLogFiles = -1
# Max. time to wait for the log buffer to become available.
# When wait is enabled the request processing thread will wait for the log
# buffer disk io to complete. If the disk subsystem cannot keep up with the
# logging it will slow down the meta server request processing.
# Default is -1. Do not wait, drop log record instead.
# metaServer.auditLogWriter.waitMicroSec = -1
#-------------------------------------------------------------------------------
# ---------------------------------- Message log. ------------------------------
# Message log level FATAL, ALERT, CRIT, ERROR, WARN, NOTICE, INFO, DEBUG
# Default is DEBUG, except for non debug builds with NDEBUG defined INFO is
# default.
metaServer.msgLogWriter.logLevel = INFO
# Colon (:) separated file name prefixes to store log segments.
# Default is empty list. The default is to use file name from the command line
# or if none specified write into file descriptor 2 -- stderror.
# metaServer.msgLogWriter.logFilePrefixes =
# Maximum log segment size.
# Default is -1 -- unlimited.
# metaServer.msgLogWriter.maxLogFileSize = -1
# Maximum number of log segments.
# Default is -1 -- unlimited.
# metaServer.msgLogWriter.maxLogFiles = -1
# Max. time to wait for the log buffer to become available.
# When wait is enabled the request processing thread will wait for the log
# buffer disk io to complete. If the disk subsystem cannot keep up with the
# logging it will slow down the meta server request processing.
# Default is -1. Do not wait, drop log record instead.
# metaServer.msgLogWriter.waitMicroSec = -1
#-------------------------------------------------------------------------------
# -------------------- Chunk servers authentication. ---------------------------
#
# Authentication is off by default. Both X509 (ssl) and Kerberos authentication
# methods can be enabled at the same time. Chunk server can negotiate
# authentication method. If both Kerberos and X509 are configured on the chunk
# server and meta server then Kerberos authentication is used.
# Chunk and meta servers perform mutual authentication with authentication
# enabled.
#
# Use of X509 authentication is recommended in order to avoid KDC dependency.
# Chunk servers have to periodically request Kerberos tickets from KDC. The meta
# server enforces Kerberos ticket expiration time, by asking chunk server to
# re-authenticate when its ticket expires. Therefore KDC unavailability for any
# reason, including network communication outage, might result in chunk servers
# disconnects. Long enough KDC unavailability might result in unrecoverable data
# loss, due to the file system unability to perform replication and recovery
# in response to disk and node failures.
#
# Please see OpenSSL documentation for detailed description about X509
# authentication configuration.
# src/test-scripts/qfsmkcerts.sh might be used as a simple example how to create
# and use certificate authority, and X509 certificates.
# Maximum authenticated session lifetime. This limits authenticated session time
# for all authentication methods. In other words, the session [connection] must
# be re-authenticated if the authentication token (Kerberos ticket, or x509
# certificate) "end time" is reached or authenticated session exists longer than
# the value of this parameter.
# Default is 24 hours.
# metaServer.clientAuthentication.maxAuthenticationValidTimeSec = 86400
# Check chunk server authenticated name against the user and group database.
# If enabled then the authenticated name must be present in the user database in
# order for chunk server to be accepted.
# Default is 0 (off), use only black and white lists, if configured, see below.
# metaServer.CSAuthentication.useUserAndGroupDb = 0
# ================= X509 authentication ========================================
# Meta server's X509 certificate file in PEM format.
# metaServer.CSAuthentication.X509.X509PemFile =
# Password if X509 PEM file is encrypted.
# metaServer.CSAuthentication.X509.X509Password =
# Meta server's private key file.
# metaServer.CSAuthentication.X509.PKeyPemFile =
# Password if private key PEM file is encrypted.
# metaServer.CSAuthentication.X509.PKeyPassword =
# Certificate authorities file. Used for both chunk server certificate
# validation and to create certificate chain with meta server's X509
# certificate.
# metaServer.CSAuthentication.X509.CAFile =
# Certificate authorities directory can be used in addition to CAFile.
# For more detailed information please see SSL_CTX_load_verify_locations manual
# page. CAFile/CADir corresponds to CAfile/CApath in the man page.
# metaServer.CSAuthentication.X509.CADir =
# If set (the default) verify peer certificate, and declare error if peer, i.e.
# chunk server, does not preset "trusted" valid X509 certificate.
# Default is on.
# metaServer.CSAuthentication.X509.verifyPeer = 1
# OpenSSL cipher configuration.
# metaServer.CSAuthentication.X509.cipher = !ADH:!AECDH:!MD5:HIGH:@STRENGTH
# SSL/TLS session cache timeout. Session cache is only used with X509
# authentication method, with non default client or server side openssl options
# that turns off use of tls session tickets.
# Default is 4 hours.
# metaServer.CSAuthentication.X509.session.timeout = 14400
# The long integer value passed to SSL_CTX_set_options() call.
# See open ssl documentation for details.
# Default is the integer value that corresponds to SSL_OP_NO_COMPRESSION
# metaServer.clientAuthentication.X509.options =
# ================= Kerberos authentication =====================================
# Kerberos principal: service/host@realm
# Meta server's Kerberos principal [service/host@realm] service name part.
# metaServer.CSAuthentication.krb5.service =
# Meta server's Kerberos principal [service/host@realm] host name part.
# metaServer.CSAuthentication.krb5.host =
# Kerberos keytab file with the key(s) that corresponds to the meta server's
# principal.
# metaServer.CSAuthentication.krb5.keytab =
# Copy keytab into memory keytab, if supported by the kerberos versions, to
# improve performance, and avoid disk access.
# Default is on.
# metaServer.CSAuthentication.krb5.copyToMemKeytab = 1
# Client's (chunk server) principal "unparse" mode.
# Can be set to space separated combination of the following modes:
# short noRealm display
# The result of the principal conversion to string is used as client's
# (chunk server's) "authenticated name".
# The default is fully qualified principal name. For chunk servers it
# would typically be in the form of service/host@realm.
# The "unparsed" chunk server name is checked against "black" and "white" chunk
# server list names as described below.
# metaServer.CSAuthentication.krb5.princUnparseMode =
# OpenSSL cipher configuration for TLS-PSK authentication method. This method
# is used with TLS-PSK and with Kerberos authentication.
# metaServer.CSAuthentication.psk.cipherpsk = !ADH:!AECDH:!MD5:!3DES:PSK:@STRENGTH
# The long integer value passed to SSL_CTX_set_options() call.
# See open ssl documentation for details.
# Default is the integer value that corresponds to the logical OR of
# SSL_OP_NO_COMPRESSION and SSL_OP_NO_TICKET
# metaServer.CSAuthentication.psk.options =
# ================= PSK authentication =========================================
# PSK chunk server authentication is intended only for testing and possibly for
# small [test] clusters with very few chunk servers, where the same
# authentication credentials [PSK "key"] are used for for all chunk servers.
# Chunk server PSK key id. This string sent to the chunk as TLS PSK "hint", and
# also used as chunk server "authenticated name".
# This effectively overrides chunk server key id.
# If chunk server key id set to non empty string, then it can be left empty.
# In such case chunk server key id is used as authenticated name. The chunk
# server key id sent as "clear text" as part of ssl handshake, and is not
# "tied" in any way known to the meta server logic to the key id, therefore any
# "name" can be used. In other words the key is the only real security
# "credential" with this authentication scheme.
# The resulting chunk server name must not be empty, and pass "black" and
# "white" list check, see below.
# metaServer.CSAuthentication.psk.keyId =
# Chunk server PSK key (the "pre-shared-key"). The same key must be used on the
# chunk server side in order for psk authentication to work.
# The default is empty key -- PSK authentication is not enabled.
# The key must be base 64 encoded, i.e. it must be valid base 64 sequence.
# metaServer.CSAuthentication.psk.key =
# ================= Chunk servers's "black" and "white" lists ==================
# Chunk server's X509 common names and/or kerberos names, "black" ("revocation")
# list. If chunk server's authenticated name matches one of the name in this
# list the authentication will fail. The names in the list are must be
# separated by spaces. Names with white space symbols are not supported.
# metaServer.CSAuthentication.blackList =
# Chunk server's X509 common names and/or kerberos names, "white list". Unless
# the list is empty the chunk server's authenticated name must match one of the
# names in the list.
# metaServer.CSAuthentication.whiteList =
#-------------------------------------------------------------------------------
# -------------------- User / "client" authentication. -------------------------
# Client X509 and kerberos authentication parameters only differ from chunk
# server's authentication parameters by metaServer.clientAuthentication prefix.
# The defaults are identical to chunk server authentication.
# Maximum authenticated session lifetime. This limits authenticated session time
# for all authentication methods. In other words, the session [connection] must
# be re-authenticated if the authentication token (delegation token, Kerberos
# ticket, or x509 certificate) "end time" is reached or authenticated session
# exists longer than the value of this parameter.
# Default is 24 hours.
# metaServer.clientAuthentication.maxAuthenticationValidTimeSec = 86400
# ================= X509 authentication ========================================
# Meta server's X509 certificate file in PEM format.
# metaServer.clientAuthentication.X509.X509PemFile =
# Password if X509 PEM file is encrypted.
# metaServer.clientAuthentication.X509.X509Password =
# Meta server's private key file.
# metaServer.clientAuthentication.X509.PKeyPemFile =
# Password if private key PEM file is encrypted.
# metaServer.clientAuthentication.X509.PKeyPassword =
# Certificate authorities file. Used for both chunk server certificate
# validation and to create certificate chain with meta server's X509
# certificate.
# metaServer.clientAuthentication.X509.CAFile =
# Certificate authorities directory can be used in addition to CAFile.
# For more detailed information please see SSL_CTX_load_verify_locations manual
# page. CAFile/CADir corresponds to CAfile/CApath in the manual page.
# metaServer.clientAuthentication.X509.CADir =
# If set (the default) verify peer certificate, and declare error if peer, i.e.
# QFS client, does not preset certificate.
# Default is on.
# metaServer.clientAuthentication.X509.verifyPeer = 1
# OpenSSL cipher configuration for X509 authentication method.
# metaServer.clientAuthentication.X509.cipher = !ADH:!AECDH:!MD5:HIGH:@STRENGTH
# SSL/TLS session cache timeout. Session cache is only used with X509
# authentication method, with non default client or server side openssl options
# that turns off use of tls session tickets.
# Default is 4 hours.
# metaServer.clientAuthentication.X509.session.timeout = 14400
# The long integer value passed to SSL_CTX_set_options() call.
# See open ssl documentation for details.
# Default is the integer value that corresponds to SSL_OP_NO_COMPRESSION
# metaServer.clientAuthentication.X509.options =
# ================= Kerberos authentication =====================================
# Kerberos principal: service/host@realm
# Meta server's Kerberos principal [service/host@realm] service name part.
# metaServer.clientAuthentication.krb5.service =
# Meta server's Kerberos principal [service/host@realm] host name part.
# metaServer.clientAuthentication.krb5.host =
# Kerberos keytab file with the key(s) that corresponds to the meta server's
# principal.
# metaServer.clientAuthentication.krb5.keytab =
# Copy keytab into memory keytab, if supported by the kerberos versions, to
# improve performance, and avoid disk access.
# Default is on.
# metaServer.clientAuthentication.krb5.copyToMemKeytab = 1
# Client's principal "unparse" mode.
# Can be set to space separated combination of the following modes:
# short noRealm display
# The result of the principal conversion to string is used as client's
# (client's) "authenticated name".
# The default is fully qualified principal name. For users this typically would
# it would be in the form user@realm
# The resulting authentication name should match password database, the meta
# server host uses. The recommended value is "short', discard realm if it
# matches the default the kerberos configuration's default realm.
# metaServer.clientAuthentication.krb5.princUnparseMode =
# OpenSSL cipher configuration for TLS-PSK authentication method. This method
# is used with delegation and with Kerberos authentication.
# metaServer.clientAuthentication.psk.cipherpsk = !ADH:!AECDH:!MD5:!3DES:PSK:@STRENGTH
# The long integer value passed to SSL_CTX_set_options() call.
# See open ssl documentation for details.
# Default is the integer value that corresponds to the logical OR of
# SSL_OP_NO_COMPRESSION and SSL_OP_NO_TICKET
# metaServer.clientAuthentication.psk.options =
# The following two parameters and respective defaults are intended to allow
# non authenticated access for the meta server web UI from the local host.
# Space separated list of host ips that RPC listed in the next parameter are
# permitted with no authentication.
# Default is 127.0.0.1
# metaServer.clientAuthentication.noAuthOpsHostIps = 127.0.0.1
# Space separated list of RPC names that allowed with no authentication, if the
# client's host ip obtained with getpeername() call matches one of the ips in
# the preceding list.
# Default is RPCs used by the meta server web UI.
# metaServer.clientAuthentication.noAuthOps = PING GET_CHUNK_SERVERS_COUNTERS GET_CHUNK_SERVER_DIRS_COUNTERS GET_REQUEST_COUNTERS DISCONNECT
# ================= Client's "black" and "white" lists =========================
# Client's (user) X509 common names and/or kerberos names, "black"
# ("revocation") list. If client's authenticated name matches one of the name in
# this list the authentication will fail. The names in the list are must be
# separated by spaces. Names with white space symbols are not supported.
# metaServer.clientAuthentication.blackList =
# Client's X509 common names and/or kerberos names, "white list". Unless
# the list is empty the client's authenticated name must match one of the
# names in the list.
# metaServer.clientAuthentication.whiteList =
# ================== Delegation ================================================
#
# Delegation token expiration time limit.
# Default is 24 hours.
# metaServer.clientAuthentication.maxDelegationValidForTimeSec = 86400
# Do not limit delegation token end time to the meta server session credentials
# (Kerberos ticket or X509 certificate) end time.
# Default is 0.
# metaServer.clientAuthentication.delegationIgnoreCredEndTime = 0
================================================================================
# Allow to use "clear text" communication mode by performing SSL/TLS shutdown
# immediately after successful authentication completion. If enabled, the QFS
# client's corresponding setting defines the communication mode between the
# client and the write master. The "clear text" communication mode between chunk
# servers (synchronous replication, re-replication, and chunk recovery) will be
# used if this parameter set to "on".
# Using this mode might make sense in order reduce chunk server CPU utilization
# and/or possibly increase IO throughput, in the cases where chunk server
# communication channel is considered to have adequate security for the purpose
# at hands.
# Default is "off" / "no"
# metaServer.clientCSAllowClearText = 0
# Chunk server access token maximum lifetime.
# Chunk server access token time defines chunk access time limit.
# Chunk access tokens have 10 min time limit -- twice chunk lease time. The
# chunk server access token effectively defines maximum client and chunk server
# to chunk server connections lifetimes. The client and chunk servers attempt
# to obtain and use a new chunk server access token before the current token
# expires, and re-open connection with the newly obtained token.
# Default is 2 hours.
# metaServer.CSAccessValidForTimeSec = 7200
# The meta server limits the write lease end time to the max of the current time
# plus the value of the following parameter, and the authentication end time.
# The parameter is intended primarily for testing, to avoid spurious write
# retries with authentication maximum life time set to very small value -- 5sec.
# (The short authentication lifetime is used in order to test the
# re-authentication logic.)
# metaServer.minWriteLeaseTimeSec = 600
#-------------------------------------------------------------------------------
# -------------------- User and group configuration. ---------------------------
# User and group database parameters.
# The meta server host's user and group configuration used for QFS file system
# when QFS authentication is enabled. The user and group database is used to map
# "authenticated names" obtained with Kerberos and X509 authentication methods
# to user and group ids, and to establish group membership. Authenticated names
# that have no corresponding user id, or user id that have no corresponding
# "user name" are considered invalid, and as the result the authentication
# fails.
# User and group id with value 4294967295 have special treatment. Access always
# denied for users with such id.
# Root user entry with name "root" and id 0 added if not present in the the user
# database, unless explicitly excluded with metaServer.userAndGroup.excludeUser
# parameter.
# With authentication enabled QFS client library does not use host's local user
# and group database, the meta server's host database is effectively used by all
# QFS clients.
# Minimal user id to include in user name to id mapping.
# Default is 0.
# metaServer.userAndGroup.minUserId = 0
# Maximum user id to include in user name to id mapping.
# Default is 4294967295.
# metaServer.userAndGroup.maxUserId = 4294967295
# Minimal group id to include in group name to group id mapping.
# Default is 0.
# metaServer.userAndGroup.minGroupId = 0
# Maximum group id to include in group name to group id mapping.
# Default is 4294967295.
# metaServer.userAndGroup.maxGroupId = 4294967295
# Omit entries with user names if it has one of the specified prefixes.
# metaServer.userAndGroup.omitUserPrefix =
# Omit entries with group names if it has one of the specified prefixes.
# Default is empty list.
# metaServer.userAndGroup.omitGroupPrefix =
# Update / re-read user and group to id mappings with every N seconds.
# By default periodic updates are effectively disabled. The parameter reload
# with HUP signal can be used to trigger user and group information update.
# Default is 315360000.
# metaServer.userAndGroup.updatePeriodSec = 315360000
# Disable user and group initial loading and/or reloading.
# Default is enabled.
# metaServer.userAndGroup.disable = 0
# Space separated list of the user names to exclude when loading or updating
# user database.
# Default is empty list.
# metaServer.userAndGroup.excludeUser =
# Space separated list of the group names to exclude when loading or updating
# group database.
# Default is empty list.
# metaServer.userAndGroup.excludeGroup =
# Space separated list of the group names, where members of these groups
# have effective user id 0 -- root.
# Default is empty list.
# metaServer.userAndGroup.rootGroups =
# Space separated list of the user names, where such users have effective user
# id 0 -- root.
# User with name root and id 0 always added, even if it isn't present or
# excluded from the user database.
# Default is empty list.
# metaServer.userAndGroup.rootUsers =
# Space separated list of the user names. Specified users are allowed to
# perform meta server administrative requests: fsck, chunk server retire,
# toggle worm, recompute directory sizes, dump to chunk to servers map,
# dump replication candidates, check chunk leases, list open files.
# Default is root user.
# metaServer.userAndGroup.metaServerAdminUsers = root
# Space separated list of group names. Members of these groups are allowed to
# perform meta server administration described in the previous parameter's
# section.
# Default is empty list.
# metaServer.userAndGroup.metaServerAdminGroups =
# Space separated list of the user names. Specified users are allowed to
# perform meta server status inquiry requests: ping, up servers, meta stats, get
# chunk servers counters, get chunk directory counters, get meta server request
# counters.